{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1089,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002761477390403866,
      "grad_norm": 0.5198726654052734,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.4317269027233124,
      "log_odds_ratio": -0.5042418837547302,
      "logits/chosen": -0.5456271767616272,
      "logits/rejected": -0.10779725015163422,
      "logps/chosen": -1.998489260673523,
      "logps/rejected": -2.3799774646759033,
      "loss": 2.1834,
      "nll_loss": 2.1329703330993652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.199848935008049,
      "rewards/margins": 0.03814878687262535,
      "rewards/rejected": -0.23799772560596466,
      "step": 1
    },
    {
      "epoch": 0.005522954780807732,
      "grad_norm": 0.7381364107131958,
      "learning_rate": 4.587155963302753e-08,
      "log_odds_chosen": 0.4411306381225586,
      "log_odds_ratio": -0.4988963305950165,
      "logits/chosen": -0.6594648361206055,
      "logits/rejected": -0.07251911610364914,
      "logps/chosen": -2.0794589519500732,
      "logps/rejected": -2.473759889602661,
      "loss": 2.2463,
      "nll_loss": 2.196385383605957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20794589817523956,
      "rewards/margins": 0.039430104196071625,
      "rewards/rejected": -0.24737600982189178,
      "step": 2
    },
    {
      "epoch": 0.008284432171211598,
      "grad_norm": 0.6377604603767395,
      "learning_rate": 9.174311926605506e-08,
      "log_odds_chosen": 0.5233978033065796,
      "log_odds_ratio": -0.4696895480155945,
      "logits/chosen": -0.7436237931251526,
      "logits/rejected": -0.04083387181162834,
      "logps/chosen": -2.0129284858703613,
      "logps/rejected": -2.4808387756347656,
      "loss": 2.1678,
      "nll_loss": 2.120811700820923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20129284262657166,
      "rewards/margins": 0.04679100960493088,
      "rewards/rejected": -0.24808385968208313,
      "step": 3
    },
    {
      "epoch": 0.011045909561615464,
      "grad_norm": 0.5183601379394531,
      "learning_rate": 1.376146788990826e-07,
      "log_odds_chosen": 0.4738181531429291,
      "log_odds_ratio": -0.4919201731681824,
      "logits/chosen": -0.5707842111587524,
      "logits/rejected": -0.06899966299533844,
      "logps/chosen": -2.006629467010498,
      "logps/rejected": -2.429466724395752,
      "loss": 2.1541,
      "nll_loss": 2.104935646057129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20066295564174652,
      "rewards/margins": 0.04228372871875763,
      "rewards/rejected": -0.24294668436050415,
      "step": 4
    },
    {
      "epoch": 0.013807386952019331,
      "grad_norm": 0.5362528562545776,
      "learning_rate": 1.8348623853211012e-07,
      "log_odds_chosen": 0.47211968898773193,
      "log_odds_ratio": -0.49660325050354004,
      "logits/chosen": -0.5831207633018494,
      "logits/rejected": -0.13220791518688202,
      "logps/chosen": -1.9650582075119019,
      "logps/rejected": -2.3817272186279297,
      "loss": 2.1187,
      "nll_loss": 2.0690417289733887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1965058296918869,
      "rewards/margins": 0.04166688770055771,
      "rewards/rejected": -0.23817269504070282,
      "step": 5
    },
    {
      "epoch": 0.016568864342423197,
      "grad_norm": 0.5392005443572998,
      "learning_rate": 2.2935779816513764e-07,
      "log_odds_chosen": 0.3942825496196747,
      "log_odds_ratio": -0.518440306186676,
      "logits/chosen": -0.5348646640777588,
      "logits/rejected": -0.23154743015766144,
      "logps/chosen": -2.012246608734131,
      "logps/rejected": -2.361220359802246,
      "loss": 2.178,
      "nll_loss": 2.126155376434326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2012246698141098,
      "rewards/margins": 0.03489736467599869,
      "rewards/rejected": -0.2361220419406891,
      "step": 6
    },
    {
      "epoch": 0.019330341732827064,
      "grad_norm": 0.5474534630775452,
      "learning_rate": 2.752293577981652e-07,
      "log_odds_chosen": 0.2622841000556946,
      "log_odds_ratio": -0.576733410358429,
      "logits/chosen": -0.5426779985427856,
      "logits/rejected": -0.2341059297323227,
      "logps/chosen": -2.074805736541748,
      "logps/rejected": -2.310166120529175,
      "loss": 2.2374,
      "nll_loss": 2.179717779159546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2074805647134781,
      "rewards/margins": 0.023536043241620064,
      "rewards/rejected": -0.231016606092453,
      "step": 7
    },
    {
      "epoch": 0.022091819123230928,
      "grad_norm": 0.5940824747085571,
      "learning_rate": 3.211009174311927e-07,
      "log_odds_chosen": 0.41454389691352844,
      "log_odds_ratio": -0.5088675618171692,
      "logits/chosen": -0.45266416668891907,
      "logits/rejected": -0.03741999715566635,
      "logps/chosen": -2.081907033920288,
      "logps/rejected": -2.452606439590454,
      "loss": 2.2474,
      "nll_loss": 2.19650936126709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2081906944513321,
      "rewards/margins": 0.037069931626319885,
      "rewards/rejected": -0.24526062607765198,
      "step": 8
    },
    {
      "epoch": 0.024853296513634795,
      "grad_norm": 0.5987945795059204,
      "learning_rate": 3.6697247706422023e-07,
      "log_odds_chosen": 0.666429340839386,
      "log_odds_ratio": -0.42349112033843994,
      "logits/chosen": -0.7504861950874329,
      "logits/rejected": -0.16368302702903748,
      "logps/chosen": -1.9702867269515991,
      "logps/rejected": -2.566361427307129,
      "loss": 2.1456,
      "nll_loss": 2.10321307182312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19702866673469543,
      "rewards/margins": 0.05960746854543686,
      "rewards/rejected": -0.2566361427307129,
      "step": 9
    },
    {
      "epoch": 0.027614773904038662,
      "grad_norm": 0.7403894662857056,
      "learning_rate": 4.128440366972478e-07,
      "log_odds_chosen": 0.44253280758857727,
      "log_odds_ratio": -0.502151370048523,
      "logits/chosen": -0.7380926012992859,
      "logits/rejected": -0.23479697108268738,
      "logps/chosen": -2.0925681591033936,
      "logps/rejected": -2.4903366565704346,
      "loss": 2.2572,
      "nll_loss": 2.206953287124634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20925681293010712,
      "rewards/margins": 0.03977686166763306,
      "rewards/rejected": -0.24903367459774017,
      "step": 10
    },
    {
      "epoch": 0.030376251294442526,
      "grad_norm": 0.6613215208053589,
      "learning_rate": 4.587155963302753e-07,
      "log_odds_chosen": 0.5335031151771545,
      "log_odds_ratio": -0.4685918688774109,
      "logits/chosen": -0.6707795262336731,
      "logits/rejected": -0.13829460740089417,
      "logps/chosen": -2.0549967288970947,
      "logps/rejected": -2.5354185104370117,
      "loss": 2.2247,
      "nll_loss": 2.1778128147125244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20549967885017395,
      "rewards/margins": 0.04804220795631409,
      "rewards/rejected": -0.25354188680648804,
      "step": 11
    },
    {
      "epoch": 0.03313772868484639,
      "grad_norm": 0.5580596327781677,
      "learning_rate": 5.045871559633028e-07,
      "log_odds_chosen": 0.4023962616920471,
      "log_odds_ratio": -0.5220973491668701,
      "logits/chosen": -0.5904384851455688,
      "logits/rejected": 0.0007063774392008781,
      "logps/chosen": -1.958735704421997,
      "logps/rejected": -2.3157620429992676,
      "loss": 2.1455,
      "nll_loss": 2.09328293800354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19587357342243195,
      "rewards/margins": 0.03570263832807541,
      "rewards/rejected": -0.23157618939876556,
      "step": 12
    },
    {
      "epoch": 0.03589920607525026,
      "grad_norm": 0.4629175662994385,
      "learning_rate": 5.504587155963304e-07,
      "log_odds_chosen": 0.4922281503677368,
      "log_odds_ratio": -0.4793952405452728,
      "logits/chosen": -0.489984929561615,
      "logits/rejected": -0.08446945250034332,
      "logps/chosen": -1.8884625434875488,
      "logps/rejected": -2.318817377090454,
      "loss": 2.0661,
      "nll_loss": 2.0181963443756104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18884626030921936,
      "rewards/margins": 0.043035492300987244,
      "rewards/rejected": -0.2318817526102066,
      "step": 13
    },
    {
      "epoch": 0.03866068346565413,
      "grad_norm": 0.6970926523208618,
      "learning_rate": 5.963302752293579e-07,
      "log_odds_chosen": 0.46570533514022827,
      "log_odds_ratio": -0.49952778220176697,
      "logits/chosen": -0.6894131898880005,
      "logits/rejected": -0.010050175711512566,
      "logps/chosen": -2.0864779949188232,
      "logps/rejected": -2.503685235977173,
      "loss": 2.2329,
      "nll_loss": 2.182929277420044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20864778757095337,
      "rewards/margins": 0.041720740497112274,
      "rewards/rejected": -0.25036853551864624,
      "step": 14
    },
    {
      "epoch": 0.04142216085605799,
      "grad_norm": 0.7035249471664429,
      "learning_rate": 6.422018348623854e-07,
      "log_odds_chosen": 0.5717816948890686,
      "log_odds_ratio": -0.44831815361976624,
      "logits/chosen": -0.7732019424438477,
      "logits/rejected": 0.13750173151493073,
      "logps/chosen": -2.0055058002471924,
      "logps/rejected": -2.513883590698242,
      "loss": 2.1667,
      "nll_loss": 2.1218552589416504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20055057108402252,
      "rewards/margins": 0.05083777755498886,
      "rewards/rejected": -0.2513883411884308,
      "step": 15
    },
    {
      "epoch": 0.044183638246461855,
      "grad_norm": 0.489059180021286,
      "learning_rate": 6.880733944954129e-07,
      "log_odds_chosen": 0.45178472995758057,
      "log_odds_ratio": -0.501604437828064,
      "logits/chosen": -0.6046928763389587,
      "logits/rejected": 0.05104057490825653,
      "logps/chosen": -1.8943061828613281,
      "logps/rejected": -2.2864151000976562,
      "loss": 2.0731,
      "nll_loss": 2.022980213165283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18943063914775848,
      "rewards/margins": 0.03921087831258774,
      "rewards/rejected": -0.22864152491092682,
      "step": 16
    },
    {
      "epoch": 0.04694511563686572,
      "grad_norm": 0.5798972249031067,
      "learning_rate": 7.339449541284405e-07,
      "log_odds_chosen": 0.521186113357544,
      "log_odds_ratio": -0.4737667441368103,
      "logits/chosen": -0.5957604646682739,
      "logits/rejected": 0.05874314904212952,
      "logps/chosen": -1.9864083528518677,
      "logps/rejected": -2.4512085914611816,
      "loss": 2.1331,
      "nll_loss": 2.085712194442749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1986408233642578,
      "rewards/margins": 0.04648003727197647,
      "rewards/rejected": -0.24512089788913727,
      "step": 17
    },
    {
      "epoch": 0.04970659302726959,
      "grad_norm": 0.602294921875,
      "learning_rate": 7.79816513761468e-07,
      "log_odds_chosen": 0.6343832612037659,
      "log_odds_ratio": -0.42983177304267883,
      "logits/chosen": -0.6467751264572144,
      "logits/rejected": -0.0914345234632492,
      "logps/chosen": -2.020120143890381,
      "logps/rejected": -2.5913381576538086,
      "loss": 2.1654,
      "nll_loss": 2.1224629878997803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2020120471715927,
      "rewards/margins": 0.05712177976965904,
      "rewards/rejected": -0.25913381576538086,
      "step": 18
    },
    {
      "epoch": 0.05246807041767346,
      "grad_norm": 0.6167095303535461,
      "learning_rate": 8.256880733944956e-07,
      "log_odds_chosen": 0.23837895691394806,
      "log_odds_ratio": -0.5851647853851318,
      "logits/chosen": -0.6367707848548889,
      "logits/rejected": -0.06072646751999855,
      "logps/chosen": -2.055112838745117,
      "logps/rejected": -2.2656140327453613,
      "loss": 2.2217,
      "nll_loss": 2.163205146789551,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20551128685474396,
      "rewards/margins": 0.021050114184617996,
      "rewards/rejected": -0.22656141221523285,
      "step": 19
    },
    {
      "epoch": 0.055229547808077324,
      "grad_norm": 0.5270997285842896,
      "learning_rate": 8.71559633027523e-07,
      "log_odds_chosen": 0.4399060010910034,
      "log_odds_ratio": -0.5041620135307312,
      "logits/chosen": -0.5500829815864563,
      "logits/rejected": 0.014720816165208817,
      "logps/chosen": -1.9720858335494995,
      "logps/rejected": -2.3600573539733887,
      "loss": 2.1571,
      "nll_loss": 2.1066458225250244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19720861315727234,
      "rewards/margins": 0.038797151297330856,
      "rewards/rejected": -0.2360057532787323,
      "step": 20
    },
    {
      "epoch": 0.057991025198481184,
      "grad_norm": 0.639258086681366,
      "learning_rate": 9.174311926605506e-07,
      "log_odds_chosen": 0.35250768065452576,
      "log_odds_ratio": -0.5405789017677307,
      "logits/chosen": -0.45625555515289307,
      "logits/rejected": -0.03890611231327057,
      "logps/chosen": -2.1359739303588867,
      "logps/rejected": -2.454291820526123,
      "loss": 2.2807,
      "nll_loss": 2.2266433238983154,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21359741687774658,
      "rewards/margins": 0.0318317785859108,
      "rewards/rejected": -0.2454291731119156,
      "step": 21
    },
    {
      "epoch": 0.06075250258888505,
      "grad_norm": 0.5805932283401489,
      "learning_rate": 9.633027522935782e-07,
      "log_odds_chosen": 0.5011261105537415,
      "log_odds_ratio": -0.4887810945510864,
      "logits/chosen": -0.6073933839797974,
      "logits/rejected": -0.18206848204135895,
      "logps/chosen": -1.9807653427124023,
      "logps/rejected": -2.4255237579345703,
      "loss": 2.1351,
      "nll_loss": 2.0862059593200684,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1980765163898468,
      "rewards/margins": 0.044475845992565155,
      "rewards/rejected": -0.24255238473415375,
      "step": 22
    },
    {
      "epoch": 0.06351397997928893,
      "grad_norm": 0.5500138998031616,
      "learning_rate": 1.0091743119266057e-06,
      "log_odds_chosen": 0.3974594175815582,
      "log_odds_ratio": -0.5229506492614746,
      "logits/chosen": -0.5682663321495056,
      "logits/rejected": -0.3609941005706787,
      "logps/chosen": -1.963416576385498,
      "logps/rejected": -2.318310499191284,
      "loss": 2.1453,
      "nll_loss": 2.0929956436157227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19634166359901428,
      "rewards/margins": 0.03548937663435936,
      "rewards/rejected": -0.23183104395866394,
      "step": 23
    },
    {
      "epoch": 0.06627545736969279,
      "grad_norm": 0.5702619552612305,
      "learning_rate": 1.055045871559633e-06,
      "log_odds_chosen": 0.522091805934906,
      "log_odds_ratio": -0.4669547379016876,
      "logits/chosen": -0.5776705741882324,
      "logits/rejected": 0.03815801814198494,
      "logps/chosen": -1.9660505056381226,
      "logps/rejected": -2.429518461227417,
      "loss": 2.1353,
      "nll_loss": 2.0886194705963135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19660505652427673,
      "rewards/margins": 0.04634679853916168,
      "rewards/rejected": -0.24295185506343842,
      "step": 24
    },
    {
      "epoch": 0.06903693476009665,
      "grad_norm": 0.6908774375915527,
      "learning_rate": 1.1009174311926608e-06,
      "log_odds_chosen": 0.6555817723274231,
      "log_odds_ratio": -0.42638978362083435,
      "logits/chosen": -0.6369448304176331,
      "logits/rejected": -0.0680353045463562,
      "logps/chosen": -1.996623158454895,
      "logps/rejected": -2.585740327835083,
      "loss": 2.157,
      "nll_loss": 2.1143739223480225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19966234266757965,
      "rewards/margins": 0.058911725878715515,
      "rewards/rejected": -0.25857406854629517,
      "step": 25
    },
    {
      "epoch": 0.07179841215050052,
      "grad_norm": 0.6247107982635498,
      "learning_rate": 1.1467889908256882e-06,
      "log_odds_chosen": 0.37967437505722046,
      "log_odds_ratio": -0.528645396232605,
      "logits/chosen": -0.5880488157272339,
      "logits/rejected": -0.11361770331859589,
      "logps/chosen": -2.0194289684295654,
      "logps/rejected": -2.3577256202697754,
      "loss": 2.1956,
      "nll_loss": 2.142758369445801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20194292068481445,
      "rewards/margins": 0.03382965177297592,
      "rewards/rejected": -0.23577255010604858,
      "step": 26
    },
    {
      "epoch": 0.07455988954090438,
      "grad_norm": 0.44772788882255554,
      "learning_rate": 1.1926605504587159e-06,
      "log_odds_chosen": 0.4675738513469696,
      "log_odds_ratio": -0.4964170753955841,
      "logits/chosen": -0.434501588344574,
      "logits/rejected": -0.17547550797462463,
      "logps/chosen": -1.865759253501892,
      "logps/rejected": -2.272789239883423,
      "loss": 2.0462,
      "nll_loss": 1.996600866317749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18657593429088593,
      "rewards/margins": 0.04070296883583069,
      "rewards/rejected": -0.2272789031267166,
      "step": 27
    },
    {
      "epoch": 0.07732136693130826,
      "grad_norm": 0.5303645730018616,
      "learning_rate": 1.2385321100917433e-06,
      "log_odds_chosen": 0.5267918109893799,
      "log_odds_ratio": -0.46872708201408386,
      "logits/chosen": -0.5293477773666382,
      "logits/rejected": -0.20712637901306152,
      "logps/chosen": -1.9557033777236938,
      "logps/rejected": -2.422588586807251,
      "loss": 2.123,
      "nll_loss": 2.0761377811431885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19557033479213715,
      "rewards/margins": 0.04668852686882019,
      "rewards/rejected": -0.24225887656211853,
      "step": 28
    },
    {
      "epoch": 0.08008284432171212,
      "grad_norm": 0.558870792388916,
      "learning_rate": 1.2844036697247707e-06,
      "log_odds_chosen": 0.4761933982372284,
      "log_odds_ratio": -0.49172335863113403,
      "logits/chosen": -0.5614610314369202,
      "logits/rejected": -0.03958575427532196,
      "logps/chosen": -2.044667959213257,
      "logps/rejected": -2.4699392318725586,
      "loss": 2.2024,
      "nll_loss": 2.153231143951416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2044668048620224,
      "rewards/margins": 0.042527101933956146,
      "rewards/rejected": -0.24699391424655914,
      "step": 29
    },
    {
      "epoch": 0.08284432171211598,
      "grad_norm": 0.6042240262031555,
      "learning_rate": 1.3302752293577984e-06,
      "log_odds_chosen": 0.6456740498542786,
      "log_odds_ratio": -0.4289007782936096,
      "logits/chosen": -0.5478062629699707,
      "logits/rejected": -0.0002663079649209976,
      "logps/chosen": -2.02502703666687,
      "logps/rejected": -2.60587477684021,
      "loss": 2.1737,
      "nll_loss": 2.1308515071868896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20250271260738373,
      "rewards/margins": 0.058084748685359955,
      "rewards/rejected": -0.2605874538421631,
      "step": 30
    },
    {
      "epoch": 0.08560579910251985,
      "grad_norm": 0.6082971692085266,
      "learning_rate": 1.3761467889908258e-06,
      "log_odds_chosen": 0.5572243928909302,
      "log_odds_ratio": -0.45846420526504517,
      "logits/chosen": -0.5786283016204834,
      "logits/rejected": -0.06976839900016785,
      "logps/chosen": -2.0147552490234375,
      "logps/rejected": -2.5146052837371826,
      "loss": 2.1834,
      "nll_loss": 2.1375184059143066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20147554576396942,
      "rewards/margins": 0.049984999001026154,
      "rewards/rejected": -0.2514605224132538,
      "step": 31
    },
    {
      "epoch": 0.08836727649292371,
      "grad_norm": 0.4792933762073517,
      "learning_rate": 1.4220183486238535e-06,
      "log_odds_chosen": 0.43500036001205444,
      "log_odds_ratio": -0.5046517252922058,
      "logits/chosen": -0.4258999824523926,
      "logits/rejected": 0.003258749842643738,
      "logps/chosen": -2.011561870574951,
      "logps/rejected": -2.398580312728882,
      "loss": 2.1831,
      "nll_loss": 2.132627010345459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20115619897842407,
      "rewards/margins": 0.03870181739330292,
      "rewards/rejected": -0.239858016371727,
      "step": 32
    },
    {
      "epoch": 0.09112875388332758,
      "grad_norm": 0.5385621786117554,
      "learning_rate": 1.467889908256881e-06,
      "log_odds_chosen": 0.573083758354187,
      "log_odds_ratio": -0.4524250328540802,
      "logits/chosen": -0.4552776515483856,
      "logits/rejected": -0.06403280049562454,
      "logps/chosen": -1.9241302013397217,
      "logps/rejected": -2.4318630695343018,
      "loss": 2.0919,
      "nll_loss": 2.046658754348755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19241301715373993,
      "rewards/margins": 0.05077328532934189,
      "rewards/rejected": -0.24318630993366241,
      "step": 33
    },
    {
      "epoch": 0.09389023127373144,
      "grad_norm": 0.4870143234729767,
      "learning_rate": 1.5137614678899084e-06,
      "log_odds_chosen": 0.6046110391616821,
      "log_odds_ratio": -0.4510130286216736,
      "logits/chosen": -0.40763065218925476,
      "logits/rejected": -0.08770006895065308,
      "logps/chosen": -1.922775387763977,
      "logps/rejected": -2.462465524673462,
      "loss": 2.0741,
      "nll_loss": 2.028975009918213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19227753579616547,
      "rewards/margins": 0.053969040513038635,
      "rewards/rejected": -0.2462465912103653,
      "step": 34
    },
    {
      "epoch": 0.09665170866413532,
      "grad_norm": 0.6609561443328857,
      "learning_rate": 1.559633027522936e-06,
      "log_odds_chosen": 0.40986764430999756,
      "log_odds_ratio": -0.5126968026161194,
      "logits/chosen": -0.8081121444702148,
      "logits/rejected": -0.20409950613975525,
      "logps/chosen": -1.9795390367507935,
      "logps/rejected": -2.3391332626342773,
      "loss": 2.152,
      "nll_loss": 2.1007797718048096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19795390963554382,
      "rewards/margins": 0.03595941513776779,
      "rewards/rejected": -0.23391331732273102,
      "step": 35
    },
    {
      "epoch": 0.09941318605453918,
      "grad_norm": 0.6118403673171997,
      "learning_rate": 1.6055045871559635e-06,
      "log_odds_chosen": 0.603810727596283,
      "log_odds_ratio": -0.4452260434627533,
      "logits/chosen": -0.6869708299636841,
      "logits/rejected": 0.01393081247806549,
      "logps/chosen": -1.9751290082931519,
      "logps/rejected": -2.515267848968506,
      "loss": 2.1348,
      "nll_loss": 2.0902533531188965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1975128948688507,
      "rewards/margins": 0.05401391163468361,
      "rewards/rejected": -0.251526802778244,
      "step": 36
    },
    {
      "epoch": 0.10217466344494304,
      "grad_norm": 0.6636885404586792,
      "learning_rate": 1.6513761467889911e-06,
      "log_odds_chosen": 0.5960386991500854,
      "log_odds_ratio": -0.44556429982185364,
      "logits/chosen": -0.5324239134788513,
      "logits/rejected": -0.1962709128856659,
      "logps/chosen": -2.1164088249206543,
      "logps/rejected": -2.6588618755340576,
      "loss": 2.2904,
      "nll_loss": 2.2458741664886475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2116408795118332,
      "rewards/margins": 0.054245319217443466,
      "rewards/rejected": -0.26588618755340576,
      "step": 37
    },
    {
      "epoch": 0.10493614083534691,
      "grad_norm": 0.5414693355560303,
      "learning_rate": 1.6972477064220186e-06,
      "log_odds_chosen": 0.4305243492126465,
      "log_odds_ratio": -0.5040389895439148,
      "logits/chosen": -0.5475431680679321,
      "logits/rejected": -0.15733303129673004,
      "logps/chosen": -1.9466073513031006,
      "logps/rejected": -2.3275275230407715,
      "loss": 2.1303,
      "nll_loss": 2.079880475997925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1946607381105423,
      "rewards/margins": 0.03809202462434769,
      "rewards/rejected": -0.2327527403831482,
      "step": 38
    },
    {
      "epoch": 0.10769761822575077,
      "grad_norm": 0.7092136740684509,
      "learning_rate": 1.743119266055046e-06,
      "log_odds_chosen": 0.49458950757980347,
      "log_odds_ratio": -0.499477356672287,
      "logits/chosen": -0.47480347752571106,
      "logits/rejected": 0.009076721966266632,
      "logps/chosen": -2.077540159225464,
      "logps/rejected": -2.527543067932129,
      "loss": 2.2296,
      "nll_loss": 2.1796536445617676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20775403082370758,
      "rewards/margins": 0.04500027373433113,
      "rewards/rejected": -0.2527543008327484,
      "step": 39
    },
    {
      "epoch": 0.11045909561615465,
      "grad_norm": 0.7840990424156189,
      "learning_rate": 1.7889908256880737e-06,
      "log_odds_chosen": 0.27720558643341064,
      "log_odds_ratio": -0.5738192796707153,
      "logits/chosen": -0.6754797101020813,
      "logits/rejected": -0.1666824221611023,
      "logps/chosen": -2.097449779510498,
      "logps/rejected": -2.346257448196411,
      "loss": 2.2706,
      "nll_loss": 2.2131741046905518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20974498987197876,
      "rewards/margins": 0.02488076500594616,
      "rewards/rejected": -0.23462577164173126,
      "step": 40
    },
    {
      "epoch": 0.11322057300655851,
      "grad_norm": 0.49594300985336304,
      "learning_rate": 1.8348623853211011e-06,
      "log_odds_chosen": 0.4892931878566742,
      "log_odds_ratio": -0.4875307083129883,
      "logits/chosen": -0.5313754081726074,
      "logits/rejected": -0.11813461780548096,
      "logps/chosen": -1.9190640449523926,
      "logps/rejected": -2.3516781330108643,
      "loss": 2.0847,
      "nll_loss": 2.035940647125244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1919064074754715,
      "rewards/margins": 0.043261416256427765,
      "rewards/rejected": -0.23516784608364105,
      "step": 41
    },
    {
      "epoch": 0.11598205039696237,
      "grad_norm": 0.6706519722938538,
      "learning_rate": 1.8807339449541288e-06,
      "log_odds_chosen": 0.3553638756275177,
      "log_odds_ratio": -0.5406981706619263,
      "logits/chosen": -0.6044270992279053,
      "logits/rejected": -0.1877760887145996,
      "logps/chosen": -2.110844850540161,
      "logps/rejected": -2.4292314052581787,
      "loss": 2.2811,
      "nll_loss": 2.2269935607910156,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2110844999551773,
      "rewards/margins": 0.031838610768318176,
      "rewards/rejected": -0.24292311072349548,
      "step": 42
    },
    {
      "epoch": 0.11874352778736624,
      "grad_norm": 0.569320023059845,
      "learning_rate": 1.9266055045871564e-06,
      "log_odds_chosen": 0.4219287633895874,
      "log_odds_ratio": -0.5136047005653381,
      "logits/chosen": -0.564371645450592,
      "logits/rejected": -0.1906774342060089,
      "logps/chosen": -1.9711859226226807,
      "logps/rejected": -2.341031312942505,
      "loss": 2.1533,
      "nll_loss": 2.101966381072998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1971185803413391,
      "rewards/margins": 0.036984533071517944,
      "rewards/rejected": -0.23410312831401825,
      "step": 43
    },
    {
      "epoch": 0.1215050051777701,
      "grad_norm": 0.6019275188446045,
      "learning_rate": 1.9724770642201837e-06,
      "log_odds_chosen": 0.3504191040992737,
      "log_odds_ratio": -0.5448323488235474,
      "logits/chosen": -0.5221514105796814,
      "logits/rejected": -0.2111213058233261,
      "logps/chosen": -2.0630786418914795,
      "logps/rejected": -2.3758115768432617,
      "loss": 2.226,
      "nll_loss": 2.1714861392974854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20630787312984467,
      "rewards/margins": 0.03127329424023628,
      "rewards/rejected": -0.23758116364479065,
      "step": 44
    },
    {
      "epoch": 0.12426648256817398,
      "grad_norm": 0.5997087955474854,
      "learning_rate": 2.0183486238532113e-06,
      "log_odds_chosen": 0.6440756916999817,
      "log_odds_ratio": -0.42918679118156433,
      "logits/chosen": -0.6537622213363647,
      "logits/rejected": -0.3867274224758148,
      "logps/chosen": -1.964981198310852,
      "logps/rejected": -2.541144371032715,
      "loss": 2.1164,
      "nll_loss": 2.073434352874756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1964981108903885,
      "rewards/margins": 0.057616353034973145,
      "rewards/rejected": -0.25411444902420044,
      "step": 45
    },
    {
      "epoch": 0.12702795995857785,
      "grad_norm": 0.5181168913841248,
      "learning_rate": 2.064220183486239e-06,
      "log_odds_chosen": 0.564142107963562,
      "log_odds_ratio": -0.4582359790802002,
      "logits/chosen": -0.5351958870887756,
      "logits/rejected": -0.1501241773366928,
      "logps/chosen": -1.84321928024292,
      "logps/rejected": -2.331892967224121,
      "loss": 2.0016,
      "nll_loss": 1.9557558298110962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18432192504405975,
      "rewards/margins": 0.04886738583445549,
      "rewards/rejected": -0.23318932950496674,
      "step": 46
    },
    {
      "epoch": 0.1297894373489817,
      "grad_norm": 0.5089595913887024,
      "learning_rate": 2.110091743119266e-06,
      "log_odds_chosen": 0.4004448652267456,
      "log_odds_ratio": -0.5262866616249084,
      "logits/chosen": -0.4425760507583618,
      "logits/rejected": -0.17353124916553497,
      "logps/chosen": -2.045093059539795,
      "logps/rejected": -2.4056942462921143,
      "loss": 2.2153,
      "nll_loss": 2.162646770477295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20450931787490845,
      "rewards/margins": 0.03606009483337402,
      "rewards/rejected": -0.24056941270828247,
      "step": 47
    },
    {
      "epoch": 0.13255091473938557,
      "grad_norm": 0.6208946108818054,
      "learning_rate": 2.155963302752294e-06,
      "log_odds_chosen": 0.18348652124404907,
      "log_odds_ratio": -0.6307563781738281,
      "logits/chosen": -0.45777827501296997,
      "logits/rejected": -0.11396709084510803,
      "logps/chosen": -2.1183199882507324,
      "logps/rejected": -2.2930657863616943,
      "loss": 2.2819,
      "nll_loss": 2.21886944770813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21183200180530548,
      "rewards/margins": 0.01747458055615425,
      "rewards/rejected": -0.22930656373500824,
      "step": 48
    },
    {
      "epoch": 0.13531239212978943,
      "grad_norm": 0.6294072866439819,
      "learning_rate": 2.2018348623853215e-06,
      "log_odds_chosen": 0.6346040964126587,
      "log_odds_ratio": -0.43739554286003113,
      "logits/chosen": -0.542172908782959,
      "logits/rejected": -0.039900042116642,
      "logps/chosen": -2.052166223526001,
      "logps/rejected": -2.628195285797119,
      "loss": 2.2026,
      "nll_loss": 2.1588120460510254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.205216646194458,
      "rewards/margins": 0.05760287865996361,
      "rewards/rejected": -0.2628195285797119,
      "step": 49
    },
    {
      "epoch": 0.1380738695201933,
      "grad_norm": 0.6679416298866272,
      "learning_rate": 2.2477064220183487e-06,
      "log_odds_chosen": 0.5639493465423584,
      "log_odds_ratio": -0.47183454036712646,
      "logits/chosen": -0.6221768856048584,
      "logits/rejected": -0.0222525242716074,
      "logps/chosen": -2.010540008544922,
      "logps/rejected": -2.51875901222229,
      "loss": 2.1569,
      "nll_loss": 2.1097309589385986,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20105400681495667,
      "rewards/margins": 0.05082192271947861,
      "rewards/rejected": -0.25187593698501587,
      "step": 50
    },
    {
      "epoch": 0.14083534691059718,
      "grad_norm": 0.6739494800567627,
      "learning_rate": 2.2935779816513764e-06,
      "log_odds_chosen": 0.4806385040283203,
      "log_odds_ratio": -0.486289381980896,
      "logits/chosen": -0.5340446829795837,
      "logits/rejected": -0.3175644278526306,
      "logps/chosen": -2.032139301300049,
      "logps/rejected": -2.4611167907714844,
      "loss": 2.1976,
      "nll_loss": 2.1489548683166504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2032139152288437,
      "rewards/margins": 0.04289776086807251,
      "rewards/rejected": -0.2461116760969162,
      "step": 51
    },
    {
      "epoch": 0.14359682430100104,
      "grad_norm": 0.5641347765922546,
      "learning_rate": 2.339449541284404e-06,
      "log_odds_chosen": 0.4302893579006195,
      "log_odds_ratio": -0.5059407353401184,
      "logits/chosen": -0.4652354121208191,
      "logits/rejected": -0.14794279634952545,
      "logps/chosen": -1.98146390914917,
      "logps/rejected": -2.362124443054199,
      "loss": 2.1395,
      "nll_loss": 2.0888633728027344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19814638793468475,
      "rewards/margins": 0.0380660817027092,
      "rewards/rejected": -0.23621246218681335,
      "step": 52
    },
    {
      "epoch": 0.1463583016914049,
      "grad_norm": 0.6226781606674194,
      "learning_rate": 2.3853211009174317e-06,
      "log_odds_chosen": 0.38262999057769775,
      "log_odds_ratio": -0.5264566540718079,
      "logits/chosen": -0.601564884185791,
      "logits/rejected": -0.23214676976203918,
      "logps/chosen": -1.9446452856063843,
      "logps/rejected": -2.2811543941497803,
      "loss": 2.1236,
      "nll_loss": 2.070988178253174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1944645196199417,
      "rewards/margins": 0.03365091234445572,
      "rewards/rejected": -0.22811545431613922,
      "step": 53
    },
    {
      "epoch": 0.14911977908180876,
      "grad_norm": 0.576812207698822,
      "learning_rate": 2.431192660550459e-06,
      "log_odds_chosen": 0.3935718238353729,
      "log_odds_ratio": -0.5239484310150146,
      "logits/chosen": -0.5574474930763245,
      "logits/rejected": -0.1828337013721466,
      "logps/chosen": -1.9322319030761719,
      "logps/rejected": -2.280305862426758,
      "loss": 2.1135,
      "nll_loss": 2.061114549636841,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19322317838668823,
      "rewards/margins": 0.03480737283825874,
      "rewards/rejected": -0.22803056240081787,
      "step": 54
    },
    {
      "epoch": 0.15188125647221262,
      "grad_norm": 0.5661758184432983,
      "learning_rate": 2.4770642201834866e-06,
      "log_odds_chosen": 0.39002981781959534,
      "log_odds_ratio": -0.5207180380821228,
      "logits/chosen": -0.6281000971794128,
      "logits/rejected": -0.12741385400295258,
      "logps/chosen": -1.9887436628341675,
      "logps/rejected": -2.3331456184387207,
      "loss": 2.1681,
      "nll_loss": 2.115997314453125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.198874369263649,
      "rewards/margins": 0.03444019705057144,
      "rewards/rejected": -0.23331457376480103,
      "step": 55
    },
    {
      "epoch": 0.1546427338626165,
      "grad_norm": 0.6655378937721252,
      "learning_rate": 2.522935779816514e-06,
      "log_odds_chosen": 0.6584038138389587,
      "log_odds_ratio": -0.427642822265625,
      "logits/chosen": -0.6279682517051697,
      "logits/rejected": -0.12986338138580322,
      "logps/chosen": -1.9852056503295898,
      "logps/rejected": -2.5761494636535645,
      "loss": 2.1584,
      "nll_loss": 2.115626335144043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19852055609226227,
      "rewards/margins": 0.059094399213790894,
      "rewards/rejected": -0.25761497020721436,
      "step": 56
    },
    {
      "epoch": 0.15740421125302037,
      "grad_norm": 0.48872971534729004,
      "learning_rate": 2.5688073394495415e-06,
      "log_odds_chosen": 0.42161181569099426,
      "log_odds_ratio": -0.514995276927948,
      "logits/chosen": -0.49338769912719727,
      "logits/rejected": -0.07346588373184204,
      "logps/chosen": -1.8429884910583496,
      "logps/rejected": -2.2116591930389404,
      "loss": 2.014,
      "nll_loss": 1.9625122547149658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18429884314537048,
      "rewards/margins": 0.036867089569568634,
      "rewards/rejected": -0.22116592526435852,
      "step": 57
    },
    {
      "epoch": 0.16016568864342423,
      "grad_norm": 0.6582737565040588,
      "learning_rate": 2.6146788990825687e-06,
      "log_odds_chosen": 0.5138990879058838,
      "log_odds_ratio": -0.4847075939178467,
      "logits/chosen": -0.5234618186950684,
      "logits/rejected": -0.12736433744430542,
      "logps/chosen": -1.9835774898529053,
      "logps/rejected": -2.4381330013275146,
      "loss": 2.1515,
      "nll_loss": 2.1030287742614746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19835779070854187,
      "rewards/margins": 0.045455530285835266,
      "rewards/rejected": -0.24381330609321594,
      "step": 58
    },
    {
      "epoch": 0.1629271660338281,
      "grad_norm": 0.5795738697052002,
      "learning_rate": 2.6605504587155968e-06,
      "log_odds_chosen": 0.2905232906341553,
      "log_odds_ratio": -0.5645147562026978,
      "logits/chosen": -0.47340482473373413,
      "logits/rejected": -0.21826684474945068,
      "logps/chosen": -1.9716663360595703,
      "logps/rejected": -2.227067470550537,
      "loss": 2.1514,
      "nll_loss": 2.0949792861938477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19716663658618927,
      "rewards/margins": 0.025540094822645187,
      "rewards/rejected": -0.22270673513412476,
      "step": 59
    },
    {
      "epoch": 0.16568864342423195,
      "grad_norm": 0.5678736567497253,
      "learning_rate": 2.706422018348624e-06,
      "log_odds_chosen": 0.3714035749435425,
      "log_odds_ratio": -0.5299574136734009,
      "logits/chosen": -0.45862269401550293,
      "logits/rejected": 0.08986371755599976,
      "logps/chosen": -2.004216432571411,
      "logps/rejected": -2.333292245864868,
      "loss": 2.1781,
      "nll_loss": 2.125133514404297,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20042164623737335,
      "rewards/margins": 0.032907579094171524,
      "rewards/rejected": -0.23332923650741577,
      "step": 60
    },
    {
      "epoch": 0.16845012081463584,
      "grad_norm": 0.5007438063621521,
      "learning_rate": 2.7522935779816517e-06,
      "log_odds_chosen": 0.667113721370697,
      "log_odds_ratio": -0.4193596839904785,
      "logits/chosen": -0.5191382765769958,
      "logits/rejected": -0.19058826565742493,
      "logps/chosen": -1.917590856552124,
      "logps/rejected": -2.5098061561584473,
      "loss": 2.0632,
      "nll_loss": 2.021275281906128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1917591094970703,
      "rewards/margins": 0.0592215433716774,
      "rewards/rejected": -0.2509806454181671,
      "step": 61
    },
    {
      "epoch": 0.1712115982050397,
      "grad_norm": 0.6429389119148254,
      "learning_rate": 2.798165137614679e-06,
      "log_odds_chosen": 0.41257914900779724,
      "log_odds_ratio": -0.5154306888580322,
      "logits/chosen": -0.45402857661247253,
      "logits/rejected": 0.11148527264595032,
      "logps/chosen": -2.081867218017578,
      "logps/rejected": -2.45068359375,
      "loss": 2.2481,
      "nll_loss": 2.1965107917785645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20818671584129333,
      "rewards/margins": 0.03688164800405502,
      "rewards/rejected": -0.24506837129592896,
      "step": 62
    },
    {
      "epoch": 0.17397307559544356,
      "grad_norm": 0.5388195514678955,
      "learning_rate": 2.844036697247707e-06,
      "log_odds_chosen": 0.5105969905853271,
      "log_odds_ratio": -0.4755265414714813,
      "logits/chosen": -0.46547558903694153,
      "logits/rejected": -0.1431473195552826,
      "logps/chosen": -1.9948811531066895,
      "logps/rejected": -2.448573589324951,
      "loss": 2.1454,
      "nll_loss": 2.0978972911834717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19948811829090118,
      "rewards/margins": 0.0453692302107811,
      "rewards/rejected": -0.24485734105110168,
      "step": 63
    },
    {
      "epoch": 0.17673455298584742,
      "grad_norm": 0.4911787211894989,
      "learning_rate": 2.8899082568807342e-06,
      "log_odds_chosen": 0.5747222304344177,
      "log_odds_ratio": -0.4552631676197052,
      "logits/chosen": -0.5521761178970337,
      "logits/rejected": -0.294252872467041,
      "logps/chosen": -1.9650574922561646,
      "logps/rejected": -2.4793307781219482,
      "loss": 2.1135,
      "nll_loss": 2.067988634109497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19650575518608093,
      "rewards/margins": 0.051427341997623444,
      "rewards/rejected": -0.24793308973312378,
      "step": 64
    },
    {
      "epoch": 0.17949603037625128,
      "grad_norm": 0.533399760723114,
      "learning_rate": 2.935779816513762e-06,
      "log_odds_chosen": 0.3648831844329834,
      "log_odds_ratio": -0.5328130722045898,
      "logits/chosen": -0.4136759638786316,
      "logits/rejected": -0.046070147305727005,
      "logps/chosen": -1.8867706060409546,
      "logps/rejected": -2.202749729156494,
      "loss": 2.0708,
      "nll_loss": 2.0175259113311768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18867707252502441,
      "rewards/margins": 0.031597916036844254,
      "rewards/rejected": -0.22027498483657837,
      "step": 65
    },
    {
      "epoch": 0.18225750776665517,
      "grad_norm": 0.5182326436042786,
      "learning_rate": 2.981651376146789e-06,
      "log_odds_chosen": 0.5400457382202148,
      "log_odds_ratio": -0.48501351475715637,
      "logits/chosen": -0.6035399436950684,
      "logits/rejected": -0.1548141986131668,
      "logps/chosen": -1.8529176712036133,
      "logps/rejected": -2.325399398803711,
      "loss": 2.022,
      "nll_loss": 1.973459243774414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18529179692268372,
      "rewards/margins": 0.04724816977977753,
      "rewards/rejected": -0.23253995180130005,
      "step": 66
    },
    {
      "epoch": 0.18501898515705903,
      "grad_norm": 0.5337976813316345,
      "learning_rate": 3.0275229357798168e-06,
      "log_odds_chosen": 0.33741873502731323,
      "log_odds_ratio": -0.5426305532455444,
      "logits/chosen": -0.39523714780807495,
      "logits/rejected": 0.0008284337818622589,
      "logps/chosen": -1.997262716293335,
      "logps/rejected": -2.2943520545959473,
      "loss": 2.1433,
      "nll_loss": 2.0890326499938965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19972628355026245,
      "rewards/margins": 0.029708925634622574,
      "rewards/rejected": -0.22943520545959473,
      "step": 67
    },
    {
      "epoch": 0.1877804625474629,
      "grad_norm": 0.5520625114440918,
      "learning_rate": 3.073394495412844e-06,
      "log_odds_chosen": 0.4403454065322876,
      "log_odds_ratio": -0.5091642141342163,
      "logits/chosen": -0.24377387762069702,
      "logits/rejected": 0.08138424903154373,
      "logps/chosen": -2.0106263160705566,
      "logps/rejected": -2.402209758758545,
      "loss": 2.2053,
      "nll_loss": 2.1543631553649902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2010626494884491,
      "rewards/margins": 0.039158351719379425,
      "rewards/rejected": -0.24022099375724792,
      "step": 68
    },
    {
      "epoch": 0.19054193993786675,
      "grad_norm": 0.650439441204071,
      "learning_rate": 3.119266055045872e-06,
      "log_odds_chosen": 0.5253904461860657,
      "log_odds_ratio": -0.4741411507129669,
      "logits/chosen": -0.4677438735961914,
      "logits/rejected": -0.013416798785328865,
      "logps/chosen": -2.077601909637451,
      "logps/rejected": -2.5524415969848633,
      "loss": 2.2423,
      "nll_loss": 2.194887161254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20776019990444183,
      "rewards/margins": 0.04748394340276718,
      "rewards/rejected": -0.2552441358566284,
      "step": 69
    },
    {
      "epoch": 0.19330341732827064,
      "grad_norm": 0.48958319425582886,
      "learning_rate": 3.1651376146788993e-06,
      "log_odds_chosen": 0.5917057991027832,
      "log_odds_ratio": -0.45470088720321655,
      "logits/chosen": -0.36797553300857544,
      "logits/rejected": -0.3181126117706299,
      "logps/chosen": -2.022463321685791,
      "logps/rejected": -2.5544497966766357,
      "loss": 2.1697,
      "nll_loss": 2.124249219894409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20224635303020477,
      "rewards/margins": 0.05319864675402641,
      "rewards/rejected": -0.2554450035095215,
      "step": 70
    },
    {
      "epoch": 0.1960648947186745,
      "grad_norm": 0.6293954253196716,
      "learning_rate": 3.211009174311927e-06,
      "log_odds_chosen": 0.44158488512039185,
      "log_odds_ratio": -0.5029429793357849,
      "logits/chosen": -0.4253910481929779,
      "logits/rejected": -0.14389574527740479,
      "logps/chosen": -1.9857234954833984,
      "logps/rejected": -2.379268169403076,
      "loss": 2.17,
      "nll_loss": 2.119678497314453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19857235252857208,
      "rewards/margins": 0.03935447335243225,
      "rewards/rejected": -0.23792681097984314,
      "step": 71
    },
    {
      "epoch": 0.19882637210907836,
      "grad_norm": 0.6024268269538879,
      "learning_rate": 3.256880733944954e-06,
      "log_odds_chosen": 0.5992636680603027,
      "log_odds_ratio": -0.4583205580711365,
      "logits/chosen": -0.3624473810195923,
      "logits/rejected": -0.16807770729064941,
      "logps/chosen": -1.8936195373535156,
      "logps/rejected": -2.403137683868408,
      "loss": 2.0566,
      "nll_loss": 2.0107438564300537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18936194479465485,
      "rewards/margins": 0.05095181241631508,
      "rewards/rejected": -0.24031376838684082,
      "step": 72
    },
    {
      "epoch": 0.20158784949948222,
      "grad_norm": 0.537467360496521,
      "learning_rate": 3.3027522935779823e-06,
      "log_odds_chosen": 0.38986673951148987,
      "log_odds_ratio": -0.518619179725647,
      "logits/chosen": -0.28752538561820984,
      "logits/rejected": -0.08893117308616638,
      "logps/chosen": -2.056885242462158,
      "logps/rejected": -2.4047675132751465,
      "loss": 2.2167,
      "nll_loss": 2.1648688316345215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2056885063648224,
      "rewards/margins": 0.03478822112083435,
      "rewards/rejected": -0.24047674238681793,
      "step": 73
    },
    {
      "epoch": 0.20434932688988608,
      "grad_norm": 0.48359447717666626,
      "learning_rate": 3.3486238532110095e-06,
      "log_odds_chosen": 0.39413708448410034,
      "log_odds_ratio": -0.5191097259521484,
      "logits/chosen": -0.28818562626838684,
      "logits/rejected": -0.17117023468017578,
      "logps/chosen": -1.9585490226745605,
      "logps/rejected": -2.3046271800994873,
      "loss": 2.1453,
      "nll_loss": 2.093356132507324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19585487246513367,
      "rewards/margins": 0.03460782766342163,
      "rewards/rejected": -0.2304627150297165,
      "step": 74
    },
    {
      "epoch": 0.20711080428028997,
      "grad_norm": 0.6197869777679443,
      "learning_rate": 3.394495412844037e-06,
      "log_odds_chosen": 0.4853059649467468,
      "log_odds_ratio": -0.487282931804657,
      "logits/chosen": -0.4553055763244629,
      "logits/rejected": -0.07368504256010056,
      "logps/chosen": -1.9564940929412842,
      "logps/rejected": -2.38771653175354,
      "loss": 2.1426,
      "nll_loss": 2.093902826309204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1956494152545929,
      "rewards/margins": 0.04312223941087723,
      "rewards/rejected": -0.23877164721488953,
      "step": 75
    },
    {
      "epoch": 0.20987228167069383,
      "grad_norm": 0.5478652119636536,
      "learning_rate": 3.4403669724770644e-06,
      "log_odds_chosen": 0.4639507532119751,
      "log_odds_ratio": -0.4932023882865906,
      "logits/chosen": -0.30065810680389404,
      "logits/rejected": -0.25181248784065247,
      "logps/chosen": -2.044691562652588,
      "logps/rejected": -2.460970401763916,
      "loss": 2.1933,
      "nll_loss": 2.1440277099609375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20446917414665222,
      "rewards/margins": 0.04162788391113281,
      "rewards/rejected": -0.24609704315662384,
      "step": 76
    },
    {
      "epoch": 0.2126337590610977,
      "grad_norm": 0.5865225195884705,
      "learning_rate": 3.486238532110092e-06,
      "log_odds_chosen": 0.40864938497543335,
      "log_odds_ratio": -0.5256964564323425,
      "logits/chosen": -0.414636492729187,
      "logits/rejected": -0.1841173619031906,
      "logps/chosen": -2.0234270095825195,
      "logps/rejected": -2.3900880813598633,
      "loss": 2.2079,
      "nll_loss": 2.155355215072632,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2023427039384842,
      "rewards/margins": 0.03666612505912781,
      "rewards/rejected": -0.239008828997612,
      "step": 77
    },
    {
      "epoch": 0.21539523645150155,
      "grad_norm": 0.5245475769042969,
      "learning_rate": 3.5321100917431193e-06,
      "log_odds_chosen": 0.3867124617099762,
      "log_odds_ratio": -0.5356163382530212,
      "logits/chosen": -0.22849449515342712,
      "logits/rejected": 0.04786547273397446,
      "logps/chosen": -2.047503709793091,
      "logps/rejected": -2.391859531402588,
      "loss": 2.2124,
      "nll_loss": 2.1587889194488525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20475035905838013,
      "rewards/margins": 0.03443560749292374,
      "rewards/rejected": -0.23918597400188446,
      "step": 78
    },
    {
      "epoch": 0.2181567138419054,
      "grad_norm": 0.4416508972644806,
      "learning_rate": 3.5779816513761473e-06,
      "log_odds_chosen": 0.457964152097702,
      "log_odds_ratio": -0.49229007959365845,
      "logits/chosen": -0.24249830842018127,
      "logits/rejected": -0.1156221255660057,
      "logps/chosen": -1.9371097087860107,
      "logps/rejected": -2.3405327796936035,
      "loss": 2.1264,
      "nll_loss": 2.077136516571045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19371098279953003,
      "rewards/margins": 0.0403422936797142,
      "rewards/rejected": -0.23405326902866364,
      "step": 79
    },
    {
      "epoch": 0.2209181912323093,
      "grad_norm": 0.511163055896759,
      "learning_rate": 3.6238532110091746e-06,
      "log_odds_chosen": 0.3918084502220154,
      "log_odds_ratio": -0.527028501033783,
      "logits/chosen": -0.3217644989490509,
      "logits/rejected": -0.12379680573940277,
      "logps/chosen": -1.9086521863937378,
      "logps/rejected": -2.250434160232544,
      "loss": 2.0875,
      "nll_loss": 2.034804105758667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19086521863937378,
      "rewards/margins": 0.03417817875742912,
      "rewards/rejected": -0.2250434011220932,
      "step": 80
    },
    {
      "epoch": 0.22367966862271316,
      "grad_norm": 0.48532167077064514,
      "learning_rate": 3.6697247706422022e-06,
      "log_odds_chosen": 0.38070449233055115,
      "log_odds_ratio": -0.5242205858230591,
      "logits/chosen": -0.24639271199703217,
      "logits/rejected": -0.02295789122581482,
      "logps/chosen": -1.9235130548477173,
      "logps/rejected": -2.256152868270874,
      "loss": 2.0966,
      "nll_loss": 2.044215440750122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.192351296544075,
      "rewards/margins": 0.03326397389173508,
      "rewards/rejected": -0.22561527788639069,
      "step": 81
    },
    {
      "epoch": 0.22644114601311702,
      "grad_norm": 0.4871878921985626,
      "learning_rate": 3.7155963302752295e-06,
      "log_odds_chosen": 0.3706399202346802,
      "log_odds_ratio": -0.5302938222885132,
      "logits/chosen": -0.32461822032928467,
      "logits/rejected": -0.07178197801113129,
      "logps/chosen": -2.002980947494507,
      "logps/rejected": -2.3308701515197754,
      "loss": 2.1902,
      "nll_loss": 2.1371781826019287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20029808580875397,
      "rewards/margins": 0.03278890624642372,
      "rewards/rejected": -0.2330869883298874,
      "step": 82
    },
    {
      "epoch": 0.22920262340352088,
      "grad_norm": 0.5049198865890503,
      "learning_rate": 3.7614678899082575e-06,
      "log_odds_chosen": 0.6436396241188049,
      "log_odds_ratio": -0.4284180700778961,
      "logits/chosen": -0.3081129789352417,
      "logits/rejected": -0.1997162252664566,
      "logps/chosen": -1.9448068141937256,
      "logps/rejected": -2.5192015171051025,
      "loss": 2.1169,
      "nll_loss": 2.0740115642547607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19448068737983704,
      "rewards/margins": 0.057439468801021576,
      "rewards/rejected": -0.2519201636314392,
      "step": 83
    },
    {
      "epoch": 0.23196410079392474,
      "grad_norm": 0.5246726870536804,
      "learning_rate": 3.8073394495412848e-06,
      "log_odds_chosen": 0.43793147802352905,
      "log_odds_ratio": -0.5015705823898315,
      "logits/chosen": -0.29296931624412537,
      "logits/rejected": -0.13174469769001007,
      "logps/chosen": -1.941349744796753,
      "logps/rejected": -2.3253884315490723,
      "loss": 2.1333,
      "nll_loss": 2.08317232131958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19413498044013977,
      "rewards/margins": 0.03840385749936104,
      "rewards/rejected": -0.2325388491153717,
      "step": 84
    },
    {
      "epoch": 0.23472557818432863,
      "grad_norm": 0.41432732343673706,
      "learning_rate": 3.853211009174313e-06,
      "log_odds_chosen": 0.5745496153831482,
      "log_odds_ratio": -0.4579155147075653,
      "logits/chosen": -0.3449418544769287,
      "logits/rejected": -0.19380588829517365,
      "logps/chosen": -1.8625164031982422,
      "logps/rejected": -2.3706510066986084,
      "loss": 2.0375,
      "nll_loss": 1.9917008876800537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1862516552209854,
      "rewards/margins": 0.050813477486371994,
      "rewards/rejected": -0.2370651215314865,
      "step": 85
    },
    {
      "epoch": 0.2374870555747325,
      "grad_norm": 0.438418984413147,
      "learning_rate": 3.89908256880734e-06,
      "log_odds_chosen": 0.580481231212616,
      "log_odds_ratio": -0.4620014429092407,
      "logits/chosen": -0.14796528220176697,
      "logits/rejected": -0.32091909646987915,
      "logps/chosen": -1.9756391048431396,
      "logps/rejected": -2.4986729621887207,
      "loss": 2.1484,
      "nll_loss": 2.102167844772339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19756391644477844,
      "rewards/margins": 0.052303388714790344,
      "rewards/rejected": -0.2498673051595688,
      "step": 86
    },
    {
      "epoch": 0.24024853296513635,
      "grad_norm": 0.4355674684047699,
      "learning_rate": 3.944954128440367e-06,
      "log_odds_chosen": 0.5157158970832825,
      "log_odds_ratio": -0.4764195382595062,
      "logits/chosen": -0.16625456511974335,
      "logits/rejected": -0.22959333658218384,
      "logps/chosen": -1.8700374364852905,
      "logps/rejected": -2.3208043575286865,
      "loss": 2.0401,
      "nll_loss": 1.9924601316452026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18700376152992249,
      "rewards/margins": 0.04507668316364288,
      "rewards/rejected": -0.23208042979240417,
      "step": 87
    },
    {
      "epoch": 0.2430100103555402,
      "grad_norm": 0.4138505756855011,
      "learning_rate": 3.9908256880733945e-06,
      "log_odds_chosen": 0.5811448693275452,
      "log_odds_ratio": -0.48126012086868286,
      "logits/chosen": -0.3125801980495453,
      "logits/rejected": -0.16787581145763397,
      "logps/chosen": -1.79574453830719,
      "logps/rejected": -2.3057985305786133,
      "loss": 1.9834,
      "nll_loss": 1.9353020191192627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17957444489002228,
      "rewards/margins": 0.051005423069000244,
      "rewards/rejected": -0.23057986795902252,
      "step": 88
    },
    {
      "epoch": 0.24577148774594407,
      "grad_norm": 0.3883982002735138,
      "learning_rate": 4.036697247706423e-06,
      "log_odds_chosen": 0.6218655705451965,
      "log_odds_ratio": -0.43868428468704224,
      "logits/chosen": -0.16164137423038483,
      "logits/rejected": -0.2850674092769623,
      "logps/chosen": -1.8971762657165527,
      "logps/rejected": -2.4469919204711914,
      "loss": 2.0667,
      "nll_loss": 2.0228123664855957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.189717635512352,
      "rewards/margins": 0.05498155951499939,
      "rewards/rejected": -0.24469918012619019,
      "step": 89
    },
    {
      "epoch": 0.24853296513634796,
      "grad_norm": 0.441839337348938,
      "learning_rate": 4.08256880733945e-06,
      "log_odds_chosen": 0.5872446894645691,
      "log_odds_ratio": -0.44401073455810547,
      "logits/chosen": -0.2670595347881317,
      "logits/rejected": -0.07647659629583359,
      "logps/chosen": -1.8952282667160034,
      "logps/rejected": -2.412400245666504,
      "loss": 2.066,
      "nll_loss": 2.021595001220703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18952281773090363,
      "rewards/margins": 0.051717206835746765,
      "rewards/rejected": -0.24124003946781158,
      "step": 90
    },
    {
      "epoch": 0.2512944425267518,
      "grad_norm": 0.4077044427394867,
      "learning_rate": 4.128440366972478e-06,
      "log_odds_chosen": 0.4384881556034088,
      "log_odds_ratio": -0.5076763033866882,
      "logits/chosen": -0.15786704421043396,
      "logits/rejected": -0.090408094227314,
      "logps/chosen": -1.878911018371582,
      "logps/rejected": -2.262026786804199,
      "loss": 2.047,
      "nll_loss": 1.9962477684020996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18789111077785492,
      "rewards/margins": 0.03831159323453903,
      "rewards/rejected": -0.22620268166065216,
      "step": 91
    },
    {
      "epoch": 0.2540559199171557,
      "grad_norm": 0.3839576244354248,
      "learning_rate": 4.174311926605505e-06,
      "log_odds_chosen": 0.42869681119918823,
      "log_odds_ratio": -0.5070521235466003,
      "logits/chosen": -0.08416657149791718,
      "logits/rejected": -0.4555465579032898,
      "logps/chosen": -1.9147799015045166,
      "logps/rejected": -2.292423725128174,
      "loss": 2.0814,
      "nll_loss": 2.0306570529937744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19147798418998718,
      "rewards/margins": 0.03776439279317856,
      "rewards/rejected": -0.22924238443374634,
      "step": 92
    },
    {
      "epoch": 0.25681739730755954,
      "grad_norm": 0.42368394136428833,
      "learning_rate": 4.220183486238532e-06,
      "log_odds_chosen": 0.5149534344673157,
      "log_odds_ratio": -0.47629106044769287,
      "logits/chosen": -0.13718552887439728,
      "logits/rejected": -0.20033138990402222,
      "logps/chosen": -1.9013878107070923,
      "logps/rejected": -2.3537254333496094,
      "loss": 2.0789,
      "nll_loss": 2.0312626361846924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1901387870311737,
      "rewards/margins": 0.04523376002907753,
      "rewards/rejected": -0.23537255823612213,
      "step": 93
    },
    {
      "epoch": 0.2595788746979634,
      "grad_norm": 0.38014668226242065,
      "learning_rate": 4.26605504587156e-06,
      "log_odds_chosen": 0.4552799165248871,
      "log_odds_ratio": -0.49941790103912354,
      "logits/chosen": -0.2649000883102417,
      "logits/rejected": -0.20182102918624878,
      "logps/chosen": -1.8007447719573975,
      "logps/rejected": -2.1948485374450684,
      "loss": 1.9637,
      "nll_loss": 1.9138000011444092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18007448315620422,
      "rewards/margins": 0.03941037505865097,
      "rewards/rejected": -0.2194848358631134,
      "step": 94
    },
    {
      "epoch": 0.26234035208836726,
      "grad_norm": 0.3550948202610016,
      "learning_rate": 4.311926605504588e-06,
      "log_odds_chosen": 0.36030256748199463,
      "log_odds_ratio": -0.533312976360321,
      "logits/chosen": -0.0772688165307045,
      "logits/rejected": -0.16052654385566711,
      "logps/chosen": -1.8254213333129883,
      "logps/rejected": -2.136554002761841,
      "loss": 2.0005,
      "nll_loss": 1.9472124576568604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1825421154499054,
      "rewards/margins": 0.03111325576901436,
      "rewards/rejected": -0.21365538239479065,
      "step": 95
    },
    {
      "epoch": 0.26510182947877114,
      "grad_norm": 0.40300413966178894,
      "learning_rate": 4.357798165137615e-06,
      "log_odds_chosen": 0.2818768620491028,
      "log_odds_ratio": -0.5676061511039734,
      "logits/chosen": -0.10504347085952759,
      "logits/rejected": -0.05843639373779297,
      "logps/chosen": -1.9888579845428467,
      "logps/rejected": -2.2374351024627686,
      "loss": 2.1718,
      "nll_loss": 2.11505126953125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19888579845428467,
      "rewards/margins": 0.02485768496990204,
      "rewards/rejected": -0.2237434834241867,
      "step": 96
    },
    {
      "epoch": 0.26786330686917503,
      "grad_norm": 0.398783802986145,
      "learning_rate": 4.403669724770643e-06,
      "log_odds_chosen": 0.5546217560768127,
      "log_odds_ratio": -0.46683651208877563,
      "logits/chosen": -0.008073419332504272,
      "logits/rejected": -0.08634719252586365,
      "logps/chosen": -1.8501688241958618,
      "logps/rejected": -2.342288017272949,
      "loss": 2.0318,
      "nll_loss": 1.9851207733154297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18501687049865723,
      "rewards/margins": 0.04921191930770874,
      "rewards/rejected": -0.23422878980636597,
      "step": 97
    },
    {
      "epoch": 0.27062478425957887,
      "grad_norm": 0.39449602365493774,
      "learning_rate": 4.44954128440367e-06,
      "log_odds_chosen": 0.4133303165435791,
      "log_odds_ratio": -0.5122652649879456,
      "logits/chosen": -0.05717798322439194,
      "logits/rejected": -0.1295921504497528,
      "logps/chosen": -1.9054274559020996,
      "logps/rejected": -2.266583204269409,
      "loss": 2.0813,
      "nll_loss": 2.030045509338379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19054275751113892,
      "rewards/margins": 0.03611557558178902,
      "rewards/rejected": -0.22665831446647644,
      "step": 98
    },
    {
      "epoch": 0.27338626164998275,
      "grad_norm": 0.39879634976387024,
      "learning_rate": 4.4954128440366975e-06,
      "log_odds_chosen": 0.547784149646759,
      "log_odds_ratio": -0.46036428213119507,
      "logits/chosen": -0.09109216928482056,
      "logits/rejected": -0.3699495494365692,
      "logps/chosen": -1.9557167291641235,
      "logps/rejected": -2.4426751136779785,
      "loss": 2.1161,
      "nll_loss": 2.070013999938965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1955716609954834,
      "rewards/margins": 0.04869585484266281,
      "rewards/rejected": -0.2442675083875656,
      "step": 99
    },
    {
      "epoch": 0.2761477390403866,
      "grad_norm": 0.3873852491378784,
      "learning_rate": 4.541284403669725e-06,
      "log_odds_chosen": 0.3448026478290558,
      "log_odds_ratio": -0.5393580198287964,
      "logits/chosen": -0.1656271517276764,
      "logits/rejected": -0.07856383174657822,
      "logps/chosen": -1.8394955396652222,
      "logps/rejected": -2.1358819007873535,
      "loss": 2.0365,
      "nll_loss": 1.9825859069824219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1839495450258255,
      "rewards/margins": 0.029638633131980896,
      "rewards/rejected": -0.2135881781578064,
      "step": 100
    },
    {
      "epoch": 0.2789092164307905,
      "grad_norm": 0.3506630063056946,
      "learning_rate": 4.587155963302753e-06,
      "log_odds_chosen": 0.33486518263816833,
      "log_odds_ratio": -0.5475890636444092,
      "logits/chosen": 0.009370687417685986,
      "logits/rejected": -0.18966984748840332,
      "logps/chosen": -1.9532427787780762,
      "logps/rejected": -2.2507667541503906,
      "loss": 2.132,
      "nll_loss": 2.077195405960083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19532427191734314,
      "rewards/margins": 0.02975238859653473,
      "rewards/rejected": -0.22507666051387787,
      "step": 101
    },
    {
      "epoch": 0.28167069382119436,
      "grad_norm": 0.33624136447906494,
      "learning_rate": 4.63302752293578e-06,
      "log_odds_chosen": 0.5504237413406372,
      "log_odds_ratio": -0.4613422155380249,
      "logits/chosen": -0.06238182261586189,
      "logits/rejected": -0.22859227657318115,
      "logps/chosen": -1.791447401046753,
      "logps/rejected": -2.2658450603485107,
      "loss": 1.9877,
      "nll_loss": 1.9415662288665771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1791447550058365,
      "rewards/margins": 0.04743976891040802,
      "rewards/rejected": -0.2265845388174057,
      "step": 102
    },
    {
      "epoch": 0.2844321712115982,
      "grad_norm": 0.3664388656616211,
      "learning_rate": 4.678899082568808e-06,
      "log_odds_chosen": 0.3437744081020355,
      "log_odds_ratio": -0.5439410209655762,
      "logits/chosen": -0.1386101245880127,
      "logits/rejected": -0.17609171569347382,
      "logps/chosen": -1.8379578590393066,
      "logps/rejected": -2.1358203887939453,
      "loss": 2.022,
      "nll_loss": 1.9676285982131958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1837957799434662,
      "rewards/margins": 0.0297862458974123,
      "rewards/rejected": -0.21358203887939453,
      "step": 103
    },
    {
      "epoch": 0.2871936486020021,
      "grad_norm": 0.3293968439102173,
      "learning_rate": 4.724770642201835e-06,
      "log_odds_chosen": 0.3870212137699127,
      "log_odds_ratio": -0.5231497883796692,
      "logits/chosen": -0.07520203292369843,
      "logits/rejected": -0.15402644872665405,
      "logps/chosen": -1.756553053855896,
      "logps/rejected": -2.086921453475952,
      "loss": 1.9488,
      "nll_loss": 1.8965202569961548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1756553202867508,
      "rewards/margins": 0.033036813139915466,
      "rewards/rejected": -0.20869213342666626,
      "step": 104
    },
    {
      "epoch": 0.2899551259924059,
      "grad_norm": 0.33918339014053345,
      "learning_rate": 4.770642201834863e-06,
      "log_odds_chosen": 0.4852727949619293,
      "log_odds_ratio": -0.4833363890647888,
      "logits/chosen": -0.004435461014509201,
      "logits/rejected": -0.1534399837255478,
      "logps/chosen": -1.7944328784942627,
      "logps/rejected": -2.2146499156951904,
      "loss": 1.9627,
      "nll_loss": 1.9144006967544556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17944329977035522,
      "rewards/margins": 0.042021699249744415,
      "rewards/rejected": -0.22146499156951904,
      "step": 105
    },
    {
      "epoch": 0.2927166033828098,
      "grad_norm": 0.3346327841281891,
      "learning_rate": 4.816513761467891e-06,
      "log_odds_chosen": 0.5177164673805237,
      "log_odds_ratio": -0.47983041405677795,
      "logits/chosen": -0.00399226788431406,
      "logits/rejected": -0.10182341933250427,
      "logps/chosen": -1.7794350385665894,
      "logps/rejected": -2.226297616958618,
      "loss": 1.9511,
      "nll_loss": 1.9030946493148804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17794349789619446,
      "rewards/margins": 0.044686250388622284,
      "rewards/rejected": -0.22262977063655853,
      "step": 106
    },
    {
      "epoch": 0.2954780807732137,
      "grad_norm": 0.3454085886478424,
      "learning_rate": 4.862385321100918e-06,
      "log_odds_chosen": 0.3980882465839386,
      "log_odds_ratio": -0.5184234380722046,
      "logits/chosen": 0.01898978091776371,
      "logits/rejected": -0.21249954402446747,
      "logps/chosen": -1.8531229496002197,
      "logps/rejected": -2.197577953338623,
      "loss": 2.0242,
      "nll_loss": 1.9724009037017822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18531231582164764,
      "rewards/margins": 0.03444547951221466,
      "rewards/rejected": -0.2197577804327011,
      "step": 107
    },
    {
      "epoch": 0.2982395581636175,
      "grad_norm": 0.3301767408847809,
      "learning_rate": 4.908256880733945e-06,
      "log_odds_chosen": 0.44768860936164856,
      "log_odds_ratio": -0.5000944137573242,
      "logits/chosen": 0.06930336356163025,
      "logits/rejected": -0.24058011174201965,
      "logps/chosen": -1.8849480152130127,
      "logps/rejected": -2.278297185897827,
      "loss": 2.0446,
      "nll_loss": 1.9945893287658691,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18849481642246246,
      "rewards/margins": 0.039334915578365326,
      "rewards/rejected": -0.2278297394514084,
      "step": 108
    },
    {
      "epoch": 0.3010010355540214,
      "grad_norm": 0.3244481086730957,
      "learning_rate": 4.954128440366973e-06,
      "log_odds_chosen": 0.6234428286552429,
      "log_odds_ratio": -0.44249945878982544,
      "logits/chosen": 0.045590970665216446,
      "logits/rejected": -0.3307313919067383,
      "logps/chosen": -1.8896514177322388,
      "logps/rejected": -2.4449257850646973,
      "loss": 2.0431,
      "nll_loss": 1.9988850355148315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18896515667438507,
      "rewards/margins": 0.0555274523794651,
      "rewards/rejected": -0.24449260532855988,
      "step": 109
    },
    {
      "epoch": 0.30376251294442524,
      "grad_norm": 0.31317761540412903,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.2916356325149536,
      "log_odds_ratio": -0.5743635892868042,
      "logits/chosen": 0.144297257065773,
      "logits/rejected": -0.26908552646636963,
      "logps/chosen": -1.8418867588043213,
      "logps/rejected": -2.0925679206848145,
      "loss": 2.0441,
      "nll_loss": 1.9866299629211426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18418867886066437,
      "rewards/margins": 0.02506813406944275,
      "rewards/rejected": -0.20925679802894592,
      "step": 110
    },
    {
      "epoch": 0.30652399033482913,
      "grad_norm": 0.29466620087623596,
      "learning_rate": 4.999987154315977e-06,
      "log_odds_chosen": 0.42353084683418274,
      "log_odds_ratio": -0.5111951231956482,
      "logits/chosen": 0.05306434631347656,
      "logits/rejected": -0.15503177046775818,
      "logps/chosen": -1.7407968044281006,
      "logps/rejected": -2.0989816188812256,
      "loss": 1.924,
      "nll_loss": 1.87285315990448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17407968640327454,
      "rewards/margins": 0.03581848740577698,
      "rewards/rejected": -0.2098981738090515,
      "step": 111
    },
    {
      "epoch": 0.309285467725233,
      "grad_norm": 0.3188883364200592,
      "learning_rate": 4.999948617395916e-06,
      "log_odds_chosen": 0.4043401777744293,
      "log_odds_ratio": -0.5218645930290222,
      "logits/chosen": 0.16884373128414154,
      "logits/rejected": -0.07719510793685913,
      "logps/chosen": -1.8007270097732544,
      "logps/rejected": -2.151782274246216,
      "loss": 1.9777,
      "nll_loss": 1.9254791736602783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18007270991802216,
      "rewards/margins": 0.035105518996715546,
      "rewards/rejected": -0.2151782363653183,
      "step": 112
    },
    {
      "epoch": 0.31204694511563685,
      "grad_norm": 0.3535216450691223,
      "learning_rate": 4.999884389635843e-06,
      "log_odds_chosen": 0.41415858268737793,
      "log_odds_ratio": -0.5124688744544983,
      "logits/chosen": 0.0558352991938591,
      "logits/rejected": -0.09803693741559982,
      "logps/chosen": -1.8450539112091064,
      "logps/rejected": -2.2039334774017334,
      "loss": 2.0336,
      "nll_loss": 1.9823946952819824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1845053881406784,
      "rewards/margins": 0.035887960344552994,
      "rewards/rejected": -0.2203933596611023,
      "step": 113
    },
    {
      "epoch": 0.31480842250604074,
      "grad_norm": 0.2991940379142761,
      "learning_rate": 4.9997944716957985e-06,
      "log_odds_chosen": 0.514029324054718,
      "log_odds_ratio": -0.47918182611465454,
      "logits/chosen": 0.16882193088531494,
      "logits/rejected": -0.24593651294708252,
      "logps/chosen": -1.8493305444717407,
      "logps/rejected": -2.298590898513794,
      "loss": 2.0272,
      "nll_loss": 1.9792673587799072,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18493306636810303,
      "rewards/margins": 0.044926032423973083,
      "rewards/rejected": -0.22985908389091492,
      "step": 114
    },
    {
      "epoch": 0.3175698998964446,
      "grad_norm": 0.28921976685523987,
      "learning_rate": 4.999678864499828e-06,
      "log_odds_chosen": 0.38468483090400696,
      "log_odds_ratio": -0.5222321152687073,
      "logits/chosen": 0.2794942557811737,
      "logits/rejected": -0.21004724502563477,
      "logps/chosen": -1.8734655380249023,
      "logps/rejected": -2.2063117027282715,
      "loss": 2.0344,
      "nll_loss": 1.9822089672088623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18734657764434814,
      "rewards/margins": 0.03328459709882736,
      "rewards/rejected": -0.2206311821937561,
      "step": 115
    },
    {
      "epoch": 0.32033137728684846,
      "grad_norm": 0.30199047923088074,
      "learning_rate": 4.999537569235975e-06,
      "log_odds_chosen": 0.35607457160949707,
      "log_odds_ratio": -0.5390717387199402,
      "logits/chosen": 0.17685924470424652,
      "logits/rejected": -0.16482022404670715,
      "logps/chosen": -1.8530224561691284,
      "logps/rejected": -2.1583948135375977,
      "loss": 2.0285,
      "nll_loss": 1.9746413230895996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1853022426366806,
      "rewards/margins": 0.030537229031324387,
      "rewards/rejected": -0.21583949029445648,
      "step": 116
    },
    {
      "epoch": 0.32309285467725235,
      "grad_norm": 0.305818647146225,
      "learning_rate": 4.999370587356267e-06,
      "log_odds_chosen": 0.3153924345970154,
      "log_odds_ratio": -0.5565903782844543,
      "logits/chosen": 0.17289672791957855,
      "logits/rejected": -0.3006405234336853,
      "logps/chosen": -1.9371519088745117,
      "logps/rejected": -2.214411497116089,
      "loss": 2.1096,
      "nll_loss": 2.0539159774780273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1937151849269867,
      "rewards/margins": 0.027725949883461,
      "rewards/rejected": -0.2214411348104477,
      "step": 117
    },
    {
      "epoch": 0.3258543320676562,
      "grad_norm": 0.324216365814209,
      "learning_rate": 4.9991779205767e-06,
      "log_odds_chosen": 0.29256871342658997,
      "log_odds_ratio": -0.5621715784072876,
      "logits/chosen": 0.055469777435064316,
      "logits/rejected": -0.5593094229698181,
      "logps/chosen": -1.808518886566162,
      "logps/rejected": -2.060049295425415,
      "loss": 1.9916,
      "nll_loss": 1.9353588819503784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18085187673568726,
      "rewards/margins": 0.02515305206179619,
      "rewards/rejected": -0.20600494742393494,
      "step": 118
    },
    {
      "epoch": 0.32861580945806007,
      "grad_norm": 0.316292941570282,
      "learning_rate": 4.998959570877224e-06,
      "log_odds_chosen": 0.391570121049881,
      "log_odds_ratio": -0.519210159778595,
      "logits/chosen": 0.29701903462409973,
      "logits/rejected": -0.248467817902565,
      "logps/chosen": -1.950477123260498,
      "logps/rejected": -2.2915000915527344,
      "loss": 2.1349,
      "nll_loss": 2.0829989910125732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19504770636558533,
      "rewards/margins": 0.034102290868759155,
      "rewards/rejected": -0.22914999723434448,
      "step": 119
    },
    {
      "epoch": 0.3313772868484639,
      "grad_norm": 0.29707539081573486,
      "learning_rate": 4.99871554050172e-06,
      "log_odds_chosen": 0.49879857897758484,
      "log_odds_ratio": -0.4770265221595764,
      "logits/chosen": 0.1394173949956894,
      "logits/rejected": -0.11445163935422897,
      "logps/chosen": -1.7406611442565918,
      "logps/rejected": -2.167613983154297,
      "loss": 1.9065,
      "nll_loss": 1.8588043451309204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17406611144542694,
      "rewards/margins": 0.042695268988609314,
      "rewards/rejected": -0.21676141023635864,
      "step": 120
    },
    {
      "epoch": 0.3341387642388678,
      "grad_norm": 0.3011818528175354,
      "learning_rate": 4.9984458319579775e-06,
      "log_odds_chosen": 0.4734145998954773,
      "log_odds_ratio": -0.48969942331314087,
      "logits/chosen": 0.10353265702724457,
      "logits/rejected": -0.22807854413986206,
      "logps/chosen": -1.7799421548843384,
      "logps/rejected": -2.1871492862701416,
      "loss": 1.9363,
      "nll_loss": 1.8872833251953125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17799422144889832,
      "rewards/margins": 0.04072071984410286,
      "rewards/rejected": -0.21871493756771088,
      "step": 121
    },
    {
      "epoch": 0.3369002416292717,
      "grad_norm": 0.28915688395500183,
      "learning_rate": 4.99815044801767e-06,
      "log_odds_chosen": 0.3440546989440918,
      "log_odds_ratio": -0.5443906784057617,
      "logits/chosen": 0.22890803217887878,
      "logits/rejected": -0.20038765668869019,
      "logps/chosen": -1.8269233703613281,
      "logps/rejected": -2.1230876445770264,
      "loss": 2.0144,
      "nll_loss": 1.959984540939331,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18269234895706177,
      "rewards/margins": 0.02961641550064087,
      "rewards/rejected": -0.21230876445770264,
      "step": 122
    },
    {
      "epoch": 0.3396617190196755,
      "grad_norm": 0.31248003244400024,
      "learning_rate": 4.9978293917163225e-06,
      "log_odds_chosen": 0.21560871601104736,
      "log_odds_ratio": -0.5958309769630432,
      "logits/chosen": 0.2199111431837082,
      "logits/rejected": -0.15359394252300262,
      "logps/chosen": -1.9724359512329102,
      "logps/rejected": -2.162384033203125,
      "loss": 2.1306,
      "nll_loss": 2.0710067749023438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1972435861825943,
      "rewards/margins": 0.01899481564760208,
      "rewards/rejected": -0.21623840928077698,
      "step": 123
    },
    {
      "epoch": 0.3424231964100794,
      "grad_norm": 0.3086913824081421,
      "learning_rate": 4.997482666353287e-06,
      "log_odds_chosen": 0.36193954944610596,
      "log_odds_ratio": -0.5371627807617188,
      "logits/chosen": 0.07905742526054382,
      "logits/rejected": -0.047993652522563934,
      "logps/chosen": -1.860518217086792,
      "logps/rejected": -2.175654649734497,
      "loss": 2.0362,
      "nll_loss": 1.982520341873169,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18605183064937592,
      "rewards/margins": 0.03151364251971245,
      "rewards/rejected": -0.21756546199321747,
      "step": 124
    },
    {
      "epoch": 0.34518467380048323,
      "grad_norm": 0.2956501543521881,
      "learning_rate": 4.997110275491702e-06,
      "log_odds_chosen": 0.5367758274078369,
      "log_odds_ratio": -0.4794245958328247,
      "logits/chosen": 0.17313790321350098,
      "logits/rejected": -0.5377508401870728,
      "logps/chosen": -1.7679404020309448,
      "logps/rejected": -2.2323243618011475,
      "loss": 1.9515,
      "nll_loss": 1.9035319089889526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17679405212402344,
      "rewards/margins": 0.04643838852643967,
      "rewards/rejected": -0.2232324331998825,
      "step": 125
    },
    {
      "epoch": 0.3479461511908871,
      "grad_norm": 0.2930620014667511,
      "learning_rate": 4.9967122229584614e-06,
      "log_odds_chosen": 0.5086734890937805,
      "log_odds_ratio": -0.47367364168167114,
      "logits/chosen": 0.20121866464614868,
      "logits/rejected": -0.3580709397792816,
      "logps/chosen": -1.772174596786499,
      "logps/rejected": -2.211575746536255,
      "loss": 1.9502,
      "nll_loss": 1.9028148651123047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17721746861934662,
      "rewards/margins": 0.04394011199474335,
      "rewards/rejected": -0.22115758061408997,
      "step": 126
    },
    {
      "epoch": 0.350707628581291,
      "grad_norm": 0.313006728887558,
      "learning_rate": 4.996288512844169e-06,
      "log_odds_chosen": 0.26984095573425293,
      "log_odds_ratio": -0.5770066380500793,
      "logits/chosen": 0.30613642930984497,
      "logits/rejected": -0.0801864042878151,
      "logps/chosen": -1.9204916954040527,
      "logps/rejected": -2.153423547744751,
      "loss": 2.1089,
      "nll_loss": 2.051222324371338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19204919040203094,
      "rewards/margins": 0.023293154314160347,
      "rewards/rejected": -0.21534234285354614,
      "step": 127
    },
    {
      "epoch": 0.35346910597169484,
      "grad_norm": 0.26883664727211,
      "learning_rate": 4.995839149503103e-06,
      "log_odds_chosen": 0.4242061972618103,
      "log_odds_ratio": -0.5166581273078918,
      "logits/chosen": 0.287309467792511,
      "logits/rejected": -0.28880101442337036,
      "logps/chosen": -1.7962003946304321,
      "logps/rejected": -2.1639606952667236,
      "loss": 1.9745,
      "nll_loss": 1.9228155612945557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17962004244327545,
      "rewards/margins": 0.036776017397642136,
      "rewards/rejected": -0.2163960486650467,
      "step": 128
    },
    {
      "epoch": 0.35623058336209873,
      "grad_norm": 0.3000660240650177,
      "learning_rate": 4.995364137553166e-06,
      "log_odds_chosen": 0.2731889486312866,
      "log_odds_ratio": -0.5769234299659729,
      "logits/chosen": 0.17702779173851013,
      "logits/rejected": -0.16427305340766907,
      "logps/chosen": -1.8508391380310059,
      "logps/rejected": -2.0855119228363037,
      "loss": 2.0272,
      "nll_loss": 1.9694753885269165,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18508392572402954,
      "rewards/margins": 0.023467278108000755,
      "rewards/rejected": -0.20855121314525604,
      "step": 129
    },
    {
      "epoch": 0.35899206075250256,
      "grad_norm": 0.27979782223701477,
      "learning_rate": 4.994863481875842e-06,
      "log_odds_chosen": 0.539387047290802,
      "log_odds_ratio": -0.4670449495315552,
      "logits/chosen": 0.19368596374988556,
      "logits/rejected": -0.4078698754310608,
      "logps/chosen": -1.7874817848205566,
      "logps/rejected": -2.2576088905334473,
      "loss": 1.9686,
      "nll_loss": 1.9218828678131104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17874819040298462,
      "rewards/margins": 0.04701270908117294,
      "rewards/rejected": -0.22576089203357697,
      "step": 130
    },
    {
      "epoch": 0.36175353814290645,
      "grad_norm": 0.2961781322956085,
      "learning_rate": 4.99433718761614e-06,
      "log_odds_chosen": 0.482613205909729,
      "log_odds_ratio": -0.48607659339904785,
      "logits/chosen": 0.3117329776287079,
      "logits/rejected": -0.19324824213981628,
      "logps/chosen": -1.8348233699798584,
      "logps/rejected": -2.2560348510742188,
      "loss": 1.9992,
      "nll_loss": 1.9505534172058105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1834823340177536,
      "rewards/margins": 0.042121127247810364,
      "rewards/rejected": -0.22560347616672516,
      "step": 131
    },
    {
      "epoch": 0.36451501553331034,
      "grad_norm": 0.27247706055641174,
      "learning_rate": 4.993785260182552e-06,
      "log_odds_chosen": 0.5040154457092285,
      "log_odds_ratio": -0.47964078187942505,
      "logits/chosen": 0.26086652278900146,
      "logits/rejected": -0.39971601963043213,
      "logps/chosen": -1.7693819999694824,
      "logps/rejected": -2.206263780593872,
      "loss": 1.9461,
      "nll_loss": 1.898113489151001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17693820595741272,
      "rewards/margins": 0.043688178062438965,
      "rewards/rejected": -0.2206263691186905,
      "step": 132
    },
    {
      "epoch": 0.36727649292371417,
      "grad_norm": 0.2559458613395691,
      "learning_rate": 4.993207705246983e-06,
      "log_odds_chosen": 0.37582507729530334,
      "log_odds_ratio": -0.529410719871521,
      "logits/chosen": 0.24356283247470856,
      "logits/rejected": -0.23510105907917023,
      "logps/chosen": -1.8250073194503784,
      "logps/rejected": -2.1527535915374756,
      "loss": 1.9998,
      "nll_loss": 1.9468413591384888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18250073492527008,
      "rewards/margins": 0.03277461230754852,
      "rewards/rejected": -0.2152753621339798,
      "step": 133
    },
    {
      "epoch": 0.37003797031411806,
      "grad_norm": 0.2661672532558441,
      "learning_rate": 4.992604528744705e-06,
      "log_odds_chosen": 0.4147174656391144,
      "log_odds_ratio": -0.5111517906188965,
      "logits/chosen": 0.23473379015922546,
      "logits/rejected": -0.5313011407852173,
      "logps/chosen": -1.8088786602020264,
      "logps/rejected": -2.1657345294952393,
      "loss": 1.985,
      "nll_loss": 1.9338966608047485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1808878630399704,
      "rewards/margins": 0.035685598850250244,
      "rewards/rejected": -0.21657346189022064,
      "step": 134
    },
    {
      "epoch": 0.37279944770452195,
      "grad_norm": 0.26244473457336426,
      "learning_rate": 4.9919757368742895e-06,
      "log_odds_chosen": 0.4870528280735016,
      "log_odds_ratio": -0.4892481565475464,
      "logits/chosen": 0.2686365842819214,
      "logits/rejected": -0.3376864194869995,
      "logps/chosen": -1.7298060655593872,
      "logps/rejected": -2.145728349685669,
      "loss": 1.9046,
      "nll_loss": 1.8556550741195679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17298059165477753,
      "rewards/margins": 0.04159224033355713,
      "rewards/rejected": -0.21457283198833466,
      "step": 135
    },
    {
      "epoch": 0.3755609250949258,
      "grad_norm": 0.26679527759552,
      "learning_rate": 4.991321336097546e-06,
      "log_odds_chosen": 0.4111718535423279,
      "log_odds_ratio": -0.512808084487915,
      "logits/chosen": 0.22426243126392365,
      "logits/rejected": -0.2763628363609314,
      "logps/chosen": -1.8473613262176514,
      "logps/rejected": -2.2056474685668945,
      "loss": 1.9977,
      "nll_loss": 1.9464530944824219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18473613262176514,
      "rewards/margins": 0.03582862764596939,
      "rewards/rejected": -0.22056476771831512,
      "step": 136
    },
    {
      "epoch": 0.37832240248532967,
      "grad_norm": 0.25576797127723694,
      "learning_rate": 4.990641333139455e-06,
      "log_odds_chosen": 0.435663640499115,
      "log_odds_ratio": -0.5039057731628418,
      "logits/chosen": 0.29907894134521484,
      "logits/rejected": -0.34149447083473206,
      "logps/chosen": -1.7228469848632812,
      "logps/rejected": -2.093015670776367,
      "loss": 1.8956,
      "nll_loss": 1.8452366590499878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17228470742702484,
      "rewards/margins": 0.037016861140728,
      "rewards/rejected": -0.20930157601833344,
      "step": 137
    },
    {
      "epoch": 0.3810838798757335,
      "grad_norm": 0.2576519250869751,
      "learning_rate": 4.989935734988098e-06,
      "log_odds_chosen": 0.32478412985801697,
      "log_odds_ratio": -0.5477995872497559,
      "logits/chosen": 0.3516974151134491,
      "logits/rejected": -0.4254434406757355,
      "logps/chosen": -1.8142131567001343,
      "logps/rejected": -2.0922889709472656,
      "loss": 1.9776,
      "nll_loss": 1.922855257987976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18142130970954895,
      "rewards/margins": 0.02780758962035179,
      "rewards/rejected": -0.20922890305519104,
      "step": 138
    },
    {
      "epoch": 0.3838453572661374,
      "grad_norm": 0.2815987765789032,
      "learning_rate": 4.989204548894589e-06,
      "log_odds_chosen": 0.32539641857147217,
      "log_odds_ratio": -0.5531569719314575,
      "logits/chosen": 0.4096101224422455,
      "logits/rejected": -0.19687263667583466,
      "logps/chosen": -1.8913159370422363,
      "logps/rejected": -2.1725118160247803,
      "loss": 2.0515,
      "nll_loss": 1.99615478515625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18913161754608154,
      "rewards/margins": 0.028119584545493126,
      "rewards/rejected": -0.21725118160247803,
      "step": 139
    },
    {
      "epoch": 0.3866068346565413,
      "grad_norm": 0.25471100211143494,
      "learning_rate": 4.988447782372996e-06,
      "log_odds_chosen": 0.3162972331047058,
      "log_odds_ratio": -0.553735077381134,
      "logits/chosen": 0.2287934273481369,
      "logits/rejected": -0.36916857957839966,
      "logps/chosen": -1.717573642730713,
      "logps/rejected": -1.986279010772705,
      "loss": 1.8883,
      "nll_loss": 1.8329135179519653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17175735533237457,
      "rewards/margins": 0.026870528236031532,
      "rewards/rejected": -0.19862788915634155,
      "step": 140
    },
    {
      "epoch": 0.3893683120469451,
      "grad_norm": 0.23676469922065735,
      "learning_rate": 4.9876654432002655e-06,
      "log_odds_chosen": 0.41759181022644043,
      "log_odds_ratio": -0.5150130391120911,
      "logits/chosen": 0.3617481589317322,
      "logits/rejected": -0.5005238056182861,
      "logps/chosen": -1.7590631246566772,
      "logps/rejected": -2.120802640914917,
      "loss": 1.9118,
      "nll_loss": 1.8603452444076538,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17590634524822235,
      "rewards/margins": 0.03617396205663681,
      "rewards/rejected": -0.21208029985427856,
      "step": 141
    },
    {
      "epoch": 0.392129789437349,
      "grad_norm": 0.23819445073604584,
      "learning_rate": 4.986857539416144e-06,
      "log_odds_chosen": 0.514154851436615,
      "log_odds_ratio": -0.47449907660484314,
      "logits/chosen": 0.2892986834049225,
      "logits/rejected": -0.5338828563690186,
      "logps/chosen": -1.732762336730957,
      "logps/rejected": -2.1715476512908936,
      "loss": 1.8923,
      "nll_loss": 1.8448883295059204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17327623069286346,
      "rewards/margins": 0.04387851804494858,
      "rewards/rejected": -0.21715475618839264,
      "step": 142
    },
    {
      "epoch": 0.39489126682775283,
      "grad_norm": 0.2597619891166687,
      "learning_rate": 4.986024079323092e-06,
      "log_odds_chosen": 0.21202370524406433,
      "log_odds_ratio": -0.5989823341369629,
      "logits/chosen": 0.31219571828842163,
      "logits/rejected": -0.2251828908920288,
      "logps/chosen": -1.7001993656158447,
      "logps/rejected": -1.8785269260406494,
      "loss": 1.9001,
      "nll_loss": 1.840247392654419,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17001992464065552,
      "rewards/margins": 0.017832759767770767,
      "rewards/rejected": -0.18785269558429718,
      "step": 143
    },
    {
      "epoch": 0.3976527442181567,
      "grad_norm": 0.24414244294166565,
      "learning_rate": 4.985165071486201e-06,
      "log_odds_chosen": 0.42103180289268494,
      "log_odds_ratio": -0.506747305393219,
      "logits/chosen": 0.24320174753665924,
      "logits/rejected": -0.47630739212036133,
      "logps/chosen": -1.813495397567749,
      "logps/rejected": -2.177159309387207,
      "loss": 1.9628,
      "nll_loss": 1.9121257066726685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18134953081607819,
      "rewards/margins": 0.03636639565229416,
      "rewards/rejected": -0.21771591901779175,
      "step": 144
    },
    {
      "epoch": 0.4004142216085606,
      "grad_norm": 0.25120964646339417,
      "learning_rate": 4.984280524733107e-06,
      "log_odds_chosen": 0.4766540825366974,
      "log_odds_ratio": -0.4867539405822754,
      "logits/chosen": 0.23771557211875916,
      "logits/rejected": -0.3588354289531708,
      "logps/chosen": -1.7543095350265503,
      "logps/rejected": -2.1605966091156006,
      "loss": 1.92,
      "nll_loss": 1.8713701963424683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17543095350265503,
      "rewards/margins": 0.040628716349601746,
      "rewards/rejected": -0.21605966985225677,
      "step": 145
    },
    {
      "epoch": 0.40317569899896444,
      "grad_norm": 0.23762984573841095,
      "learning_rate": 4.983370448153896e-06,
      "log_odds_chosen": 0.3954075574874878,
      "log_odds_ratio": -0.5178768038749695,
      "logits/chosen": 0.2783900201320648,
      "logits/rejected": -0.3363473117351532,
      "logps/chosen": -1.7586162090301514,
      "logps/rejected": -2.097499370574951,
      "loss": 1.9169,
      "nll_loss": 1.8651440143585205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1758616417646408,
      "rewards/margins": 0.033888280391693115,
      "rewards/rejected": -0.20974992215633392,
      "step": 146
    },
    {
      "epoch": 0.4059371763893683,
      "grad_norm": 0.250926673412323,
      "learning_rate": 4.9824348511010115e-06,
      "log_odds_chosen": 0.37487757205963135,
      "log_odds_ratio": -0.5328046083450317,
      "logits/chosen": 0.3899734616279602,
      "logits/rejected": -0.3820725679397583,
      "logps/chosen": -1.8538525104522705,
      "logps/rejected": -2.1806483268737793,
      "loss": 2.0058,
      "nll_loss": 1.9525387287139893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18538527190685272,
      "rewards/margins": 0.03267957270145416,
      "rewards/rejected": -0.2180648297071457,
      "step": 147
    },
    {
      "epoch": 0.40869865377977216,
      "grad_norm": 0.24456721544265747,
      "learning_rate": 4.981473743189163e-06,
      "log_odds_chosen": 0.6152381300926208,
      "log_odds_ratio": -0.4415377974510193,
      "logits/chosen": 0.15124794840812683,
      "logits/rejected": -0.6728506684303284,
      "logps/chosen": -1.630881905555725,
      "logps/rejected": -2.1548542976379395,
      "loss": 1.8038,
      "nll_loss": 1.7596325874328613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16308818757534027,
      "rewards/margins": 0.05239725112915039,
      "rewards/rejected": -0.21548543870449066,
      "step": 148
    },
    {
      "epoch": 0.41146013117017605,
      "grad_norm": 0.2503897547721863,
      "learning_rate": 4.98048713429522e-06,
      "log_odds_chosen": 0.4051974415779114,
      "log_odds_ratio": -0.5133127570152283,
      "logits/chosen": 0.3494684100151062,
      "logits/rejected": -0.11723777651786804,
      "logps/chosen": -1.7437934875488281,
      "logps/rejected": -2.086435079574585,
      "loss": 1.9057,
      "nll_loss": 1.8543728590011597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1743793487548828,
      "rewards/margins": 0.034264158457517624,
      "rewards/rejected": -0.20864351093769073,
      "step": 149
    },
    {
      "epoch": 0.41422160856057993,
      "grad_norm": 0.26401370763778687,
      "learning_rate": 4.979475034558115e-06,
      "log_odds_chosen": 0.35506629943847656,
      "log_odds_ratio": -0.534768283367157,
      "logits/chosen": 0.4498825669288635,
      "logits/rejected": -0.32746273279190063,
      "logps/chosen": -1.8957396745681763,
      "logps/rejected": -2.205925464630127,
      "loss": 2.042,
      "nll_loss": 1.988513469696045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1895739734172821,
      "rewards/margins": 0.0310185756534338,
      "rewards/rejected": -0.22059254348278046,
      "step": 150
    },
    {
      "epoch": 0.41698308595098377,
      "grad_norm": 0.23593436181545258,
      "learning_rate": 4.978437454378741e-06,
      "log_odds_chosen": 0.36985495686531067,
      "log_odds_ratio": -0.5328302979469299,
      "logits/chosen": 0.4250433146953583,
      "logits/rejected": -0.38277071714401245,
      "logps/chosen": -1.7761732339859009,
      "logps/rejected": -2.0909667015075684,
      "loss": 1.942,
      "nll_loss": 1.8887526988983154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17761734127998352,
      "rewards/margins": 0.03147934749722481,
      "rewards/rejected": -0.20909668505191803,
      "step": 151
    },
    {
      "epoch": 0.41974456334138766,
      "grad_norm": 0.26158878207206726,
      "learning_rate": 4.977374404419838e-06,
      "log_odds_chosen": 0.39601171016693115,
      "log_odds_ratio": -0.5228374004364014,
      "logits/chosen": 0.42059236764907837,
      "logits/rejected": -0.3891626000404358,
      "logps/chosen": -1.7959572076797485,
      "logps/rejected": -2.140069007873535,
      "loss": 1.9499,
      "nll_loss": 1.8975740671157837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1795957237482071,
      "rewards/margins": 0.03441117703914642,
      "rewards/rejected": -0.21400688588619232,
      "step": 152
    },
    {
      "epoch": 0.4225060407317915,
      "grad_norm": 0.2561565339565277,
      "learning_rate": 4.976285895605888e-06,
      "log_odds_chosen": 0.4899553656578064,
      "log_odds_ratio": -0.4833296239376068,
      "logits/chosen": 0.37404921650886536,
      "logits/rejected": -0.5243082046508789,
      "logps/chosen": -1.7399574518203735,
      "logps/rejected": -2.1531152725219727,
      "loss": 1.8971,
      "nll_loss": 1.8487193584442139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1739957481622696,
      "rewards/margins": 0.04131579399108887,
      "rewards/rejected": -0.21531155705451965,
      "step": 153
    },
    {
      "epoch": 0.4252675181221954,
      "grad_norm": 0.2550884485244751,
      "learning_rate": 4.9751719391230055e-06,
      "log_odds_chosen": 0.25457680225372314,
      "log_odds_ratio": -0.5827968120574951,
      "logits/chosen": 0.426510751247406,
      "logits/rejected": -0.38418132066726685,
      "logps/chosen": -1.710974931716919,
      "logps/rejected": -1.9265732765197754,
      "loss": 1.8985,
      "nll_loss": 1.840250015258789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17109748721122742,
      "rewards/margins": 0.02155984938144684,
      "rewards/rejected": -0.19265732169151306,
      "step": 154
    },
    {
      "epoch": 0.42802899551259926,
      "grad_norm": 0.25417274236679077,
      "learning_rate": 4.974032546418816e-06,
      "log_odds_chosen": 0.47524771094322205,
      "log_odds_ratio": -0.49181026220321655,
      "logits/chosen": 0.43296438455581665,
      "logits/rejected": -0.41207337379455566,
      "logps/chosen": -1.756546139717102,
      "logps/rejected": -2.1634457111358643,
      "loss": 1.9365,
      "nll_loss": 1.8873004913330078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17565463483333588,
      "rewards/margins": 0.04068994149565697,
      "rewards/rejected": -0.21634456515312195,
      "step": 155
    },
    {
      "epoch": 0.4307904729030031,
      "grad_norm": 0.24467967450618744,
      "learning_rate": 4.9728677292023405e-06,
      "log_odds_chosen": 0.19028525054454803,
      "log_odds_ratio": -0.6092196702957153,
      "logits/chosen": 0.5236613750457764,
      "logits/rejected": -0.3672065734863281,
      "logps/chosen": -1.851803183555603,
      "logps/rejected": -2.0145559310913086,
      "loss": 2.0125,
      "nll_loss": 1.9516232013702393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18518033623695374,
      "rewards/margins": 0.016275260597467422,
      "rewards/rejected": -0.20145559310913086,
      "step": 156
    },
    {
      "epoch": 0.433551950293407,
      "grad_norm": 0.24561214447021484,
      "learning_rate": 4.971677499443882e-06,
      "log_odds_chosen": 0.34714600443840027,
      "log_odds_ratio": -0.538092315196991,
      "logits/chosen": 0.42429813742637634,
      "logits/rejected": -0.4320365786552429,
      "logps/chosen": -1.7550561428070068,
      "logps/rejected": -2.0503358840942383,
      "loss": 1.9273,
      "nll_loss": 1.873533010482788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1755056083202362,
      "rewards/margins": 0.029527965933084488,
      "rewards/rejected": -0.2050335705280304,
      "step": 157
    },
    {
      "epoch": 0.4363134276838108,
      "grad_norm": 0.23480936884880066,
      "learning_rate": 4.97046186937489e-06,
      "log_odds_chosen": 0.3259715735912323,
      "log_odds_ratio": -0.5574356913566589,
      "logits/chosen": 0.33684611320495605,
      "logits/rejected": -0.48997414112091064,
      "logps/chosen": -1.733799934387207,
      "logps/rejected": -2.011017084121704,
      "loss": 1.8941,
      "nll_loss": 1.8383519649505615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17338000237941742,
      "rewards/margins": 0.027721701189875603,
      "rewards/rejected": -0.20110172033309937,
      "step": 158
    },
    {
      "epoch": 0.4390749050742147,
      "grad_norm": 0.23975060880184174,
      "learning_rate": 4.9692208514878445e-06,
      "log_odds_chosen": 0.2344578355550766,
      "log_odds_ratio": -0.5858049392700195,
      "logits/chosen": 0.4837522804737091,
      "logits/rejected": -0.2647075951099396,
      "logps/chosen": -1.8640694618225098,
      "logps/rejected": -2.064138412475586,
      "loss": 2.0361,
      "nll_loss": 1.9775654077529907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1864069551229477,
      "rewards/margins": 0.02000689134001732,
      "rewards/rejected": -0.2064138650894165,
      "step": 159
    },
    {
      "epoch": 0.4418363824646186,
      "grad_norm": 0.25472894310951233,
      "learning_rate": 4.967954458536126e-06,
      "log_odds_chosen": 0.34545353055000305,
      "log_odds_ratio": -0.5416699051856995,
      "logits/chosen": 0.4849855303764343,
      "logits/rejected": -0.14890551567077637,
      "logps/chosen": -1.6954622268676758,
      "logps/rejected": -1.9883880615234375,
      "loss": 1.8756,
      "nll_loss": 1.821388602256775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1695462316274643,
      "rewards/margins": 0.029292574152350426,
      "rewards/rejected": -0.19883880019187927,
      "step": 160
    },
    {
      "epoch": 0.4445978598550224,
      "grad_norm": 0.2551233172416687,
      "learning_rate": 4.96666270353388e-06,
      "log_odds_chosen": 0.34612518548965454,
      "log_odds_ratio": -0.5391160845756531,
      "logits/chosen": 0.4446476995944977,
      "logits/rejected": -0.4060593545436859,
      "logps/chosen": -1.7577486038208008,
      "logps/rejected": -2.0534539222717285,
      "loss": 1.9068,
      "nll_loss": 1.8529114723205566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17577485740184784,
      "rewards/margins": 0.029570531100034714,
      "rewards/rejected": -0.20534539222717285,
      "step": 161
    },
    {
      "epoch": 0.4473593372454263,
      "grad_norm": 0.2522425651550293,
      "learning_rate": 4.965345599755888e-06,
      "log_odds_chosen": 0.3961338400840759,
      "log_odds_ratio": -0.520170271396637,
      "logits/chosen": 0.38032418489456177,
      "logits/rejected": -0.48995065689086914,
      "logps/chosen": -1.815320372581482,
      "logps/rejected": -2.156792640686035,
      "loss": 1.9657,
      "nll_loss": 1.9137252569198608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18153204023838043,
      "rewards/margins": 0.0341472253203392,
      "rewards/rejected": -0.21567925810813904,
      "step": 162
    },
    {
      "epoch": 0.45012081463583015,
      "grad_norm": 0.23575294017791748,
      "learning_rate": 4.964003160737429e-06,
      "log_odds_chosen": 0.42496663331985474,
      "log_odds_ratio": -0.5144410133361816,
      "logits/chosen": 0.4265897274017334,
      "logits/rejected": -0.6324371695518494,
      "logps/chosen": -1.7239199876785278,
      "logps/rejected": -2.0822784900665283,
      "loss": 1.8878,
      "nll_loss": 1.8363168239593506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17239199578762054,
      "rewards/margins": 0.03583585098385811,
      "rewards/rejected": -0.20822784304618835,
      "step": 163
    },
    {
      "epoch": 0.45288229202623403,
      "grad_norm": 0.22830626368522644,
      "learning_rate": 4.9626354002741424e-06,
      "log_odds_chosen": 0.39489883184432983,
      "log_odds_ratio": -0.519772469997406,
      "logits/chosen": 0.43731188774108887,
      "logits/rejected": -0.5182772874832153,
      "logps/chosen": -1.6721280813217163,
      "logps/rejected": -2.0018677711486816,
      "loss": 1.8343,
      "nll_loss": 1.782306432723999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1672128140926361,
      "rewards/margins": 0.032973967492580414,
      "rewards/rejected": -0.20018678903579712,
      "step": 164
    },
    {
      "epoch": 0.4556437694166379,
      "grad_norm": 0.25069254636764526,
      "learning_rate": 4.9612423324218816e-06,
      "log_odds_chosen": 0.4574624300003052,
      "log_odds_ratio": -0.4939710795879364,
      "logits/chosen": 0.4768182933330536,
      "logits/rejected": -0.6430546045303345,
      "logps/chosen": -1.8401563167572021,
      "logps/rejected": -2.23877215385437,
      "loss": 1.9978,
      "nll_loss": 1.9484000205993652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18401561677455902,
      "rewards/margins": 0.03986157849431038,
      "rewards/rejected": -0.2238771915435791,
      "step": 165
    },
    {
      "epoch": 0.45840524680704176,
      "grad_norm": 0.25569233298301697,
      "learning_rate": 4.959823971496575e-06,
      "log_odds_chosen": 0.31097978353500366,
      "log_odds_ratio": -0.5631955862045288,
      "logits/chosen": 0.39253953099250793,
      "logits/rejected": -0.3464045226573944,
      "logps/chosen": -1.7822060585021973,
      "logps/rejected": -2.0535507202148438,
      "loss": 1.9515,
      "nll_loss": 1.8951623439788818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17822058498859406,
      "rewards/margins": 0.027134478092193604,
      "rewards/rejected": -0.20535509288311005,
      "step": 166
    },
    {
      "epoch": 0.46116672419744564,
      "grad_norm": 0.2754010260105133,
      "learning_rate": 4.958380332074074e-06,
      "log_odds_chosen": 0.38330915570259094,
      "log_odds_ratio": -0.5226565003395081,
      "logits/chosen": 0.5406702756881714,
      "logits/rejected": -0.2918284833431244,
      "logps/chosen": -1.744588851928711,
      "logps/rejected": -2.0728600025177,
      "loss": 1.9256,
      "nll_loss": 1.873305320739746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17445889115333557,
      "rewards/margins": 0.03282713145017624,
      "rewards/rejected": -0.2072860151529312,
      "step": 167
    },
    {
      "epoch": 0.4639282015878495,
      "grad_norm": 0.24440248310565948,
      "learning_rate": 4.95691142899001e-06,
      "log_odds_chosen": 0.4813528060913086,
      "log_odds_ratio": -0.49559223651885986,
      "logits/chosen": 0.42939212918281555,
      "logits/rejected": -0.5100030303001404,
      "logps/chosen": -1.7333720922470093,
      "logps/rejected": -2.148804187774658,
      "loss": 1.8988,
      "nll_loss": 1.8492889404296875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17333722114562988,
      "rewards/margins": 0.04154320806264877,
      "rewards/rejected": -0.21488040685653687,
      "step": 168
    },
    {
      "epoch": 0.46668967897825336,
      "grad_norm": 0.24897295236587524,
      "learning_rate": 4.955417277339633e-06,
      "log_odds_chosen": 0.3721994459629059,
      "log_odds_ratio": -0.5288943648338318,
      "logits/chosen": 0.4496070444583893,
      "logits/rejected": -0.5069053173065186,
      "logps/chosen": -1.7618987560272217,
      "logps/rejected": -2.0793538093566895,
      "loss": 1.9112,
      "nll_loss": 1.858320713043213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17618988454341888,
      "rewards/margins": 0.03174550086259842,
      "rewards/rejected": -0.2079353779554367,
      "step": 169
    },
    {
      "epoch": 0.46945115636865725,
      "grad_norm": 0.24506047368049622,
      "learning_rate": 4.953897892477664e-06,
      "log_odds_chosen": 0.4158302843570709,
      "log_odds_ratio": -0.524972677230835,
      "logits/chosen": 0.39215707778930664,
      "logits/rejected": -0.6964855790138245,
      "logps/chosen": -1.6938952207565308,
      "logps/rejected": -2.0564043521881104,
      "loss": 1.8573,
      "nll_loss": 1.8047559261322021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1693895161151886,
      "rewards/margins": 0.03625092655420303,
      "rewards/rejected": -0.20564045011997223,
      "step": 170
    },
    {
      "epoch": 0.4722126337590611,
      "grad_norm": 0.25454598665237427,
      "learning_rate": 4.952353290018132e-06,
      "log_odds_chosen": 0.39419156312942505,
      "log_odds_ratio": -0.5228413939476013,
      "logits/chosen": 0.4652084410190582,
      "logits/rejected": -0.09862995892763138,
      "logps/chosen": -1.7650055885314941,
      "logps/rejected": -2.102964162826538,
      "loss": 1.9365,
      "nll_loss": 1.8842390775680542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17650054395198822,
      "rewards/margins": 0.033795878291130066,
      "rewards/rejected": -0.2102964073419571,
      "step": 171
    },
    {
      "epoch": 0.474974111149465,
      "grad_norm": 0.23166699707508087,
      "learning_rate": 4.950783485834218e-06,
      "log_odds_chosen": 0.45087021589279175,
      "log_odds_ratio": -0.4937146306037903,
      "logits/chosen": 0.43644362688064575,
      "logits/rejected": -0.45175108313560486,
      "logps/chosen": -1.6984238624572754,
      "logps/rejected": -2.0791146755218506,
      "loss": 1.8668,
      "nll_loss": 1.817420482635498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16984236240386963,
      "rewards/margins": 0.03806909918785095,
      "rewards/rejected": -0.20791146159172058,
      "step": 172
    },
    {
      "epoch": 0.4777355885398688,
      "grad_norm": 0.22430026531219482,
      "learning_rate": 4.949188496058089e-06,
      "log_odds_chosen": 0.3452419340610504,
      "log_odds_ratio": -0.5469062328338623,
      "logits/chosen": 0.42078280448913574,
      "logits/rejected": -0.4557611346244812,
      "logps/chosen": -1.6979789733886719,
      "logps/rejected": -1.9911997318267822,
      "loss": 1.8648,
      "nll_loss": 1.8101435899734497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1697978973388672,
      "rewards/margins": 0.029322080314159393,
      "rewards/rejected": -0.19911997020244598,
      "step": 173
    },
    {
      "epoch": 0.4804970659302727,
      "grad_norm": 0.25257107615470886,
      "learning_rate": 4.947568337080733e-06,
      "log_odds_chosen": 0.1399567574262619,
      "log_odds_ratio": -0.6351712942123413,
      "logits/chosen": 0.3786250352859497,
      "logits/rejected": -0.433152973651886,
      "logps/chosen": -1.8261394500732422,
      "logps/rejected": -1.9481170177459717,
      "loss": 1.9906,
      "nll_loss": 1.9270497560501099,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18261395394802094,
      "rewards/margins": 0.012197760865092278,
      "rewards/rejected": -0.19481170177459717,
      "step": 174
    },
    {
      "epoch": 0.4832585433206766,
      "grad_norm": 0.25380203127861023,
      "learning_rate": 4.945923025551789e-06,
      "log_odds_chosen": 0.37491294741630554,
      "log_odds_ratio": -0.5253455638885498,
      "logits/chosen": 0.44007402658462524,
      "logits/rejected": -0.5158473253250122,
      "logps/chosen": -1.7649321556091309,
      "logps/rejected": -2.0859415531158447,
      "loss": 1.9061,
      "nll_loss": 1.8535852432250977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17649321258068085,
      "rewards/margins": 0.032100923359394073,
      "rewards/rejected": -0.20859414339065552,
      "step": 175
    },
    {
      "epoch": 0.4860200207110804,
      "grad_norm": 0.23105277121067047,
      "learning_rate": 4.944252578379379e-06,
      "log_odds_chosen": 0.2923380434513092,
      "log_odds_ratio": -0.5638449788093567,
      "logits/chosen": 0.3620964288711548,
      "logits/rejected": -0.7222499847412109,
      "logps/chosen": -1.6879165172576904,
      "logps/rejected": -1.9339208602905273,
      "loss": 1.8481,
      "nll_loss": 1.7917577028274536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16879163682460785,
      "rewards/margins": 0.024600449949502945,
      "rewards/rejected": -0.1933920979499817,
      "step": 176
    },
    {
      "epoch": 0.4887814981014843,
      "grad_norm": 0.2548852264881134,
      "learning_rate": 4.942557012729933e-06,
      "log_odds_chosen": 0.37703031301498413,
      "log_odds_ratio": -0.5374451875686646,
      "logits/chosen": 0.451770544052124,
      "logits/rejected": -0.6359795331954956,
      "logps/chosen": -1.7573479413986206,
      "logps/rejected": -2.0781285762786865,
      "loss": 1.9276,
      "nll_loss": 1.873888373374939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17573478817939758,
      "rewards/margins": 0.03207805007696152,
      "rewards/rejected": -0.2078128606081009,
      "step": 177
    },
    {
      "epoch": 0.49154297549188813,
      "grad_norm": 0.24361710250377655,
      "learning_rate": 4.940836346028011e-06,
      "log_odds_chosen": 0.564251184463501,
      "log_odds_ratio": -0.4567793309688568,
      "logits/chosen": 0.36331912875175476,
      "logits/rejected": -0.6997017860412598,
      "logps/chosen": -1.7021052837371826,
      "logps/rejected": -2.1804330348968506,
      "loss": 1.8632,
      "nll_loss": 1.8174842596054077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17021054029464722,
      "rewards/margins": 0.047832753509283066,
      "rewards/rejected": -0.2180432826280594,
      "step": 178
    },
    {
      "epoch": 0.494304452882292,
      "grad_norm": 0.25140658020973206,
      "learning_rate": 4.9390905959561254e-06,
      "log_odds_chosen": 0.42371851205825806,
      "log_odds_ratio": -0.5053101778030396,
      "logits/chosen": 0.4960322678089142,
      "logits/rejected": -0.7143914699554443,
      "logps/chosen": -1.8060599565505981,
      "logps/rejected": -2.168437957763672,
      "loss": 1.9361,
      "nll_loss": 1.8855350017547607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18060600757598877,
      "rewards/margins": 0.03623779118061066,
      "rewards/rejected": -0.21684379875659943,
      "step": 179
    },
    {
      "epoch": 0.4970659302726959,
      "grad_norm": 0.25267475843429565,
      "learning_rate": 4.937319780454559e-06,
      "log_odds_chosen": 0.24488888680934906,
      "log_odds_ratio": -0.5824177861213684,
      "logits/chosen": 0.43725982308387756,
      "logits/rejected": -0.5809499621391296,
      "logps/chosen": -1.7686806917190552,
      "logps/rejected": -1.9778132438659668,
      "loss": 1.9319,
      "nll_loss": 1.873632550239563,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17686808109283447,
      "rewards/margins": 0.020913248881697655,
      "rewards/rejected": -0.19778132438659668,
      "step": 180
    },
    {
      "epoch": 0.49982740766309974,
      "grad_norm": 0.2517687976360321,
      "learning_rate": 4.935523917721182e-06,
      "log_odds_chosen": 0.3661497235298157,
      "log_odds_ratio": -0.5325534343719482,
      "logits/chosen": 0.4761643409729004,
      "logits/rejected": -0.4684900641441345,
      "logps/chosen": -1.7682512998580933,
      "logps/rejected": -2.079028844833374,
      "loss": 1.9513,
      "nll_loss": 1.8980205059051514,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1768251359462738,
      "rewards/margins": 0.031077751889824867,
      "rewards/rejected": -0.20790287852287292,
      "step": 181
    },
    {
      "epoch": 0.5025888850535036,
      "grad_norm": 0.23882247507572174,
      "learning_rate": 4.933703026211262e-06,
      "log_odds_chosen": 0.40052229166030884,
      "log_odds_ratio": -0.5199273824691772,
      "logits/chosen": 0.5116904377937317,
      "logits/rejected": -0.8737332820892334,
      "logps/chosen": -1.7753740549087524,
      "logps/rejected": -2.1211981773376465,
      "loss": 1.9275,
      "nll_loss": 1.8754714727401733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17753739655017853,
      "rewards/margins": 0.03458239883184433,
      "rewards/rejected": -0.21211980283260345,
      "step": 182
    },
    {
      "epoch": 0.5053503624439075,
      "grad_norm": 0.23901337385177612,
      "learning_rate": 4.931857124637276e-06,
      "log_odds_chosen": 0.3831828832626343,
      "log_odds_ratio": -0.5294877886772156,
      "logits/chosen": 0.44559329748153687,
      "logits/rejected": -0.512153685092926,
      "logps/chosen": -1.660269021987915,
      "logps/rejected": -1.9844969511032104,
      "loss": 1.8228,
      "nll_loss": 1.7698148488998413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16602689027786255,
      "rewards/margins": 0.032422810792922974,
      "rewards/rejected": -0.19844970107078552,
      "step": 183
    },
    {
      "epoch": 0.5081118398343114,
      "grad_norm": 0.26273974776268005,
      "learning_rate": 4.92998623196872e-06,
      "log_odds_chosen": 0.40838003158569336,
      "log_odds_ratio": -0.5120916366577148,
      "logits/chosen": 0.36793801188468933,
      "logits/rejected": -0.883270263671875,
      "logps/chosen": -1.7183860540390015,
      "logps/rejected": -2.0646042823791504,
      "loss": 1.8679,
      "nll_loss": 1.8166720867156982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17183861136436462,
      "rewards/margins": 0.034621842205524445,
      "rewards/rejected": -0.20646044611930847,
      "step": 184
    },
    {
      "epoch": 0.5108733172247152,
      "grad_norm": 0.24954968690872192,
      "learning_rate": 4.92809036743191e-06,
      "log_odds_chosen": 0.44134509563446045,
      "log_odds_ratio": -0.5009733438491821,
      "logits/chosen": 0.38025662302970886,
      "logits/rejected": -0.6959909200668335,
      "logps/chosen": -1.6237648725509644,
      "logps/rejected": -1.991347312927246,
      "loss": 1.7726,
      "nll_loss": 1.7225314378738403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1623764932155609,
      "rewards/margins": 0.03675824776291847,
      "rewards/rejected": -0.19913475215435028,
      "step": 185
    },
    {
      "epoch": 0.5136347946151191,
      "grad_norm": 0.24785040318965912,
      "learning_rate": 4.926169550509787e-06,
      "log_odds_chosen": 0.304913192987442,
      "log_odds_ratio": -0.5551115274429321,
      "logits/chosen": 0.49904897809028625,
      "logits/rejected": -0.5060792565345764,
      "logps/chosen": -1.752536416053772,
      "logps/rejected": -2.009927749633789,
      "loss": 1.8926,
      "nll_loss": 1.837074875831604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17525365948677063,
      "rewards/margins": 0.0257391519844532,
      "rewards/rejected": -0.20099279284477234,
      "step": 186
    },
    {
      "epoch": 0.516396272005523,
      "grad_norm": 0.25260496139526367,
      "learning_rate": 4.924223800941718e-06,
      "log_odds_chosen": 0.18274395167827606,
      "log_odds_ratio": -0.6112073659896851,
      "logits/chosen": 0.4272249937057495,
      "logits/rejected": -0.5964298248291016,
      "logps/chosen": -1.7245608568191528,
      "logps/rejected": -1.87729012966156,
      "loss": 1.8858,
      "nll_loss": 1.824722170829773,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17245608568191528,
      "rewards/margins": 0.015272947028279305,
      "rewards/rejected": -0.18772903084754944,
      "step": 187
    },
    {
      "epoch": 0.5191577493959268,
      "grad_norm": 0.25974321365356445,
      "learning_rate": 4.9222531387232885e-06,
      "log_odds_chosen": 0.2772579491138458,
      "log_odds_ratio": -0.5700020790100098,
      "logits/chosen": 0.5035889148712158,
      "logits/rejected": -0.6021788120269775,
      "logps/chosen": -1.810874581336975,
      "logps/rejected": -2.048560380935669,
      "loss": 1.9601,
      "nll_loss": 1.903147578239441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.181087464094162,
      "rewards/margins": 0.023768583312630653,
      "rewards/rejected": -0.2048560529947281,
      "step": 188
    },
    {
      "epoch": 0.5219192267863307,
      "grad_norm": 0.2509472668170929,
      "learning_rate": 4.920257584106104e-06,
      "log_odds_chosen": 0.2922664284706116,
      "log_odds_ratio": -0.5610611438751221,
      "logits/chosen": 0.4223700761795044,
      "logits/rejected": -0.8161606192588806,
      "logps/chosen": -1.7685585021972656,
      "logps/rejected": -2.017383575439453,
      "loss": 1.9162,
      "nll_loss": 1.8600515127182007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17685584723949432,
      "rewards/margins": 0.02488252893090248,
      "rewards/rejected": -0.2017383873462677,
      "step": 189
    },
    {
      "epoch": 0.5246807041767345,
      "grad_norm": 0.26333701610565186,
      "learning_rate": 4.918237157597574e-06,
      "log_odds_chosen": 0.367781400680542,
      "log_odds_ratio": -0.5288804769515991,
      "logits/chosen": 0.34821969270706177,
      "logits/rejected": -0.5903292894363403,
      "logps/chosen": -1.7087079286575317,
      "logps/rejected": -2.0187838077545166,
      "loss": 1.8792,
      "nll_loss": 1.8263163566589355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1708707958459854,
      "rewards/margins": 0.03100760281085968,
      "rewards/rejected": -0.2018783837556839,
      "step": 190
    },
    {
      "epoch": 0.5274421815671384,
      "grad_norm": 0.2659012973308563,
      "learning_rate": 4.916191879960708e-06,
      "log_odds_chosen": 0.41825154423713684,
      "log_odds_ratio": -0.5096725225448608,
      "logits/chosen": 0.3918278217315674,
      "logits/rejected": -0.7869745492935181,
      "logps/chosen": -1.819606900215149,
      "logps/rejected": -2.178121328353882,
      "loss": 1.9644,
      "nll_loss": 1.9134495258331299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18196068704128265,
      "rewards/margins": 0.03585144132375717,
      "rewards/rejected": -0.21781213581562042,
      "step": 191
    },
    {
      "epoch": 0.5302036589575423,
      "grad_norm": 0.2279636263847351,
      "learning_rate": 4.914121772213898e-06,
      "log_odds_chosen": 0.3906615972518921,
      "log_odds_ratio": -0.5185546875,
      "logits/chosen": 0.4513569176197052,
      "logits/rejected": -0.5819852948188782,
      "logps/chosen": -1.6937000751495361,
      "logps/rejected": -2.0224616527557373,
      "loss": 1.8302,
      "nll_loss": 1.778322458267212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16937001049518585,
      "rewards/margins": 0.03287617862224579,
      "rewards/rejected": -0.20224617421627045,
      "step": 192
    },
    {
      "epoch": 0.5329651363479462,
      "grad_norm": 0.25850534439086914,
      "learning_rate": 4.912026855630703e-06,
      "log_odds_chosen": 0.4198724031448364,
      "log_odds_ratio": -0.5083851218223572,
      "logits/chosen": 0.44823789596557617,
      "logits/rejected": -0.45316439867019653,
      "logps/chosen": -1.7062734365463257,
      "logps/rejected": -2.061528205871582,
      "loss": 1.8534,
      "nll_loss": 1.8025155067443848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17062735557556152,
      "rewards/margins": 0.035525478422641754,
      "rewards/rejected": -0.20615282654762268,
      "step": 193
    },
    {
      "epoch": 0.5357266137383501,
      "grad_norm": 0.2361551970243454,
      "learning_rate": 4.909907151739634e-06,
      "log_odds_chosen": 0.3865026831626892,
      "log_odds_ratio": -0.527399480342865,
      "logits/chosen": 0.3759016692638397,
      "logits/rejected": -0.6874773502349854,
      "logps/chosen": -1.6837282180786133,
      "logps/rejected": -2.00811505317688,
      "loss": 1.8406,
      "nll_loss": 1.7878108024597168,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16837282478809357,
      "rewards/margins": 0.03243869170546532,
      "rewards/rejected": -0.200811505317688,
      "step": 194
    },
    {
      "epoch": 0.5384880911287538,
      "grad_norm": 0.25059905648231506,
      "learning_rate": 4.907762682323926e-06,
      "log_odds_chosen": 0.5617655515670776,
      "log_odds_ratio": -0.4947778284549713,
      "logits/chosen": 0.4162059426307678,
      "logits/rejected": -0.6656520366668701,
      "logps/chosen": -1.6942963600158691,
      "logps/rejected": -2.1570441722869873,
      "loss": 1.8694,
      "nll_loss": 1.8199554681777954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16942965984344482,
      "rewards/margins": 0.046274758875370026,
      "rewards/rejected": -0.21570439636707306,
      "step": 195
    },
    {
      "epoch": 0.5412495685191577,
      "grad_norm": 0.24844138324260712,
      "learning_rate": 4.905593469421323e-06,
      "log_odds_chosen": 0.2864172160625458,
      "log_odds_ratio": -0.5658568143844604,
      "logits/chosen": 0.40823209285736084,
      "logits/rejected": -1.0584533214569092,
      "logps/chosen": -1.6995760202407837,
      "logps/rejected": -1.9419208765029907,
      "loss": 1.8573,
      "nll_loss": 1.8007633686065674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16995760798454285,
      "rewards/margins": 0.024234486743807793,
      "rewards/rejected": -0.1941920816898346,
      "step": 196
    },
    {
      "epoch": 0.5440110459095616,
      "grad_norm": 0.25183677673339844,
      "learning_rate": 4.90339953532384e-06,
      "log_odds_chosen": 0.32565394043922424,
      "log_odds_ratio": -0.5478559732437134,
      "logits/chosen": 0.4504649341106415,
      "logits/rejected": -0.6915105581283569,
      "logps/chosen": -1.723501443862915,
      "logps/rejected": -1.9977482557296753,
      "loss": 1.8793,
      "nll_loss": 1.8244800567626953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17235015332698822,
      "rewards/margins": 0.027424685657024384,
      "rewards/rejected": -0.199774831533432,
      "step": 197
    },
    {
      "epoch": 0.5467725232999655,
      "grad_norm": 0.24120980501174927,
      "learning_rate": 4.901180902577549e-06,
      "log_odds_chosen": 0.36884185671806335,
      "log_odds_ratio": -0.5313685536384583,
      "logits/chosen": 0.47393080592155457,
      "logits/rejected": -0.8454681038856506,
      "logps/chosen": -1.7091484069824219,
      "logps/rejected": -2.021761655807495,
      "loss": 1.8562,
      "nll_loss": 1.80307936668396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17091482877731323,
      "rewards/margins": 0.03126133233308792,
      "rewards/rejected": -0.20217616856098175,
      "step": 198
    },
    {
      "epoch": 0.5495340006903694,
      "grad_norm": 0.26695704460144043,
      "learning_rate": 4.8989375939823305e-06,
      "log_odds_chosen": 0.2716186046600342,
      "log_odds_ratio": -0.5816208124160767,
      "logits/chosen": 0.43514788150787354,
      "logits/rejected": -0.9399389028549194,
      "logps/chosen": -1.6752939224243164,
      "logps/rejected": -1.9062312841415405,
      "loss": 1.829,
      "nll_loss": 1.770880937576294,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1675294041633606,
      "rewards/margins": 0.023093728348612785,
      "rewards/rejected": -0.19062313437461853,
      "step": 199
    },
    {
      "epoch": 0.5522954780807732,
      "grad_norm": 0.248373344540596,
      "learning_rate": 4.896669632591652e-06,
      "log_odds_chosen": 0.43290525674819946,
      "log_odds_ratio": -0.5045643448829651,
      "logits/chosen": 0.41075634956359863,
      "logits/rejected": -0.6819908618927002,
      "logps/chosen": -1.6926066875457764,
      "logps/rejected": -2.0592057704925537,
      "loss": 1.8492,
      "nll_loss": 1.7987439632415771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1692606657743454,
      "rewards/margins": 0.03665991127490997,
      "rewards/rejected": -0.20592057704925537,
      "step": 200
    },
    {
      "epoch": 0.5550569554711771,
      "grad_norm": 0.2426389902830124,
      "learning_rate": 4.894377041712327e-06,
      "log_odds_chosen": 0.3046472668647766,
      "log_odds_ratio": -0.5560302138328552,
      "logits/chosen": 0.4586237370967865,
      "logits/rejected": -0.889277458190918,
      "logps/chosen": -1.7483210563659668,
      "logps/rejected": -2.0069689750671387,
      "loss": 1.8995,
      "nll_loss": 1.8438801765441895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17483210563659668,
      "rewards/margins": 0.025864800438284874,
      "rewards/rejected": -0.2006969153881073,
      "step": 201
    },
    {
      "epoch": 0.557818432861581,
      "grad_norm": 0.2384442389011383,
      "learning_rate": 4.892059844904273e-06,
      "log_odds_chosen": 0.35407793521881104,
      "log_odds_ratio": -0.5373241901397705,
      "logits/chosen": 0.3391227722167969,
      "logits/rejected": -0.8238649964332581,
      "logps/chosen": -1.6727503538131714,
      "logps/rejected": -1.971817135810852,
      "loss": 1.8314,
      "nll_loss": 1.7776691913604736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16727504134178162,
      "rewards/margins": 0.02990666963160038,
      "rewards/rejected": -0.19718170166015625,
      "step": 202
    },
    {
      "epoch": 0.5605799102519848,
      "grad_norm": 0.2237497717142105,
      "learning_rate": 4.889718065980272e-06,
      "log_odds_chosen": 0.42560863494873047,
      "log_odds_ratio": -0.5102251768112183,
      "logits/chosen": 0.38399800658226013,
      "logits/rejected": -0.940986156463623,
      "logps/chosen": -1.7369155883789062,
      "logps/rejected": -2.0991053581237793,
      "loss": 1.8724,
      "nll_loss": 1.821379542350769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1736915558576584,
      "rewards/margins": 0.03621895611286163,
      "rewards/rejected": -0.2099105268716812,
      "step": 203
    },
    {
      "epoch": 0.5633413876423887,
      "grad_norm": 0.23220422863960266,
      "learning_rate": 4.8873517290057265e-06,
      "log_odds_chosen": 0.5465662479400635,
      "log_odds_ratio": -0.45910075306892395,
      "logits/chosen": 0.33986854553222656,
      "logits/rejected": -0.9626595973968506,
      "logps/chosen": -1.7113467454910278,
      "logps/rejected": -2.1808066368103027,
      "loss": 1.8594,
      "nll_loss": 1.8135318756103516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17113468050956726,
      "rewards/margins": 0.046945974230766296,
      "rewards/rejected": -0.21808065474033356,
      "step": 204
    },
    {
      "epoch": 0.5661028650327925,
      "grad_norm": 0.22727084159851074,
      "learning_rate": 4.88496085829841e-06,
      "log_odds_chosen": 0.3183567523956299,
      "log_odds_ratio": -0.5485174059867859,
      "logits/chosen": 0.38077837228775024,
      "logits/rejected": -0.3718608617782593,
      "logps/chosen": -1.682438611984253,
      "logps/rejected": -1.9494574069976807,
      "loss": 1.8312,
      "nll_loss": 1.7763221263885498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.168243870139122,
      "rewards/margins": 0.026701876893639565,
      "rewards/rejected": -0.19494575262069702,
      "step": 205
    },
    {
      "epoch": 0.5688643424231964,
      "grad_norm": 0.2435847967863083,
      "learning_rate": 4.882545478428219e-06,
      "log_odds_chosen": 0.4757111072540283,
      "log_odds_ratio": -0.5007758140563965,
      "logits/chosen": 0.4316224753856659,
      "logits/rejected": -0.9769002199172974,
      "logps/chosen": -1.6722346544265747,
      "logps/rejected": -2.073765277862549,
      "loss": 1.8362,
      "nll_loss": 1.7861530780792236,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1672234833240509,
      "rewards/margins": 0.04015304520726204,
      "rewards/rejected": -0.20737652480602264,
      "step": 206
    },
    {
      "epoch": 0.5716258198136003,
      "grad_norm": 0.28022557497024536,
      "learning_rate": 4.880105614216917e-06,
      "log_odds_chosen": 0.42549797892570496,
      "log_odds_ratio": -0.507975161075592,
      "logits/chosen": 0.42804068326950073,
      "logits/rejected": -0.8541020750999451,
      "logps/chosen": -1.8415427207946777,
      "logps/rejected": -2.209498405456543,
      "loss": 1.9733,
      "nll_loss": 1.9224536418914795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18415425717830658,
      "rewards/margins": 0.03679555654525757,
      "rewards/rejected": -0.22094982862472534,
      "step": 207
    },
    {
      "epoch": 0.5743872972040042,
      "grad_norm": 0.26431816816329956,
      "learning_rate": 4.8776412907378845e-06,
      "log_odds_chosen": 0.3722533583641052,
      "log_odds_ratio": -0.5265185832977295,
      "logits/chosen": 0.46087294816970825,
      "logits/rejected": -0.8973901271820068,
      "logps/chosen": -1.7529377937316895,
      "logps/rejected": -2.0698721408843994,
      "loss": 1.8939,
      "nll_loss": 1.8412883281707764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17529378831386566,
      "rewards/margins": 0.03169342502951622,
      "rewards/rejected": -0.206987202167511,
      "step": 208
    },
    {
      "epoch": 0.577148774594408,
      "grad_norm": 0.2286507785320282,
      "learning_rate": 4.875152533315859e-06,
      "log_odds_chosen": 0.42754417657852173,
      "log_odds_ratio": -0.5049228668212891,
      "logits/chosen": 0.3295610547065735,
      "logits/rejected": -0.821418285369873,
      "logps/chosen": -1.6234990358352661,
      "logps/rejected": -1.9790537357330322,
      "loss": 1.7729,
      "nll_loss": 1.7223597764968872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1623499095439911,
      "rewards/margins": 0.035555459558963776,
      "rewards/rejected": -0.19790537655353546,
      "step": 209
    },
    {
      "epoch": 0.5799102519848118,
      "grad_norm": 0.23942524194717407,
      "learning_rate": 4.872639367526672e-06,
      "log_odds_chosen": 0.4114229083061218,
      "log_odds_ratio": -0.5152523517608643,
      "logits/chosen": 0.3920520842075348,
      "logits/rejected": -0.7173675298690796,
      "logps/chosen": -1.6646122932434082,
      "logps/rejected": -2.0126848220825195,
      "loss": 1.8132,
      "nll_loss": 1.7616626024246216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16646124422550201,
      "rewards/margins": 0.0348072350025177,
      "rewards/rejected": -0.20126846432685852,
      "step": 210
    },
    {
      "epoch": 0.5826717293752157,
      "grad_norm": 0.259086012840271,
      "learning_rate": 4.870101819196992e-06,
      "log_odds_chosen": 0.37038204073905945,
      "log_odds_ratio": -0.5485732555389404,
      "logits/chosen": 0.33591514825820923,
      "logits/rejected": -0.7169915437698364,
      "logps/chosen": -1.7096450328826904,
      "logps/rejected": -2.028204917907715,
      "loss": 1.863,
      "nll_loss": 1.8081302642822266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17096450924873352,
      "rewards/margins": 0.031855978071689606,
      "rewards/rejected": -0.20282049477100372,
      "step": 211
    },
    {
      "epoch": 0.5854332067656196,
      "grad_norm": 0.23236605525016785,
      "learning_rate": 4.8675399144040535e-06,
      "log_odds_chosen": 0.48778051137924194,
      "log_odds_ratio": -0.4863608777523041,
      "logits/chosen": 0.3406648337841034,
      "logits/rejected": -1.1020050048828125,
      "logps/chosen": -1.6164040565490723,
      "logps/rejected": -2.0246694087982178,
      "loss": 1.7551,
      "nll_loss": 1.7064510583877563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16164040565490723,
      "rewards/margins": 0.04082653671503067,
      "rewards/rejected": -0.2024669200181961,
      "step": 212
    },
    {
      "epoch": 0.5881946841560235,
      "grad_norm": 0.2472049593925476,
      "learning_rate": 4.864953679475392e-06,
      "log_odds_chosen": 0.4090927541255951,
      "log_odds_ratio": -0.5185278654098511,
      "logits/chosen": 0.3806914687156677,
      "logits/rejected": -1.1645220518112183,
      "logps/chosen": -1.6705214977264404,
      "logps/rejected": -2.010647773742676,
      "loss": 1.7979,
      "nll_loss": 1.7460622787475586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16705216467380524,
      "rewards/margins": 0.03401262313127518,
      "rewards/rejected": -0.201064795255661,
      "step": 213
    },
    {
      "epoch": 0.5909561615464274,
      "grad_norm": 0.26932069659233093,
      "learning_rate": 4.862343140988573e-06,
      "log_odds_chosen": 0.38164323568344116,
      "log_odds_ratio": -0.5293680429458618,
      "logits/chosen": 0.35298576951026917,
      "logits/rejected": -0.9924853444099426,
      "logps/chosen": -1.7351062297821045,
      "logps/rejected": -2.062347173690796,
      "loss": 1.8652,
      "nll_loss": 1.8122597932815552,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1735106110572815,
      "rewards/margins": 0.03272408992052078,
      "rewards/rejected": -0.20623470842838287,
      "step": 214
    },
    {
      "epoch": 0.5937176389368312,
      "grad_norm": 0.26750069856643677,
      "learning_rate": 4.859708325770919e-06,
      "log_odds_chosen": 0.3453901410102844,
      "log_odds_ratio": -0.5410705804824829,
      "logits/chosen": 0.402724027633667,
      "logits/rejected": -0.7651013135910034,
      "logps/chosen": -1.736122727394104,
      "logps/rejected": -2.027735948562622,
      "loss": 1.901,
      "nll_loss": 1.8469077348709106,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17361226677894592,
      "rewards/margins": 0.029161330312490463,
      "rewards/rejected": -0.20277361571788788,
      "step": 215
    },
    {
      "epoch": 0.596479116327235,
      "grad_norm": 0.25545212626457214,
      "learning_rate": 4.857049260899233e-06,
      "log_odds_chosen": 0.42634040117263794,
      "log_odds_ratio": -0.5089380145072937,
      "logits/chosen": 0.2999016046524048,
      "logits/rejected": -0.9428575038909912,
      "logps/chosen": -1.6035995483398438,
      "logps/rejected": -1.9544857740402222,
      "loss": 1.7566,
      "nll_loss": 1.7056996822357178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1603599637746811,
      "rewards/margins": 0.03508862853050232,
      "rewards/rejected": -0.19544857740402222,
      "step": 216
    },
    {
      "epoch": 0.5992405937176389,
      "grad_norm": 0.2341543734073639,
      "learning_rate": 4.854365973699519e-06,
      "log_odds_chosen": 0.29267483949661255,
      "log_odds_ratio": -0.5718191862106323,
      "logits/chosen": 0.32583779096603394,
      "logits/rejected": -0.7815302610397339,
      "logps/chosen": -1.6951828002929688,
      "logps/rejected": -1.9353394508361816,
      "loss": 1.8596,
      "nll_loss": 1.8024464845657349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16951829195022583,
      "rewards/margins": 0.024015672504901886,
      "rewards/rejected": -0.19353395700454712,
      "step": 217
    },
    {
      "epoch": 0.6020020711080428,
      "grad_norm": 0.24366186559200287,
      "learning_rate": 4.851658491746707e-06,
      "log_odds_chosen": 0.3909580707550049,
      "log_odds_ratio": -0.5210633277893066,
      "logits/chosen": 0.25465232133865356,
      "logits/rejected": -0.9531072378158569,
      "logps/chosen": -1.6791120767593384,
      "logps/rejected": -2.0105059146881104,
      "loss": 1.8187,
      "nll_loss": 1.7665891647338867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16791123151779175,
      "rewards/margins": 0.03313935548067093,
      "rewards/rejected": -0.20105057954788208,
      "step": 218
    },
    {
      "epoch": 0.6047635484984467,
      "grad_norm": 0.24001233279705048,
      "learning_rate": 4.848926842864361e-06,
      "log_odds_chosen": 0.4899923503398895,
      "log_odds_ratio": -0.48127448558807373,
      "logits/chosen": 0.25574028491973877,
      "logits/rejected": -1.256288766860962,
      "logps/chosen": -1.6502352952957153,
      "logps/rejected": -2.064323902130127,
      "loss": 1.7891,
      "nll_loss": 1.740965723991394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.165023535490036,
      "rewards/margins": 0.04140886664390564,
      "rewards/rejected": -0.20643240213394165,
      "step": 219
    },
    {
      "epoch": 0.6075250258888505,
      "grad_norm": 0.23009978234767914,
      "learning_rate": 4.846171055124401e-06,
      "log_odds_chosen": 0.3280143141746521,
      "log_odds_ratio": -0.55921870470047,
      "logits/chosen": 0.35827893018722534,
      "logits/rejected": -0.7289958000183105,
      "logps/chosen": -1.7372446060180664,
      "logps/rejected": -2.02004075050354,
      "loss": 1.8677,
      "nll_loss": 1.811750888824463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1737244725227356,
      "rewards/margins": 0.028279609978199005,
      "rewards/rejected": -0.2020040899515152,
      "step": 220
    },
    {
      "epoch": 0.6102865032792544,
      "grad_norm": 0.24643385410308838,
      "learning_rate": 4.843391156846811e-06,
      "log_odds_chosen": 0.49847206473350525,
      "log_odds_ratio": -0.4771438539028168,
      "logits/chosen": 0.38028058409690857,
      "logits/rejected": -0.8463162779808044,
      "logps/chosen": -1.6676466464996338,
      "logps/rejected": -2.0871989727020264,
      "loss": 1.8045,
      "nll_loss": 1.756779432296753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16676466166973114,
      "rewards/margins": 0.04195523262023926,
      "rewards/rejected": -0.2087198942899704,
      "step": 221
    },
    {
      "epoch": 0.6130479806696583,
      "grad_norm": 0.23624517023563385,
      "learning_rate": 4.8405871765993435e-06,
      "log_odds_chosen": 0.5461086630821228,
      "log_odds_ratio": -0.4661559462547302,
      "logits/chosen": 0.2758100926876068,
      "logits/rejected": -0.9698406457901001,
      "logps/chosen": -1.716646671295166,
      "logps/rejected": -2.178037643432617,
      "loss": 1.8539,
      "nll_loss": 1.807306170463562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17166468501091003,
      "rewards/margins": 0.04613909870386124,
      "rewards/rejected": -0.21780376136302948,
      "step": 222
    },
    {
      "epoch": 0.6158094580600622,
      "grad_norm": 0.24216988682746887,
      "learning_rate": 4.837759143197237e-06,
      "log_odds_chosen": 0.33277636766433716,
      "log_odds_ratio": -0.543420672416687,
      "logits/chosen": 0.25846540927886963,
      "logits/rejected": -1.1584213972091675,
      "logps/chosen": -1.6766406297683716,
      "logps/rejected": -1.959399700164795,
      "loss": 1.818,
      "nll_loss": 1.7636725902557373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1676640659570694,
      "rewards/margins": 0.028275907039642334,
      "rewards/rejected": -0.19593995809555054,
      "step": 223
    },
    {
      "epoch": 0.618570935450466,
      "grad_norm": 0.22247816622257233,
      "learning_rate": 4.834907085702909e-06,
      "log_odds_chosen": 0.42663368582725525,
      "log_odds_ratio": -0.5054609775543213,
      "logits/chosen": 0.2946387827396393,
      "logits/rejected": -0.8483214378356934,
      "logps/chosen": -1.6203222274780273,
      "logps/rejected": -1.9752922058105469,
      "loss": 1.7654,
      "nll_loss": 1.7148088216781616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16203221678733826,
      "rewards/margins": 0.03549701347947121,
      "rewards/rejected": -0.19752921164035797,
      "step": 224
    },
    {
      "epoch": 0.6213324128408698,
      "grad_norm": 0.23438099026679993,
      "learning_rate": 4.832031033425663e-06,
      "log_odds_chosen": 0.4974308907985687,
      "log_odds_ratio": -0.4770738184452057,
      "logits/chosen": 0.22860127687454224,
      "logits/rejected": -1.1651134490966797,
      "logps/chosen": -1.5770584344863892,
      "logps/rejected": -1.98891282081604,
      "loss": 1.7128,
      "nll_loss": 1.6650840044021606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15770584344863892,
      "rewards/margins": 0.04118544980883598,
      "rewards/rejected": -0.198891282081604,
      "step": 225
    },
    {
      "epoch": 0.6240938902312737,
      "grad_norm": 0.2453928291797638,
      "learning_rate": 4.829131015921386e-06,
      "log_odds_chosen": 0.336994469165802,
      "log_odds_ratio": -0.5416135787963867,
      "logits/chosen": 0.40966925024986267,
      "logits/rejected": -0.7484245300292969,
      "logps/chosen": -1.7475402355194092,
      "logps/rejected": -2.035691499710083,
      "loss": 1.8881,
      "nll_loss": 1.833910346031189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17475402355194092,
      "rewards/margins": 0.0288151316344738,
      "rewards/rejected": -0.2035691738128662,
      "step": 226
    },
    {
      "epoch": 0.6268553676216776,
      "grad_norm": 0.22821033000946045,
      "learning_rate": 4.826207062992245e-06,
      "log_odds_chosen": 0.47645366191864014,
      "log_odds_ratio": -0.4914059638977051,
      "logits/chosen": 0.2782054543495178,
      "logits/rejected": -0.9900674223899841,
      "logps/chosen": -1.8174843788146973,
      "logps/rejected": -2.2320914268493652,
      "loss": 1.95,
      "nll_loss": 1.9008376598358154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1817484349012375,
      "rewards/margins": 0.041460707783699036,
      "rewards/rejected": -0.22320915758609772,
      "step": 227
    },
    {
      "epoch": 0.6296168450120815,
      "grad_norm": 0.2569068968296051,
      "learning_rate": 4.82325920468638e-06,
      "log_odds_chosen": 0.2413053661584854,
      "log_odds_ratio": -0.5858334302902222,
      "logits/chosen": 0.32741737365722656,
      "logits/rejected": -0.9221272468566895,
      "logps/chosen": -1.7791626453399658,
      "logps/rejected": -1.9837150573730469,
      "loss": 1.9203,
      "nll_loss": 1.8617504835128784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1779162585735321,
      "rewards/margins": 0.020455272868275642,
      "rewards/rejected": -0.1983715444803238,
      "step": 228
    },
    {
      "epoch": 0.6323783224024854,
      "grad_norm": 0.2548074424266815,
      "learning_rate": 4.820287471297598e-06,
      "log_odds_chosen": 0.4002269208431244,
      "log_odds_ratio": -0.5239609479904175,
      "logits/chosen": 0.21428313851356506,
      "logits/rejected": -0.9460724592208862,
      "logps/chosen": -1.6473997831344604,
      "logps/rejected": -1.9846879243850708,
      "loss": 1.7941,
      "nll_loss": 1.7416696548461914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1647399663925171,
      "rewards/margins": 0.03372883051633835,
      "rewards/rejected": -0.19846880435943604,
      "step": 229
    },
    {
      "epoch": 0.6351397997928891,
      "grad_norm": 0.22254504263401031,
      "learning_rate": 4.817291893365055e-06,
      "log_odds_chosen": 0.46622079610824585,
      "log_odds_ratio": -0.4967763423919678,
      "logits/chosen": 0.07556484639644623,
      "logits/rejected": -1.116629719734192,
      "logps/chosen": -1.5138487815856934,
      "logps/rejected": -1.8917471170425415,
      "loss": 1.675,
      "nll_loss": 1.625287413597107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15138490498065948,
      "rewards/margins": 0.03778982535004616,
      "rewards/rejected": -0.18917471170425415,
      "step": 230
    },
    {
      "epoch": 0.637901277183293,
      "grad_norm": 0.28197240829467773,
      "learning_rate": 4.81427250167295e-06,
      "log_odds_chosen": 0.24183571338653564,
      "log_odds_ratio": -0.583247721195221,
      "logits/chosen": 0.15436850488185883,
      "logits/rejected": -1.1790400743484497,
      "logps/chosen": -1.6747009754180908,
      "logps/rejected": -1.8760582208633423,
      "loss": 1.8184,
      "nll_loss": 1.7600996494293213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16747009754180908,
      "rewards/margins": 0.020135723054409027,
      "rewards/rejected": -0.1876058131456375,
      "step": 231
    },
    {
      "epoch": 0.6406627545736969,
      "grad_norm": 0.24836094677448273,
      "learning_rate": 4.811229327250204e-06,
      "log_odds_chosen": 0.45771628618240356,
      "log_odds_ratio": -0.5010443329811096,
      "logits/chosen": 0.25003117322921753,
      "logits/rejected": -1.1424946784973145,
      "logps/chosen": -1.714363694190979,
      "logps/rejected": -2.1092028617858887,
      "loss": 1.8544,
      "nll_loss": 1.8042795658111572,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1714363694190979,
      "rewards/margins": 0.03948391228914261,
      "rewards/rejected": -0.2109202891588211,
      "step": 232
    },
    {
      "epoch": 0.6434242319641008,
      "grad_norm": 0.23518826067447662,
      "learning_rate": 4.8081624013701435e-06,
      "log_odds_chosen": 0.4311872124671936,
      "log_odds_ratio": -0.5104647278785706,
      "logits/chosen": 0.1991138905286789,
      "logits/rejected": -1.3669785261154175,
      "logps/chosen": -1.6888374090194702,
      "logps/rejected": -2.053189992904663,
      "loss": 1.8231,
      "nll_loss": 1.7720434665679932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16888374090194702,
      "rewards/margins": 0.03643525391817093,
      "rewards/rejected": -0.20531900227069855,
      "step": 233
    },
    {
      "epoch": 0.6461857093545047,
      "grad_norm": 0.23462118208408356,
      "learning_rate": 4.805071755550177e-06,
      "log_odds_chosen": 0.394045889377594,
      "log_odds_ratio": -0.5184462070465088,
      "logits/chosen": 0.2896556854248047,
      "logits/rejected": -1.3101093769073486,
      "logps/chosen": -1.713099479675293,
      "logps/rejected": -2.048048496246338,
      "loss": 1.8398,
      "nll_loss": 1.7879210710525513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1713099479675293,
      "rewards/margins": 0.03349488973617554,
      "rewards/rejected": -0.20480485260486603,
      "step": 234
    },
    {
      "epoch": 0.6489471867449085,
      "grad_norm": 0.23507662117481232,
      "learning_rate": 4.8019574215514705e-06,
      "log_odds_chosen": 0.3388752043247223,
      "log_odds_ratio": -0.5402787327766418,
      "logits/chosen": 0.20577961206436157,
      "logits/rejected": -0.8569263219833374,
      "logps/chosen": -1.6788839101791382,
      "logps/rejected": -1.9616367816925049,
      "loss": 1.8351,
      "nll_loss": 1.7810907363891602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16788838803768158,
      "rewards/margins": 0.028275297954678535,
      "rewards/rejected": -0.19616368412971497,
      "step": 235
    },
    {
      "epoch": 0.6517086641353124,
      "grad_norm": 0.2070939689874649,
      "learning_rate": 4.7988194313786275e-06,
      "log_odds_chosen": 0.37065887451171875,
      "log_odds_ratio": -0.5329977869987488,
      "logits/chosen": 0.22679734230041504,
      "logits/rejected": -0.9689663052558899,
      "logps/chosen": -1.629687786102295,
      "logps/rejected": -1.9382522106170654,
      "loss": 1.7528,
      "nll_loss": 1.6994796991348267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16296879947185516,
      "rewards/margins": 0.03085644729435444,
      "rewards/rejected": -0.19382524490356445,
      "step": 236
    },
    {
      "epoch": 0.6544701415257163,
      "grad_norm": 0.2251017838716507,
      "learning_rate": 4.795657817279349e-06,
      "log_odds_chosen": 0.3826752305030823,
      "log_odds_ratio": -0.5331037044525146,
      "logits/chosen": 0.143580362200737,
      "logits/rejected": -1.0959559679031372,
      "logps/chosen": -1.568124532699585,
      "logps/rejected": -1.8851323127746582,
      "loss": 1.7164,
      "nll_loss": 1.663122296333313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15681245923042297,
      "rewards/margins": 0.03170077130198479,
      "rewards/rejected": -0.18851323425769806,
      "step": 237
    },
    {
      "epoch": 0.6572316189161201,
      "grad_norm": 0.24112077057361603,
      "learning_rate": 4.7924726117441135e-06,
      "log_odds_chosen": 0.45494502782821655,
      "log_odds_ratio": -0.5097072124481201,
      "logits/chosen": 0.12531203031539917,
      "logits/rejected": -1.1530197858810425,
      "logps/chosen": -1.6881966590881348,
      "logps/rejected": -2.074317216873169,
      "loss": 1.8279,
      "nll_loss": 1.7769427299499512,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16881968080997467,
      "rewards/margins": 0.03861205279827118,
      "rewards/rejected": -0.20743173360824585,
      "step": 238
    },
    {
      "epoch": 0.659993096306524,
      "grad_norm": 0.23394882678985596,
      "learning_rate": 4.789263847505835e-06,
      "log_odds_chosen": 0.48239994049072266,
      "log_odds_ratio": -0.484576016664505,
      "logits/chosen": 0.19327585399150848,
      "logits/rejected": -1.0120890140533447,
      "logps/chosen": -1.673211693763733,
      "logps/rejected": -2.078700542449951,
      "loss": 1.8042,
      "nll_loss": 1.7557491064071655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16732117533683777,
      "rewards/margins": 0.04054888337850571,
      "rewards/rejected": -0.20787005126476288,
      "step": 239
    },
    {
      "epoch": 0.6627545736969278,
      "grad_norm": 0.24529801309108734,
      "learning_rate": 4.786031557539532e-06,
      "log_odds_chosen": 0.6541503667831421,
      "log_odds_ratio": -0.42781609296798706,
      "logits/chosen": 0.1258109211921692,
      "logits/rejected": -1.3018333911895752,
      "logps/chosen": -1.6192426681518555,
      "logps/rejected": -2.17771577835083,
      "loss": 1.7489,
      "nll_loss": 1.7061513662338257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16192427277565002,
      "rewards/margins": 0.055847302079200745,
      "rewards/rejected": -0.21777155995368958,
      "step": 240
    },
    {
      "epoch": 0.6655160510873317,
      "grad_norm": 0.21988217532634735,
      "learning_rate": 4.782775775061983e-06,
      "log_odds_chosen": 0.4216863214969635,
      "log_odds_ratio": -0.5078084468841553,
      "logits/chosen": 0.22119548916816711,
      "logits/rejected": -1.0060391426086426,
      "logps/chosen": -1.6245155334472656,
      "logps/rejected": -1.9757689237594604,
      "loss": 1.7524,
      "nll_loss": 1.7016619443893433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16245155036449432,
      "rewards/margins": 0.03512535244226456,
      "rewards/rejected": -0.19757691025733948,
      "step": 241
    },
    {
      "epoch": 0.6682775284777356,
      "grad_norm": 0.20372723042964935,
      "learning_rate": 4.779496533531393e-06,
      "log_odds_chosen": 0.5198055505752563,
      "log_odds_ratio": -0.4691554009914398,
      "logits/chosen": 0.20225608348846436,
      "logits/rejected": -1.1654834747314453,
      "logps/chosen": -1.5811524391174316,
      "logps/rejected": -2.01274037361145,
      "loss": 1.7073,
      "nll_loss": 1.6603847742080688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15811526775360107,
      "rewards/margins": 0.043158773332834244,
      "rewards/rejected": -0.20127403736114502,
      "step": 242
    },
    {
      "epoch": 0.6710390058681395,
      "grad_norm": 0.22578665614128113,
      "learning_rate": 4.7761938666470405e-06,
      "log_odds_chosen": 0.4481002688407898,
      "log_odds_ratio": -0.5030589699745178,
      "logits/chosen": 0.21000558137893677,
      "logits/rejected": -1.1474783420562744,
      "logps/chosen": -1.6968859434127808,
      "logps/rejected": -2.079129695892334,
      "loss": 1.8331,
      "nll_loss": 1.7828097343444824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16968859732151031,
      "rewards/margins": 0.03822438418865204,
      "rewards/rejected": -0.20791295170783997,
      "step": 243
    },
    {
      "epoch": 0.6738004832585434,
      "grad_norm": 0.21789832413196564,
      "learning_rate": 4.7728678083489375e-06,
      "log_odds_chosen": 0.3310457170009613,
      "log_odds_ratio": -0.5467555522918701,
      "logits/chosen": 0.21008087694644928,
      "logits/rejected": -1.071950912475586,
      "logps/chosen": -1.6163996458053589,
      "logps/rejected": -1.8934293985366821,
      "loss": 1.7531,
      "nll_loss": 1.6983906030654907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1616399735212326,
      "rewards/margins": 0.027702966704964638,
      "rewards/rejected": -0.1893429309129715,
      "step": 244
    },
    {
      "epoch": 0.6765619606489471,
      "grad_norm": 0.21386297047138214,
      "learning_rate": 4.7695183928174804e-06,
      "log_odds_chosen": 0.4576282799243927,
      "log_odds_ratio": -0.4972766637802124,
      "logits/chosen": 0.11432070285081863,
      "logits/rejected": -1.215945839881897,
      "logps/chosen": -1.6433749198913574,
      "logps/rejected": -2.0280091762542725,
      "loss": 1.7829,
      "nll_loss": 1.7331418991088867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16433750092983246,
      "rewards/margins": 0.03846340626478195,
      "rewards/rejected": -0.2028008997440338,
      "step": 245
    },
    {
      "epoch": 0.679323438039351,
      "grad_norm": 0.23491686582565308,
      "learning_rate": 4.766145654473096e-06,
      "log_odds_chosen": 0.37732306122779846,
      "log_odds_ratio": -0.5303936004638672,
      "logits/chosen": 0.18962648510932922,
      "logits/rejected": -1.3029686212539673,
      "logps/chosen": -1.716407299041748,
      "logps/rejected": -2.0386903285980225,
      "loss": 1.8439,
      "nll_loss": 1.790850043296814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17164072394371033,
      "rewards/margins": 0.03222829848527908,
      "rewards/rejected": -0.2038690447807312,
      "step": 246
    },
    {
      "epoch": 0.6820849154297549,
      "grad_norm": 0.2103213518857956,
      "learning_rate": 4.762749627975888e-06,
      "log_odds_chosen": 0.40977245569229126,
      "log_odds_ratio": -0.5140390396118164,
      "logits/chosen": 0.0982648953795433,
      "logits/rejected": -1.1915839910507202,
      "logps/chosen": -1.557995319366455,
      "logps/rejected": -1.8929803371429443,
      "loss": 1.6972,
      "nll_loss": 1.6458226442337036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557995229959488,
      "rewards/margins": 0.033498503267765045,
      "rewards/rejected": -0.18929803371429443,
      "step": 247
    },
    {
      "epoch": 0.6848463928201588,
      "grad_norm": 0.2225971668958664,
      "learning_rate": 4.7593303482252835e-06,
      "log_odds_chosen": 0.36240053176879883,
      "log_odds_ratio": -0.5350769758224487,
      "logits/chosen": 0.16154634952545166,
      "logits/rejected": -0.829785943031311,
      "logps/chosen": -1.6611778736114502,
      "logps/rejected": -1.9659464359283447,
      "loss": 1.7928,
      "nll_loss": 1.7392576932907104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16611778736114502,
      "rewards/margins": 0.03047686442732811,
      "rewards/rejected": -0.19659464061260223,
      "step": 248
    },
    {
      "epoch": 0.6876078702105627,
      "grad_norm": 0.19730301201343536,
      "learning_rate": 4.755887850359673e-06,
      "log_odds_chosen": 0.598614513874054,
      "log_odds_ratio": -0.4566551446914673,
      "logits/chosen": 0.10718496143817902,
      "logits/rejected": -1.343011498451233,
      "logps/chosen": -1.5110077857971191,
      "logps/rejected": -2.002279043197632,
      "loss": 1.6562,
      "nll_loss": 1.6105355024337769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1511007696390152,
      "rewards/margins": 0.04912712052464485,
      "rewards/rejected": -0.20022790133953094,
      "step": 249
    },
    {
      "epoch": 0.6903693476009665,
      "grad_norm": 0.2222234308719635,
      "learning_rate": 4.752422169756048e-06,
      "log_odds_chosen": 0.4699355959892273,
      "log_odds_ratio": -0.4920656085014343,
      "logits/chosen": 0.1497102826833725,
      "logits/rejected": -1.4104342460632324,
      "logps/chosen": -1.6844984292984009,
      "logps/rejected": -2.0840110778808594,
      "loss": 1.8276,
      "nll_loss": 1.7784277200698853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16844984889030457,
      "rewards/margins": 0.03995127975940704,
      "rewards/rejected": -0.20840111374855042,
      "step": 250
    },
    {
      "epoch": 0.6931308249913704,
      "grad_norm": 0.2100534588098526,
      "learning_rate": 4.748933342029639e-06,
      "log_odds_chosen": 0.5825514793395996,
      "log_odds_ratio": -0.4510309398174286,
      "logits/chosen": 0.1121891662478447,
      "logits/rejected": -1.3764441013336182,
      "logps/chosen": -1.560391902923584,
      "logps/rejected": -2.048152446746826,
      "loss": 1.7094,
      "nll_loss": 1.6642597913742065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15603917837142944,
      "rewards/margins": 0.0487760454416275,
      "rewards/rejected": -0.20481522381305695,
      "step": 251
    },
    {
      "epoch": 0.6958923023817742,
      "grad_norm": 0.20206160843372345,
      "learning_rate": 4.745421403033548e-06,
      "log_odds_chosen": 0.4050910174846649,
      "log_odds_ratio": -0.5125004649162292,
      "logits/chosen": 0.10816405713558197,
      "logits/rejected": -1.1761468648910522,
      "logps/chosen": -1.6354482173919678,
      "logps/rejected": -1.9715569019317627,
      "loss": 1.7615,
      "nll_loss": 1.7102546691894531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16354484856128693,
      "rewards/margins": 0.033610858023166656,
      "rewards/rejected": -0.1971556842327118,
      "step": 252
    },
    {
      "epoch": 0.6986537797721781,
      "grad_norm": 0.2380354106426239,
      "learning_rate": 4.741886388858384e-06,
      "log_odds_chosen": 0.3426484167575836,
      "log_odds_ratio": -0.543499231338501,
      "logits/chosen": 0.17392773926258087,
      "logits/rejected": -1.2038366794586182,
      "logps/chosen": -1.6468617916107178,
      "logps/rejected": -1.928429365158081,
      "loss": 1.79,
      "nll_loss": 1.7356586456298828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1646861881017685,
      "rewards/margins": 0.02815674990415573,
      "rewards/rejected": -0.19284293055534363,
      "step": 253
    },
    {
      "epoch": 0.701415257162582,
      "grad_norm": 0.2207070291042328,
      "learning_rate": 4.738328335831883e-06,
      "log_odds_chosen": 0.3862311542034149,
      "log_odds_ratio": -0.5232746005058289,
      "logits/chosen": 0.13721241056919098,
      "logits/rejected": -1.4115301370620728,
      "logps/chosen": -1.612336277961731,
      "logps/rejected": -1.9344983100891113,
      "loss": 1.7501,
      "nll_loss": 1.6977391242980957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16123361885547638,
      "rewards/margins": 0.03221620246767998,
      "rewards/rejected": -0.19344982504844666,
      "step": 254
    },
    {
      "epoch": 0.7041767345529858,
      "grad_norm": 0.2061997801065445,
      "learning_rate": 4.734747280518549e-06,
      "log_odds_chosen": 0.47361427545547485,
      "log_odds_ratio": -0.48697221279144287,
      "logits/chosen": 0.03231241926550865,
      "logits/rejected": -1.5338258743286133,
      "logps/chosen": -1.6500543355941772,
      "logps/rejected": -2.0473241806030273,
      "loss": 1.7861,
      "nll_loss": 1.737368106842041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16500544548034668,
      "rewards/margins": 0.039726972579956055,
      "rewards/rejected": -0.20473241806030273,
      "step": 255
    },
    {
      "epoch": 0.7069382119433897,
      "grad_norm": 0.20688936114311218,
      "learning_rate": 4.7311432597192655e-06,
      "log_odds_chosen": 0.39435428380966187,
      "log_odds_ratio": -0.5248243808746338,
      "logits/chosen": 0.03083261288702488,
      "logits/rejected": -1.4570684432983398,
      "logps/chosen": -1.648856520652771,
      "logps/rejected": -1.9787046909332275,
      "loss": 1.7765,
      "nll_loss": 1.7240355014801025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16488566994667053,
      "rewards/margins": 0.03298482671380043,
      "rewards/rejected": -0.19787049293518066,
      "step": 256
    },
    {
      "epoch": 0.7096996893337936,
      "grad_norm": 0.22945424914360046,
      "learning_rate": 4.72751631047092e-06,
      "log_odds_chosen": 0.5417366027832031,
      "log_odds_ratio": -0.46491706371307373,
      "logits/chosen": -0.006479084491729736,
      "logits/rejected": -1.1154435873031616,
      "logps/chosen": -1.6812236309051514,
      "logps/rejected": -2.1395444869995117,
      "loss": 1.8262,
      "nll_loss": 1.7797247171401978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16812236607074738,
      "rewards/margins": 0.0458320677280426,
      "rewards/rejected": -0.21395443379878998,
      "step": 257
    },
    {
      "epoch": 0.7124611667241975,
      "grad_norm": 0.22095176577568054,
      "learning_rate": 4.72386647004603e-06,
      "log_odds_chosen": 0.4106917977333069,
      "log_odds_ratio": -0.5133163332939148,
      "logits/chosen": 0.1232781782746315,
      "logits/rejected": -1.1046854257583618,
      "logps/chosen": -1.6649987697601318,
      "logps/rejected": -2.0084848403930664,
      "loss": 1.811,
      "nll_loss": 1.7596914768218994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16649989783763885,
      "rewards/margins": 0.03434859216213226,
      "rewards/rejected": -0.20084848999977112,
      "step": 258
    },
    {
      "epoch": 0.7152226441146013,
      "grad_norm": 0.22131314873695374,
      "learning_rate": 4.720193775952352e-06,
      "log_odds_chosen": 0.2770199477672577,
      "log_odds_ratio": -0.5675607919692993,
      "logits/chosen": 0.09717811644077301,
      "logits/rejected": -1.1877737045288086,
      "logps/chosen": -1.6597505807876587,
      "logps/rejected": -1.889330267906189,
      "loss": 1.7861,
      "nll_loss": 1.729378581047058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16597506403923035,
      "rewards/margins": 0.022957956418395042,
      "rewards/rejected": -0.18893301486968994,
      "step": 259
    },
    {
      "epoch": 0.7179841215050051,
      "grad_norm": 0.2108003944158554,
      "learning_rate": 4.716498265932501e-06,
      "log_odds_chosen": 0.5200653076171875,
      "log_odds_ratio": -0.47012218832969666,
      "logits/chosen": -0.01136242039501667,
      "logits/rejected": -1.1188238859176636,
      "logps/chosen": -1.4902641773223877,
      "logps/rejected": -1.915147066116333,
      "loss": 1.6221,
      "nll_loss": 1.575091004371643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14902642369270325,
      "rewards/margins": 0.04248826950788498,
      "rewards/rejected": -0.19151470065116882,
      "step": 260
    },
    {
      "epoch": 0.720745598895409,
      "grad_norm": 0.22381585836410522,
      "learning_rate": 4.712779977963559e-06,
      "log_odds_chosen": 0.39622795581817627,
      "log_odds_ratio": -0.5201148390769958,
      "logits/chosen": 0.10159610211849213,
      "logits/rejected": -0.859241247177124,
      "logps/chosen": -1.5900012254714966,
      "logps/rejected": -1.917615294456482,
      "loss": 1.7383,
      "nll_loss": 1.6862510442733765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15900012850761414,
      "rewards/margins": 0.03276140242815018,
      "rewards/rejected": -0.1917615383863449,
      "step": 261
    },
    {
      "epoch": 0.7235070762858129,
      "grad_norm": 0.2046324461698532,
      "learning_rate": 4.7090389502566884e-06,
      "log_odds_chosen": 0.5120800733566284,
      "log_odds_ratio": -0.47404342889785767,
      "logits/chosen": 0.017112823203206062,
      "logits/rejected": -1.1166574954986572,
      "logps/chosen": -1.6049623489379883,
      "logps/rejected": -2.0321714878082275,
      "loss": 1.7168,
      "nll_loss": 1.6693758964538574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16049623489379883,
      "rewards/margins": 0.042720913887023926,
      "rewards/rejected": -0.20321716368198395,
      "step": 262
    },
    {
      "epoch": 0.7262685536762168,
      "grad_norm": 0.20920297503471375,
      "learning_rate": 4.705275221256738e-06,
      "log_odds_chosen": 0.42567548155784607,
      "log_odds_ratio": -0.5072777271270752,
      "logits/chosen": 0.11223579943180084,
      "logits/rejected": -1.1794720888137817,
      "logps/chosen": -1.6436134576797485,
      "logps/rejected": -2.0019237995147705,
      "loss": 1.7837,
      "nll_loss": 1.7329978942871094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16436134278774261,
      "rewards/margins": 0.035831037908792496,
      "rewards/rejected": -0.2001923769712448,
      "step": 263
    },
    {
      "epoch": 0.7290300310666207,
      "grad_norm": 0.215216726064682,
      "learning_rate": 4.701488829641845e-06,
      "log_odds_chosen": 0.3972318172454834,
      "log_odds_ratio": -0.5163053870201111,
      "logits/chosen": 0.07385722547769547,
      "logits/rejected": -1.1585350036621094,
      "logps/chosen": -1.5981651544570923,
      "logps/rejected": -1.9258975982666016,
      "loss": 1.7361,
      "nll_loss": 1.6844836473464966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15981650352478027,
      "rewards/margins": 0.032773248851299286,
      "rewards/rejected": -0.19258975982666016,
      "step": 264
    },
    {
      "epoch": 0.7317915084570245,
      "grad_norm": 0.20215153694152832,
      "learning_rate": 4.697679814323044e-06,
      "log_odds_chosen": 0.373761922121048,
      "log_odds_ratio": -0.5314816832542419,
      "logits/chosen": 0.0431194081902504,
      "logits/rejected": -1.3062759637832642,
      "logps/chosen": -1.6237438917160034,
      "logps/rejected": -1.9311178922653198,
      "loss": 1.746,
      "nll_loss": 1.6928824186325073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1623743772506714,
      "rewards/margins": 0.030737407505512238,
      "rewards/rejected": -0.19311177730560303,
      "step": 265
    },
    {
      "epoch": 0.7345529858474283,
      "grad_norm": 0.2043098360300064,
      "learning_rate": 4.693848214443858e-06,
      "log_odds_chosen": 0.3958456516265869,
      "log_odds_ratio": -0.5203114151954651,
      "logits/chosen": 0.016560683026909828,
      "logits/rejected": -1.4892038106918335,
      "logps/chosen": -1.690280795097351,
      "logps/rejected": -2.024770736694336,
      "loss": 1.8103,
      "nll_loss": 1.7582213878631592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16902808845043182,
      "rewards/margins": 0.03344898298382759,
      "rewards/rejected": -0.2024770826101303,
      "step": 266
    },
    {
      "epoch": 0.7373144632378322,
      "grad_norm": 0.2200057953596115,
      "learning_rate": 4.689994069379905e-06,
      "log_odds_chosen": 0.6603919863700867,
      "log_odds_ratio": -0.4194304049015045,
      "logits/chosen": 0.0796060711145401,
      "logits/rejected": -1.6784858703613281,
      "logps/chosen": -1.5854812860488892,
      "logps/rejected": -2.141174554824829,
      "loss": 1.7061,
      "nll_loss": 1.6641736030578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15854813158512115,
      "rewards/margins": 0.05556933209300041,
      "rewards/rejected": -0.21411746740341187,
      "step": 267
    },
    {
      "epoch": 0.7400759406282361,
      "grad_norm": 0.207722008228302,
      "learning_rate": 4.686117418738489e-06,
      "log_odds_chosen": 0.4980109930038452,
      "log_odds_ratio": -0.477446049451828,
      "logits/chosen": 0.0639527440071106,
      "logits/rejected": -1.2904075384140015,
      "logps/chosen": -1.632141351699829,
      "logps/rejected": -2.049818515777588,
      "loss": 1.7618,
      "nll_loss": 1.7140535116195679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16321413218975067,
      "rewards/margins": 0.041767701506614685,
      "rewards/rejected": -0.20498183369636536,
      "step": 268
    },
    {
      "epoch": 0.74283741801864,
      "grad_norm": 0.20242716372013092,
      "learning_rate": 4.6822183023581945e-06,
      "log_odds_chosen": 0.42001479864120483,
      "log_odds_ratio": -0.517326831817627,
      "logits/chosen": 0.07094614952802658,
      "logits/rejected": -1.5565531253814697,
      "logps/chosen": -1.6155306100845337,
      "logps/rejected": -1.9663753509521484,
      "loss": 1.7551,
      "nll_loss": 1.7033692598342896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1615530550479889,
      "rewards/margins": 0.035084471106529236,
      "rewards/rejected": -0.19663754105567932,
      "step": 269
    },
    {
      "epoch": 0.7455988954090439,
      "grad_norm": 0.19576282799243927,
      "learning_rate": 4.678296760308474e-06,
      "log_odds_chosen": 0.34514501690864563,
      "log_odds_ratio": -0.5403321385383606,
      "logits/chosen": -0.05599237233400345,
      "logits/rejected": -1.4408526420593262,
      "logps/chosen": -1.5795042514801025,
      "logps/rejected": -1.8630210161209106,
      "loss": 1.7087,
      "nll_loss": 1.6546752452850342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15795043110847473,
      "rewards/margins": 0.028351658955216408,
      "rewards/rejected": -0.1863020956516266,
      "step": 270
    },
    {
      "epoch": 0.7483603727994477,
      "grad_norm": 0.22211046516895294,
      "learning_rate": 4.674352832889239e-06,
      "log_odds_chosen": 0.5620113611221313,
      "log_odds_ratio": -0.4555080533027649,
      "logits/chosen": -0.005663935095071793,
      "logits/rejected": -1.2683230638504028,
      "logps/chosen": -1.6520458459854126,
      "logps/rejected": -2.1268177032470703,
      "loss": 1.7996,
      "nll_loss": 1.7540650367736816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16520458459854126,
      "rewards/margins": 0.04747716709971428,
      "rewards/rejected": -0.21268175542354584,
      "step": 271
    },
    {
      "epoch": 0.7511218501898516,
      "grad_norm": 0.1922963708639145,
      "learning_rate": 4.670386560630446e-06,
      "log_odds_chosen": 0.444943368434906,
      "log_odds_ratio": -0.5015792846679688,
      "logits/chosen": -0.06714704632759094,
      "logits/rejected": -1.379449725151062,
      "logps/chosen": -1.5659842491149902,
      "logps/rejected": -1.9318914413452148,
      "loss": 1.6989,
      "nll_loss": 1.6487019062042236,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15659843385219574,
      "rewards/margins": 0.036590706557035446,
      "rewards/rejected": -0.19318914413452148,
      "step": 272
    },
    {
      "epoch": 0.7538833275802554,
      "grad_norm": 0.19772395491600037,
      "learning_rate": 4.66639798429168e-06,
      "log_odds_chosen": 0.5909014940261841,
      "log_odds_ratio": -0.4440915584564209,
      "logits/chosen": -0.010252359323203564,
      "logits/rejected": -1.6117687225341797,
      "logps/chosen": -1.6227033138275146,
      "logps/rejected": -2.1234261989593506,
      "loss": 1.7368,
      "nll_loss": 1.69236421585083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16227032244205475,
      "rewards/margins": 0.05007229745388031,
      "rewards/rejected": -0.21234261989593506,
      "step": 273
    },
    {
      "epoch": 0.7566448049706593,
      "grad_norm": 0.21296410262584686,
      "learning_rate": 4.6623871448617345e-06,
      "log_odds_chosen": 0.37625253200531006,
      "log_odds_ratio": -0.5279226303100586,
      "logits/chosen": -0.11291900277137756,
      "logits/rejected": -1.474963665008545,
      "logps/chosen": -1.5688221454620361,
      "logps/rejected": -1.8770910501480103,
      "loss": 1.6973,
      "nll_loss": 1.6444581747055054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15688221156597137,
      "rewards/margins": 0.03082689456641674,
      "rewards/rejected": -0.18770912289619446,
      "step": 274
    },
    {
      "epoch": 0.7594062823610632,
      "grad_norm": 0.21504846215248108,
      "learning_rate": 4.6583540835581885e-06,
      "log_odds_chosen": 0.446832537651062,
      "log_odds_ratio": -0.5026010870933533,
      "logits/chosen": -0.0652085542678833,
      "logits/rejected": -1.2765593528747559,
      "logps/chosen": -1.5856618881225586,
      "logps/rejected": -1.9579052925109863,
      "loss": 1.7239,
      "nll_loss": 1.6736685037612915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1585661768913269,
      "rewards/margins": 0.037224359810352325,
      "rewards/rejected": -0.19579055905342102,
      "step": 275
    },
    {
      "epoch": 0.762167759751467,
      "grad_norm": 0.1951994001865387,
      "learning_rate": 4.654298841826988e-06,
      "log_odds_chosen": 0.38752269744873047,
      "log_odds_ratio": -0.5222585797309875,
      "logits/chosen": -0.09070023894309998,
      "logits/rejected": -1.2031896114349365,
      "logps/chosen": -1.5128428936004639,
      "logps/rejected": -1.8278968334197998,
      "loss": 1.6382,
      "nll_loss": 1.586016058921814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15128430724143982,
      "rewards/margins": 0.031505391001701355,
      "rewards/rejected": -0.18278968334197998,
      "step": 276
    },
    {
      "epoch": 0.7649292371418709,
      "grad_norm": 0.20356132090091705,
      "learning_rate": 4.6502214613420164e-06,
      "log_odds_chosen": 0.661637008190155,
      "log_odds_ratio": -0.4188510775566101,
      "logits/chosen": -0.07356397807598114,
      "logits/rejected": -1.270960807800293,
      "logps/chosen": -1.4853070974349976,
      "logps/rejected": -2.030834674835205,
      "loss": 1.6179,
      "nll_loss": 1.5760544538497925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14853070676326752,
      "rewards/margins": 0.054552774876356125,
      "rewards/rejected": -0.20308347046375275,
      "step": 277
    },
    {
      "epoch": 0.7676907145322748,
      "grad_norm": 0.21808800101280212,
      "learning_rate": 4.646121984004666e-06,
      "log_odds_chosen": 0.535017192363739,
      "log_odds_ratio": -0.4694536030292511,
      "logits/chosen": -0.058985427021980286,
      "logits/rejected": -1.204667329788208,
      "logps/chosen": -1.6129412651062012,
      "logps/rejected": -2.0625391006469727,
      "loss": 1.7558,
      "nll_loss": 1.7088611125946045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1612941473722458,
      "rewards/margins": 0.04495978727936745,
      "rewards/rejected": -0.20625391602516174,
      "step": 278
    },
    {
      "epoch": 0.7704521919226787,
      "grad_norm": 0.21462294459342957,
      "learning_rate": 4.642000451943409e-06,
      "log_odds_chosen": 0.4302963316440582,
      "log_odds_ratio": -0.5060604214668274,
      "logits/chosen": 0.040374599397182465,
      "logits/rejected": -1.187546968460083,
      "logps/chosen": -1.6709057092666626,
      "logps/rejected": -2.035818576812744,
      "loss": 1.7989,
      "nll_loss": 1.7482545375823975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16709057986736298,
      "rewards/margins": 0.03649128973484039,
      "rewards/rejected": -0.20358186960220337,
      "step": 279
    },
    {
      "epoch": 0.7732136693130826,
      "grad_norm": 0.19471004605293274,
      "learning_rate": 4.637856907513366e-06,
      "log_odds_chosen": 0.5729100704193115,
      "log_odds_ratio": -0.45996299386024475,
      "logits/chosen": -0.0875316932797432,
      "logits/rejected": -1.2949634790420532,
      "logps/chosen": -1.6255837678909302,
      "logps/rejected": -2.107527256011963,
      "loss": 1.7409,
      "nll_loss": 1.6949416399002075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1625583916902542,
      "rewards/margins": 0.04819435626268387,
      "rewards/rejected": -0.21075274050235748,
      "step": 280
    },
    {
      "epoch": 0.7759751467034863,
      "grad_norm": 0.20401322841644287,
      "learning_rate": 4.633691393295865e-06,
      "log_odds_chosen": 0.3522525429725647,
      "log_odds_ratio": -0.5362752676010132,
      "logits/chosen": -0.07532086223363876,
      "logits/rejected": -1.3928956985473633,
      "logps/chosen": -1.6393187046051025,
      "logps/rejected": -1.9348175525665283,
      "loss": 1.7628,
      "nll_loss": 1.7092012166976929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16393187642097473,
      "rewards/margins": 0.02954990416765213,
      "rewards/rejected": -0.19348177313804626,
      "step": 281
    },
    {
      "epoch": 0.7787366240938902,
      "grad_norm": 0.19418881833553314,
      "learning_rate": 4.629503952098011e-06,
      "log_odds_chosen": 0.6238572597503662,
      "log_odds_ratio": -0.43670332431793213,
      "logits/chosen": -0.08066678047180176,
      "logits/rejected": -1.6776320934295654,
      "logps/chosen": -1.5727202892303467,
      "logps/rejected": -2.097163200378418,
      "loss": 1.6952,
      "nll_loss": 1.6514896154403687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15727202594280243,
      "rewards/margins": 0.05244428664445877,
      "rewards/rejected": -0.2097163200378418,
      "step": 282
    },
    {
      "epoch": 0.7814981014842941,
      "grad_norm": 0.19461016356945038,
      "learning_rate": 4.6252946269522406e-06,
      "log_odds_chosen": 0.41456982493400574,
      "log_odds_ratio": -0.5249388813972473,
      "logits/chosen": -0.09783484041690826,
      "logits/rejected": -1.5897575616836548,
      "logps/chosen": -1.5939539670944214,
      "logps/rejected": -1.9444385766983032,
      "loss": 1.7211,
      "nll_loss": 1.6686402559280396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15939539670944214,
      "rewards/margins": 0.0350484773516655,
      "rewards/rejected": -0.19444386661052704,
      "step": 283
    },
    {
      "epoch": 0.784259578874698,
      "grad_norm": 0.2026386708021164,
      "learning_rate": 4.621063461115882e-06,
      "log_odds_chosen": 0.42722252011299133,
      "log_odds_ratio": -0.5157784223556519,
      "logits/chosen": -0.04370000213384628,
      "logits/rejected": -1.5634028911590576,
      "logps/chosen": -1.6642380952835083,
      "logps/rejected": -2.0263044834136963,
      "loss": 1.7849,
      "nll_loss": 1.733304738998413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16642381250858307,
      "rewards/margins": 0.03620663285255432,
      "rewards/rejected": -0.20263046026229858,
      "step": 284
    },
    {
      "epoch": 0.7870210562651019,
      "grad_norm": 0.20439012348651886,
      "learning_rate": 4.6168104980707105e-06,
      "log_odds_chosen": 0.4680078625679016,
      "log_odds_ratio": -0.4906473755836487,
      "logits/chosen": -0.09853056073188782,
      "logits/rejected": -1.5130950212478638,
      "logps/chosen": -1.5937169790267944,
      "logps/rejected": -1.9836535453796387,
      "loss": 1.7178,
      "nll_loss": 1.668696641921997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15937167406082153,
      "rewards/margins": 0.038993678987026215,
      "rewards/rejected": -0.19836536049842834,
      "step": 285
    },
    {
      "epoch": 0.7897825336555057,
      "grad_norm": 0.1877906173467636,
      "learning_rate": 4.612535781522504e-06,
      "log_odds_chosen": 0.37254124879837036,
      "log_odds_ratio": -0.525785505771637,
      "logits/chosen": -0.08526084572076797,
      "logits/rejected": -1.4526267051696777,
      "logps/chosen": -1.5574032068252563,
      "logps/rejected": -1.863482117652893,
      "loss": 1.6777,
      "nll_loss": 1.6251548528671265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15574032068252563,
      "rewards/margins": 0.03060789778828621,
      "rewards/rejected": -0.18634822964668274,
      "step": 286
    },
    {
      "epoch": 0.7925440110459095,
      "grad_norm": 0.19580195844173431,
      "learning_rate": 4.6082393554005855e-06,
      "log_odds_chosen": 0.5562997460365295,
      "log_odds_ratio": -0.4619132876396179,
      "logits/chosen": 0.01294594258069992,
      "logits/rejected": -1.4304238557815552,
      "logps/chosen": -1.5266389846801758,
      "logps/rejected": -1.9919108152389526,
      "loss": 1.6616,
      "nll_loss": 1.6153795719146729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.152663916349411,
      "rewards/margins": 0.046527184545993805,
      "rewards/rejected": -0.19919107854366302,
      "step": 287
    },
    {
      "epoch": 0.7953054884363134,
      "grad_norm": 0.1980113834142685,
      "learning_rate": 4.6039212638573835e-06,
      "log_odds_chosen": 0.3346819281578064,
      "log_odds_ratio": -0.5443198680877686,
      "logits/chosen": -0.15275517106056213,
      "logits/rejected": -1.6041910648345947,
      "logps/chosen": -1.6962547302246094,
      "logps/rejected": -1.9770543575286865,
      "loss": 1.8067,
      "nll_loss": 1.752286672592163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16962547600269318,
      "rewards/margins": 0.028079960495233536,
      "rewards/rejected": -0.1977054327726364,
      "step": 288
    },
    {
      "epoch": 0.7980669658267173,
      "grad_norm": 0.2273865044116974,
      "learning_rate": 4.599581551267969e-06,
      "log_odds_chosen": 0.5466289520263672,
      "log_odds_ratio": -0.46321243047714233,
      "logits/chosen": 0.03993244469165802,
      "logits/rejected": -1.286005973815918,
      "logps/chosen": -1.6738882064819336,
      "logps/rejected": -2.1377532482147217,
      "loss": 1.8,
      "nll_loss": 1.7537211179733276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16738884150981903,
      "rewards/margins": 0.046386465430259705,
      "rewards/rejected": -0.21377530694007874,
      "step": 289
    },
    {
      "epoch": 0.8008284432171212,
      "grad_norm": 0.20244112610816956,
      "learning_rate": 4.5952202622296015e-06,
      "log_odds_chosen": 0.3539222180843353,
      "log_odds_ratio": -0.53514164686203,
      "logits/chosen": -0.11812002211809158,
      "logits/rejected": -1.4588350057601929,
      "logps/chosen": -1.5985196828842163,
      "logps/rejected": -1.8924399614334106,
      "loss": 1.7271,
      "nll_loss": 1.6736091375350952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15985198318958282,
      "rewards/margins": 0.029392031952738762,
      "rewards/rejected": -0.18924400210380554,
      "step": 290
    },
    {
      "epoch": 0.803589920607525,
      "grad_norm": 0.20306488871574402,
      "learning_rate": 4.590837441561277e-06,
      "log_odds_chosen": 0.4768354296684265,
      "log_odds_ratio": -0.49164149165153503,
      "logits/chosen": -0.09056994318962097,
      "logits/rejected": -1.4742063283920288,
      "logps/chosen": -1.6228840351104736,
      "logps/rejected": -2.022940158843994,
      "loss": 1.7341,
      "nll_loss": 1.6849125623703003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1622883826494217,
      "rewards/margins": 0.040005628019571304,
      "rewards/rejected": -0.2022940218448639,
      "step": 291
    },
    {
      "epoch": 0.8063513979979289,
      "grad_norm": 0.19205592572689056,
      "learning_rate": 4.586433134303257e-06,
      "log_odds_chosen": 0.5734292268753052,
      "log_odds_ratio": -0.44972163438796997,
      "logits/chosen": -0.07964983582496643,
      "logits/rejected": -1.3166429996490479,
      "logps/chosen": -1.4941394329071045,
      "logps/rejected": -1.9636625051498413,
      "loss": 1.6364,
      "nll_loss": 1.5913902521133423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14941394329071045,
      "rewards/margins": 0.04695230349898338,
      "rewards/rejected": -0.19636625051498413,
      "step": 292
    },
    {
      "epoch": 0.8091128753883328,
      "grad_norm": 0.20325274765491486,
      "learning_rate": 4.582007385716614e-06,
      "log_odds_chosen": 0.4024369418621063,
      "log_odds_ratio": -0.5175961852073669,
      "logits/chosen": -0.07684268802404404,
      "logits/rejected": -1.502617597579956,
      "logps/chosen": -1.6142741441726685,
      "logps/rejected": -1.9492229223251343,
      "loss": 1.7574,
      "nll_loss": 1.7056207656860352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16142742335796356,
      "rewards/margins": 0.033494893461465836,
      "rewards/rejected": -0.1949223130941391,
      "step": 293
    },
    {
      "epoch": 0.8118743527787367,
      "grad_norm": 0.20083336532115936,
      "learning_rate": 4.57756024128276e-06,
      "log_odds_chosen": 0.5593162178993225,
      "log_odds_ratio": -0.45524081587791443,
      "logits/chosen": -0.17213015258312225,
      "logits/rejected": -1.3977055549621582,
      "logps/chosen": -1.5258231163024902,
      "logps/rejected": -1.9875348806381226,
      "loss": 1.6559,
      "nll_loss": 1.610384225845337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1525823026895523,
      "rewards/margins": 0.04617120325565338,
      "rewards/rejected": -0.1987534910440445,
      "step": 294
    },
    {
      "epoch": 0.8146358301691405,
      "grad_norm": 0.19001390039920807,
      "learning_rate": 4.573091746702988e-06,
      "log_odds_chosen": 0.5878125429153442,
      "log_odds_ratio": -0.44817692041397095,
      "logits/chosen": -0.16976626217365265,
      "logits/rejected": -1.3345102071762085,
      "logps/chosen": -1.5175349712371826,
      "logps/rejected": -2.003750801086426,
      "loss": 1.6573,
      "nll_loss": 1.612461805343628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517535001039505,
      "rewards/margins": 0.048621561378240585,
      "rewards/rejected": -0.20037508010864258,
      "step": 295
    },
    {
      "epoch": 0.8173973075595443,
      "grad_norm": 0.18019473552703857,
      "learning_rate": 4.5686019478979915e-06,
      "log_odds_chosen": 0.6261818408966064,
      "log_odds_ratio": -0.43093276023864746,
      "logits/chosen": -0.045413050800561905,
      "logits/rejected": -1.8382006883621216,
      "logps/chosen": -1.5509614944458008,
      "logps/rejected": -2.0732412338256836,
      "loss": 1.6609,
      "nll_loss": 1.617802619934082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1550961434841156,
      "rewards/margins": 0.05222797393798828,
      "rewards/rejected": -0.20732411742210388,
      "step": 296
    },
    {
      "epoch": 0.8201587849499482,
      "grad_norm": 0.20908498764038086,
      "learning_rate": 4.564090891007401e-06,
      "log_odds_chosen": 0.5261347889900208,
      "log_odds_ratio": -0.4701959192752838,
      "logits/chosen": -0.1871597170829773,
      "logits/rejected": -1.5989326238632202,
      "logps/chosen": -1.648465633392334,
      "logps/rejected": -2.0937981605529785,
      "loss": 1.7682,
      "nll_loss": 1.7211995124816895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16484656929969788,
      "rewards/margins": 0.04453325271606445,
      "rewards/rejected": -0.20937982201576233,
      "step": 297
    },
    {
      "epoch": 0.8229202623403521,
      "grad_norm": 0.188289076089859,
      "learning_rate": 4.559558622389304e-06,
      "log_odds_chosen": 0.5777133703231812,
      "log_odds_ratio": -0.4563387632369995,
      "logits/chosen": -0.13260145485401154,
      "logits/rejected": -1.7002828121185303,
      "logps/chosen": -1.5394691228866577,
      "logps/rejected": -2.0138742923736572,
      "loss": 1.6561,
      "nll_loss": 1.6105040311813354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1539469212293625,
      "rewards/margins": 0.047440510243177414,
      "rewards/rejected": -0.201387420296669,
      "step": 298
    },
    {
      "epoch": 0.825681739730756,
      "grad_norm": 0.19627498090267181,
      "learning_rate": 4.555005188619776e-06,
      "log_odds_chosen": 0.5525295734405518,
      "log_odds_ratio": -0.4643310308456421,
      "logits/chosen": -0.20222769677639008,
      "logits/rejected": -1.6397647857666016,
      "logps/chosen": -1.5733509063720703,
      "logps/rejected": -2.0376038551330566,
      "loss": 1.6892,
      "nll_loss": 1.642791748046875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573350876569748,
      "rewards/margins": 0.04642530530691147,
      "rewards/rejected": -0.20376040041446686,
      "step": 299
    },
    {
      "epoch": 0.8284432171211599,
      "grad_norm": 0.2084610015153885,
      "learning_rate": 4.55043063649239e-06,
      "log_odds_chosen": 0.6468226313591003,
      "log_odds_ratio": -0.4257048964500427,
      "logits/chosen": -0.23111680150032043,
      "logits/rejected": -1.959672212600708,
      "logps/chosen": -1.5862557888031006,
      "logps/rejected": -2.1299490928649902,
      "loss": 1.7007,
      "nll_loss": 1.6581330299377441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15862558782100677,
      "rewards/margins": 0.054369326680898666,
      "rewards/rejected": -0.21299490332603455,
      "step": 300
    },
    {
      "epoch": 0.8312046945115636,
      "grad_norm": 0.18870840966701508,
      "learning_rate": 4.54583501301775e-06,
      "log_odds_chosen": 0.5869650840759277,
      "log_odds_ratio": -0.45076417922973633,
      "logits/chosen": -0.1821564882993698,
      "logits/rejected": -1.9625955820083618,
      "logps/chosen": -1.6092771291732788,
      "logps/rejected": -2.104759693145752,
      "loss": 1.7134,
      "nll_loss": 1.668313980102539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16092771291732788,
      "rewards/margins": 0.04954826086759567,
      "rewards/rejected": -0.21047596633434296,
      "step": 301
    },
    {
      "epoch": 0.8339661719019675,
      "grad_norm": 0.1871059238910675,
      "learning_rate": 4.541218365422997e-06,
      "log_odds_chosen": 0.6819782257080078,
      "log_odds_ratio": -0.4187195301055908,
      "logits/chosen": -0.2113940417766571,
      "logits/rejected": -1.5733145475387573,
      "logps/chosen": -1.4704827070236206,
      "logps/rejected": -2.02266001701355,
      "loss": 1.5975,
      "nll_loss": 1.5556209087371826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14704826474189758,
      "rewards/margins": 0.05521773174405098,
      "rewards/rejected": -0.20226599276065826,
      "step": 302
    },
    {
      "epoch": 0.8367276492923714,
      "grad_norm": 0.1986740678548813,
      "learning_rate": 4.536580741151328e-06,
      "log_odds_chosen": 0.427354633808136,
      "log_odds_ratio": -0.5044746398925781,
      "logits/chosen": -0.17407816648483276,
      "logits/rejected": -1.2958568334579468,
      "logps/chosen": -1.6126006841659546,
      "logps/rejected": -1.9662362337112427,
      "loss": 1.7285,
      "nll_loss": 1.678093433380127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16126006841659546,
      "rewards/margins": 0.03536355867981911,
      "rewards/rejected": -0.19662362337112427,
      "step": 303
    },
    {
      "epoch": 0.8394891266827753,
      "grad_norm": 0.20910044014453888,
      "learning_rate": 4.531922187861507e-06,
      "log_odds_chosen": 0.6855639219284058,
      "log_odds_ratio": -0.41133368015289307,
      "logits/chosen": -0.17175668478012085,
      "logits/rejected": -1.4931915998458862,
      "logps/chosen": -1.491321086883545,
      "logps/rejected": -2.059537649154663,
      "loss": 1.631,
      "nll_loss": 1.589834451675415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1491321176290512,
      "rewards/margins": 0.056821659207344055,
      "rewards/rejected": -0.20595377683639526,
      "step": 304
    },
    {
      "epoch": 0.8422506040731792,
      "grad_norm": 0.2135782539844513,
      "learning_rate": 4.527242753427378e-06,
      "log_odds_chosen": 0.4499048590660095,
      "log_odds_ratio": -0.4998108744621277,
      "logits/chosen": -0.23703832924365997,
      "logits/rejected": -1.2717434167861938,
      "logps/chosen": -1.6932170391082764,
      "logps/rejected": -2.074625015258789,
      "loss": 1.8238,
      "nll_loss": 1.7738076448440552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1693217009305954,
      "rewards/margins": 0.038140811026096344,
      "rewards/rejected": -0.20746250450611115,
      "step": 305
    },
    {
      "epoch": 0.845012081463583,
      "grad_norm": 0.1993100643157959,
      "learning_rate": 4.522542485937369e-06,
      "log_odds_chosen": 0.6878387331962585,
      "log_odds_ratio": -0.4140303134918213,
      "logits/chosen": -0.17333604395389557,
      "logits/rejected": -1.6610162258148193,
      "logps/chosen": -1.5485466718673706,
      "logps/rejected": -2.122166395187378,
      "loss": 1.6836,
      "nll_loss": 1.6422399282455444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548546552658081,
      "rewards/margins": 0.057361967861652374,
      "rewards/rejected": -0.21221664547920227,
      "step": 306
    },
    {
      "epoch": 0.8477735588539869,
      "grad_norm": 0.20089736580848694,
      "learning_rate": 4.5178214336940015e-06,
      "log_odds_chosen": 0.5427862405776978,
      "log_odds_ratio": -0.4643491506576538,
      "logits/chosen": -0.1863488256931305,
      "logits/rejected": -1.4939963817596436,
      "logps/chosen": -1.556074857711792,
      "logps/rejected": -2.007810592651367,
      "loss": 1.6867,
      "nll_loss": 1.640239953994751,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15560749173164368,
      "rewards/margins": 0.04517355561256409,
      "rewards/rejected": -0.20078104734420776,
      "step": 307
    },
    {
      "epoch": 0.8505350362443908,
      "grad_norm": 0.19465354084968567,
      "learning_rate": 4.513079645213391e-06,
      "log_odds_chosen": 0.6006003618240356,
      "log_odds_ratio": -0.4448007047176361,
      "logits/chosen": -0.21229855716228485,
      "logits/rejected": -1.460700273513794,
      "logps/chosen": -1.4619970321655273,
      "logps/rejected": -1.951730728149414,
      "loss": 1.5808,
      "nll_loss": 1.5363428592681885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14619971811771393,
      "rewards/margins": 0.04897337406873703,
      "rewards/rejected": -0.19517306983470917,
      "step": 308
    },
    {
      "epoch": 0.8532965136347946,
      "grad_norm": 0.22921526432037354,
      "learning_rate": 4.508317169224752e-06,
      "log_odds_chosen": 0.28590530157089233,
      "log_odds_ratio": -0.5672011375427246,
      "logits/chosen": -0.3028235137462616,
      "logits/rejected": -1.3352372646331787,
      "logps/chosen": -1.5896085500717163,
      "logps/rejected": -1.8217551708221436,
      "loss": 1.7142,
      "nll_loss": 1.6574809551239014,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15896086394786835,
      "rewards/margins": 0.0232146717607975,
      "rewards/rejected": -0.18217553198337555,
      "step": 309
    },
    {
      "epoch": 0.8560579910251985,
      "grad_norm": 0.20834492146968842,
      "learning_rate": 4.5035340546698915e-06,
      "log_odds_chosen": 0.5302640795707703,
      "log_odds_ratio": -0.47237080335617065,
      "logits/chosen": -0.13226553797721863,
      "logits/rejected": -1.6689988374710083,
      "logps/chosen": -1.5815542936325073,
      "logps/rejected": -2.017597198486328,
      "loss": 1.7019,
      "nll_loss": 1.6546366214752197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1581554412841797,
      "rewards/margins": 0.0436042957007885,
      "rewards/rejected": -0.2017597258090973,
      "step": 310
    },
    {
      "epoch": 0.8588194684156023,
      "grad_norm": 0.2186000496149063,
      "learning_rate": 4.4987303507027155e-06,
      "log_odds_chosen": 0.3287478983402252,
      "log_odds_ratio": -0.549705982208252,
      "logits/chosen": -0.11287423223257065,
      "logits/rejected": -1.349088430404663,
      "logps/chosen": -1.6460450887680054,
      "logps/rejected": -1.9183804988861084,
      "loss": 1.7738,
      "nll_loss": 1.718807339668274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16460449993610382,
      "rewards/margins": 0.027233552187681198,
      "rewards/rejected": -0.19183805584907532,
      "step": 311
    },
    {
      "epoch": 0.8615809458060062,
      "grad_norm": 0.19687776267528534,
      "learning_rate": 4.493906106688712e-06,
      "log_odds_chosen": 0.6679433584213257,
      "log_odds_ratio": -0.41612738370895386,
      "logits/chosen": -0.1544291228055954,
      "logits/rejected": -1.6344720125198364,
      "logps/chosen": -1.5363831520080566,
      "logps/rejected": -2.0934810638427734,
      "loss": 1.6699,
      "nll_loss": 1.6282765865325928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1536383181810379,
      "rewards/margins": 0.05570977181196213,
      "rewards/rejected": -0.20934809744358063,
      "step": 312
    },
    {
      "epoch": 0.8643424231964101,
      "grad_norm": 0.18424390256404877,
      "learning_rate": 4.4890613722044526e-06,
      "log_odds_chosen": 0.5501468181610107,
      "log_odds_ratio": -0.4591679573059082,
      "logits/chosen": -0.21150004863739014,
      "logits/rejected": -1.5727708339691162,
      "logps/chosen": -1.529168963432312,
      "logps/rejected": -1.9859933853149414,
      "loss": 1.6331,
      "nll_loss": 1.587223768234253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15291690826416016,
      "rewards/margins": 0.04568243771791458,
      "rewards/rejected": -0.19859933853149414,
      "step": 313
    },
    {
      "epoch": 0.867103900586814,
      "grad_norm": 0.20299668610095978,
      "learning_rate": 4.484196197037082e-06,
      "log_odds_chosen": 0.5865733027458191,
      "log_odds_ratio": -0.4465975761413574,
      "logits/chosen": -0.13622766733169556,
      "logits/rejected": -1.5913082361221313,
      "logps/chosen": -1.5737249851226807,
      "logps/rejected": -2.0672624111175537,
      "loss": 1.691,
      "nll_loss": 1.6463013887405396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15737250447273254,
      "rewards/margins": 0.04935373738408089,
      "rewards/rejected": -0.20672622323036194,
      "step": 314
    },
    {
      "epoch": 0.8698653779772179,
      "grad_norm": 0.19080907106399536,
      "learning_rate": 4.4793106311838e-06,
      "log_odds_chosen": 0.556647002696991,
      "log_odds_ratio": -0.46323782205581665,
      "logits/chosen": -0.2553403377532959,
      "logits/rejected": -1.4489879608154297,
      "logps/chosen": -1.562751054763794,
      "logps/rejected": -2.0281355381011963,
      "loss": 1.6685,
      "nll_loss": 1.6221669912338257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15627512335777283,
      "rewards/margins": 0.04653845354914665,
      "rewards/rejected": -0.20281356573104858,
      "step": 315
    },
    {
      "epoch": 0.8726268553676216,
      "grad_norm": 0.21013295650482178,
      "learning_rate": 4.474404724851356e-06,
      "log_odds_chosen": 0.5066735148429871,
      "log_odds_ratio": -0.48284393548965454,
      "logits/chosen": -0.18822398781776428,
      "logits/rejected": -1.6378931999206543,
      "logps/chosen": -1.6176685094833374,
      "logps/rejected": -2.041299819946289,
      "loss": 1.7356,
      "nll_loss": 1.687273621559143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16176684200763702,
      "rewards/margins": 0.04236314073204994,
      "rewards/rejected": -0.20412999391555786,
      "step": 316
    },
    {
      "epoch": 0.8753883327580255,
      "grad_norm": 0.2111610472202301,
      "learning_rate": 4.469478528455529e-06,
      "log_odds_chosen": 0.4905741810798645,
      "log_odds_ratio": -0.4830451011657715,
      "logits/chosen": -0.3353807330131531,
      "logits/rejected": -1.3652547597885132,
      "logps/chosen": -1.6042860746383667,
      "logps/rejected": -2.011214017868042,
      "loss": 1.7255,
      "nll_loss": 1.677234411239624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16042861342430115,
      "rewards/margins": 0.04069279134273529,
      "rewards/rejected": -0.20112140476703644,
      "step": 317
    },
    {
      "epoch": 0.8781498101484294,
      "grad_norm": 0.20293498039245605,
      "learning_rate": 4.464532092620607e-06,
      "log_odds_chosen": 0.523048996925354,
      "log_odds_ratio": -0.47193020582199097,
      "logits/chosen": -0.18486103415489197,
      "logits/rejected": -1.4264463186264038,
      "logps/chosen": -1.638154149055481,
      "logps/rejected": -2.0816562175750732,
      "loss": 1.7478,
      "nll_loss": 1.7006094455718994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1638154238462448,
      "rewards/margins": 0.04435021057724953,
      "rewards/rejected": -0.20816563069820404,
      "step": 318
    },
    {
      "epoch": 0.8809112875388333,
      "grad_norm": 0.17995508015155792,
      "learning_rate": 4.4595654681788715e-06,
      "log_odds_chosen": 0.647110641002655,
      "log_odds_ratio": -0.428227037191391,
      "logits/chosen": -0.23671314120292664,
      "logits/rejected": -1.8144798278808594,
      "logps/chosen": -1.5134212970733643,
      "logps/rejected": -2.050638437271118,
      "loss": 1.6212,
      "nll_loss": 1.5783425569534302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15134215354919434,
      "rewards/margins": 0.053721703588962555,
      "rewards/rejected": -0.2050638496875763,
      "step": 319
    },
    {
      "epoch": 0.8836727649292372,
      "grad_norm": 0.20032061636447906,
      "learning_rate": 4.454578706170075e-06,
      "log_odds_chosen": 0.35773491859436035,
      "log_odds_ratio": -0.5383328795433044,
      "logits/chosen": -0.1582796573638916,
      "logits/rejected": -1.6286273002624512,
      "logps/chosen": -1.6196863651275635,
      "logps/rejected": -1.9159085750579834,
      "loss": 1.7371,
      "nll_loss": 1.6832914352416992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1619686335325241,
      "rewards/margins": 0.02962222881615162,
      "rewards/rejected": -0.19159086048603058,
      "step": 320
    },
    {
      "epoch": 0.886434242319641,
      "grad_norm": 0.2055967003107071,
      "learning_rate": 4.449571857840911e-06,
      "log_odds_chosen": 0.586254358291626,
      "log_odds_ratio": -0.45622390508651733,
      "logits/chosen": -0.09812657535076141,
      "logits/rejected": -1.682092547416687,
      "logps/chosen": -1.5706590414047241,
      "logps/rejected": -2.0691263675689697,
      "loss": 1.6908,
      "nll_loss": 1.6451488733291626,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15706591308116913,
      "rewards/margins": 0.04984673112630844,
      "rewards/rejected": -0.20691262185573578,
      "step": 321
    },
    {
      "epoch": 0.8891957197100449,
      "grad_norm": 0.2170560508966446,
      "learning_rate": 4.444544974644493e-06,
      "log_odds_chosen": 0.42108941078186035,
      "log_odds_ratio": -0.5122984647750854,
      "logits/chosen": -0.20832902193069458,
      "logits/rejected": -1.468292236328125,
      "logps/chosen": -1.5347344875335693,
      "logps/rejected": -1.8808050155639648,
      "loss": 1.6619,
      "nll_loss": 1.61066472530365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15347345173358917,
      "rewards/margins": 0.03460706025362015,
      "rewards/rejected": -0.18808050453662872,
      "step": 322
    },
    {
      "epoch": 0.8919571971004487,
      "grad_norm": 0.20523445308208466,
      "learning_rate": 4.4394981082398254e-06,
      "log_odds_chosen": 0.4536086320877075,
      "log_odds_ratio": -0.5041500926017761,
      "logits/chosen": -0.26630857586860657,
      "logits/rejected": -1.5968291759490967,
      "logps/chosen": -1.5988963842391968,
      "logps/rejected": -1.9788540601730347,
      "loss": 1.7272,
      "nll_loss": 1.676782250404358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15988965332508087,
      "rewards/margins": 0.037995755672454834,
      "rewards/rejected": -0.19788537919521332,
      "step": 323
    },
    {
      "epoch": 0.8947186744908526,
      "grad_norm": 0.21546316146850586,
      "learning_rate": 4.434431310491267e-06,
      "log_odds_chosen": 0.6247555017471313,
      "log_odds_ratio": -0.43717044591903687,
      "logits/chosen": -0.27398669719696045,
      "logits/rejected": -1.5846881866455078,
      "logps/chosen": -1.660336971282959,
      "logps/rejected": -2.190629243850708,
      "loss": 1.7647,
      "nll_loss": 1.7209898233413696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16603372991085052,
      "rewards/margins": 0.05302921682596207,
      "rewards/rejected": -0.2190629243850708,
      "step": 324
    },
    {
      "epoch": 0.8974801518812565,
      "grad_norm": 0.21981805562973022,
      "learning_rate": 4.429344633468005e-06,
      "log_odds_chosen": 0.5096076130867004,
      "log_odds_ratio": -0.4749549627304077,
      "logits/chosen": -0.22183284163475037,
      "logits/rejected": -1.6317569017410278,
      "logps/chosen": -1.528499722480774,
      "logps/rejected": -1.9499012231826782,
      "loss": 1.6449,
      "nll_loss": 1.597440242767334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1528499722480774,
      "rewards/margins": 0.0421401672065258,
      "rewards/rejected": -0.1949901133775711,
      "step": 325
    },
    {
      "epoch": 0.9002416292716603,
      "grad_norm": 0.23145584762096405,
      "learning_rate": 4.424238129443515e-06,
      "log_odds_chosen": 0.4823288321495056,
      "log_odds_ratio": -0.48343366384506226,
      "logits/chosen": -0.1781751811504364,
      "logits/rejected": -1.2557498216629028,
      "logps/chosen": -1.6358767747879028,
      "logps/rejected": -2.0398380756378174,
      "loss": 1.7597,
      "nll_loss": 1.7113655805587769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16358768939971924,
      "rewards/margins": 0.04039612039923668,
      "rewards/rejected": -0.20398379862308502,
      "step": 326
    },
    {
      "epoch": 0.9030031066620642,
      "grad_norm": 0.21986520290374756,
      "learning_rate": 4.4191118508950286e-06,
      "log_odds_chosen": 0.5763283371925354,
      "log_odds_ratio": -0.4508843421936035,
      "logits/chosen": -0.2733724117279053,
      "logits/rejected": -1.4346749782562256,
      "logps/chosen": -1.5722405910491943,
      "logps/rejected": -2.0537078380584717,
      "loss": 1.7093,
      "nll_loss": 1.6641618013381958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15722407400608063,
      "rewards/margins": 0.048146720975637436,
      "rewards/rejected": -0.20537079870700836,
      "step": 327
    },
    {
      "epoch": 0.9057645840524681,
      "grad_norm": 0.19916215538978577,
      "learning_rate": 4.413965850502987e-06,
      "log_odds_chosen": 0.6543524265289307,
      "log_odds_ratio": -0.42543232440948486,
      "logits/chosen": -0.28132179379463196,
      "logits/rejected": -1.6163063049316406,
      "logps/chosen": -1.4477849006652832,
      "logps/rejected": -1.9837119579315186,
      "loss": 1.5427,
      "nll_loss": 1.5002020597457886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14477849006652832,
      "rewards/margins": 0.05359271913766861,
      "rewards/rejected": -0.19837118685245514,
      "step": 328
    },
    {
      "epoch": 0.908526061442872,
      "grad_norm": 0.20786328613758087,
      "learning_rate": 4.408800181150509e-06,
      "log_odds_chosen": 0.7103330492973328,
      "log_odds_ratio": -0.405758798122406,
      "logits/chosen": -0.27468955516815186,
      "logits/rejected": -1.7239093780517578,
      "logps/chosen": -1.5232479572296143,
      "logps/rejected": -2.113443374633789,
      "loss": 1.6308,
      "nll_loss": 1.5901868343353271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15232481062412262,
      "rewards/margins": 0.0590195432305336,
      "rewards/rejected": -0.21134433150291443,
      "step": 329
    },
    {
      "epoch": 0.9112875388332758,
      "grad_norm": 0.21173468232154846,
      "learning_rate": 4.4036148959228365e-06,
      "log_odds_chosen": 0.5421361327171326,
      "log_odds_ratio": -0.45979243516921997,
      "logits/chosen": -0.19346265494823456,
      "logits/rejected": -1.3866770267486572,
      "logps/chosen": -1.5939050912857056,
      "logps/rejected": -2.0476269721984863,
      "loss": 1.7068,
      "nll_loss": 1.660808801651001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15939049422740936,
      "rewards/margins": 0.045372217893600464,
      "rewards/rejected": -0.20476271212100983,
      "step": 330
    },
    {
      "epoch": 0.9140490162236796,
      "grad_norm": 0.20240680873394012,
      "learning_rate": 4.3984100481068e-06,
      "log_odds_chosen": 0.5311475992202759,
      "log_odds_ratio": -0.47231799364089966,
      "logits/chosen": -0.10619790852069855,
      "logits/rejected": -1.8089945316314697,
      "logps/chosen": -1.528999924659729,
      "logps/rejected": -1.9681410789489746,
      "loss": 1.6383,
      "nll_loss": 1.5910669565200806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15289999544620514,
      "rewards/margins": 0.04391412436962128,
      "rewards/rejected": -0.19681411981582642,
      "step": 331
    },
    {
      "epoch": 0.9168104936140835,
      "grad_norm": 0.18672843277454376,
      "learning_rate": 4.3931856911902635e-06,
      "log_odds_chosen": 0.6374708414077759,
      "log_odds_ratio": -0.433654248714447,
      "logits/chosen": -0.31257641315460205,
      "logits/rejected": -1.766423225402832,
      "logps/chosen": -1.5535167455673218,
      "logps/rejected": -2.087543487548828,
      "loss": 1.6746,
      "nll_loss": 1.631203293800354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553516834974289,
      "rewards/margins": 0.05340268462896347,
      "rewards/rejected": -0.20875434577465057,
      "step": 332
    },
    {
      "epoch": 0.9195719710044874,
      "grad_norm": 0.20278650522232056,
      "learning_rate": 4.387941878861578e-06,
      "log_odds_chosen": 0.442794531583786,
      "log_odds_ratio": -0.5059284567832947,
      "logits/chosen": -0.2818780243396759,
      "logits/rejected": -1.41943359375,
      "logps/chosen": -1.475003957748413,
      "logps/rejected": -1.838566541671753,
      "loss": 1.5936,
      "nll_loss": 1.5429768562316895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1475003957748413,
      "rewards/margins": 0.03635626286268234,
      "rewards/rejected": -0.18385665118694305,
      "step": 333
    },
    {
      "epoch": 0.9223334483948913,
      "grad_norm": 0.21072961390018463,
      "learning_rate": 4.382678665009028e-06,
      "log_odds_chosen": 0.5503413677215576,
      "log_odds_ratio": -0.4611153304576874,
      "logits/chosen": -0.24195455014705658,
      "logits/rejected": -1.77567720413208,
      "logps/chosen": -1.6017379760742188,
      "logps/rejected": -2.0620176792144775,
      "loss": 1.7086,
      "nll_loss": 1.6624857187271118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16017380356788635,
      "rewards/margins": 0.04602799564599991,
      "rewards/rejected": -0.20620179176330566,
      "step": 334
    },
    {
      "epoch": 0.9250949257852952,
      "grad_norm": 0.21240665018558502,
      "learning_rate": 4.3773961037202784e-06,
      "log_odds_chosen": 0.5787122249603271,
      "log_odds_ratio": -0.4496590793132782,
      "logits/chosen": -0.24867500364780426,
      "logits/rejected": -1.7745471000671387,
      "logps/chosen": -1.6565507650375366,
      "logps/rejected": -2.1457858085632324,
      "loss": 1.7518,
      "nll_loss": 1.706835150718689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16565507650375366,
      "rewards/margins": 0.04892349988222122,
      "rewards/rejected": -0.21457859873771667,
      "step": 335
    },
    {
      "epoch": 0.927856403175699,
      "grad_norm": 0.20056602358818054,
      "learning_rate": 4.37209424928182e-06,
      "log_odds_chosen": 0.5505763292312622,
      "log_odds_ratio": -0.4596712589263916,
      "logits/chosen": -0.18031375110149384,
      "logits/rejected": -1.4376002550125122,
      "logps/chosen": -1.557979941368103,
      "logps/rejected": -2.017378091812134,
      "loss": 1.6716,
      "nll_loss": 1.6256715059280396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557980179786682,
      "rewards/margins": 0.04593981057405472,
      "rewards/rejected": -0.20173780620098114,
      "step": 336
    },
    {
      "epoch": 0.9306178805661028,
      "grad_norm": 0.18808571994304657,
      "learning_rate": 4.366773156178413e-06,
      "log_odds_chosen": 0.42912667989730835,
      "log_odds_ratio": -0.5055819153785706,
      "logits/chosen": -0.3135528564453125,
      "logits/rejected": -1.5518022775650024,
      "logps/chosen": -1.481281042098999,
      "logps/rejected": -1.826270341873169,
      "loss": 1.5991,
      "nll_loss": 1.5485769510269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14812810719013214,
      "rewards/margins": 0.0344989076256752,
      "rewards/rejected": -0.18262703716754913,
      "step": 337
    },
    {
      "epoch": 0.9333793579565067,
      "grad_norm": 0.21728971600532532,
      "learning_rate": 4.361432879092518e-06,
      "log_odds_chosen": 0.5635970234870911,
      "log_odds_ratio": -0.4642696976661682,
      "logits/chosen": -0.32608091831207275,
      "logits/rejected": -1.478355050086975,
      "logps/chosen": -1.5051325559616089,
      "logps/rejected": -1.974266529083252,
      "loss": 1.6273,
      "nll_loss": 1.5808420181274414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15051327645778656,
      "rewards/margins": 0.04691339656710625,
      "rewards/rejected": -0.1974266618490219,
      "step": 338
    },
    {
      "epoch": 0.9361408353469106,
      "grad_norm": 0.20923133194446564,
      "learning_rate": 4.356073472903747e-06,
      "log_odds_chosen": 0.5899335145950317,
      "log_odds_ratio": -0.44590240716934204,
      "logits/chosen": -0.17045272886753082,
      "logits/rejected": -1.5179771184921265,
      "logps/chosen": -1.4507163763046265,
      "logps/rejected": -1.9340299367904663,
      "loss": 1.5672,
      "nll_loss": 1.5226441621780396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14507164061069489,
      "rewards/margins": 0.048331368714571,
      "rewards/rejected": -0.1934029906988144,
      "step": 339
    },
    {
      "epoch": 0.9389023127373145,
      "grad_norm": 0.20438268780708313,
      "learning_rate": 4.350694992688289e-06,
      "log_odds_chosen": 0.6362269520759583,
      "log_odds_ratio": -0.43733319640159607,
      "logits/chosen": -0.14821594953536987,
      "logits/rejected": -1.5649621486663818,
      "logps/chosen": -1.4848688840866089,
      "logps/rejected": -2.009694814682007,
      "loss": 1.6031,
      "nll_loss": 1.5594022274017334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1484868973493576,
      "rewards/margins": 0.05248260498046875,
      "rewards/rejected": -0.20096951723098755,
      "step": 340
    },
    {
      "epoch": 0.9416637901277183,
      "grad_norm": 0.20621763169765472,
      "learning_rate": 4.345297493718352e-06,
      "log_odds_chosen": 0.547203540802002,
      "log_odds_ratio": -0.462637722492218,
      "logits/chosen": -0.31976550817489624,
      "logits/rejected": -1.318708062171936,
      "logps/chosen": -1.4836325645446777,
      "logps/rejected": -1.930004358291626,
      "loss": 1.6093,
      "nll_loss": 1.563034176826477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14836326241493225,
      "rewards/margins": 0.044637180864810944,
      "rewards/rejected": -0.1930004358291626,
      "step": 341
    },
    {
      "epoch": 0.9444252675181222,
      "grad_norm": 0.18609599769115448,
      "learning_rate": 4.339881031461588e-06,
      "log_odds_chosen": 0.47142109274864197,
      "log_odds_ratio": -0.48913687467575073,
      "logits/chosen": -0.26566094160079956,
      "logits/rejected": -1.550631046295166,
      "logps/chosen": -1.4443541765213013,
      "logps/rejected": -1.8148910999298096,
      "loss": 1.578,
      "nll_loss": 1.5290637016296387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14443542063236237,
      "rewards/margins": 0.03705369308590889,
      "rewards/rejected": -0.18148910999298096,
      "step": 342
    },
    {
      "epoch": 0.9471867449085261,
      "grad_norm": 0.1996508240699768,
      "learning_rate": 4.334445661580527e-06,
      "log_odds_chosen": 0.5693928599357605,
      "log_odds_ratio": -0.4520787000656128,
      "logits/chosen": -0.33670923113822937,
      "logits/rejected": -1.7492713928222656,
      "logps/chosen": -1.5564992427825928,
      "logps/rejected": -2.030904531478882,
      "loss": 1.6725,
      "nll_loss": 1.6272705793380737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15564994513988495,
      "rewards/margins": 0.04744052141904831,
      "rewards/rejected": -0.20309044420719147,
      "step": 343
    },
    {
      "epoch": 0.94994822229893,
      "grad_norm": 0.19215835630893707,
      "learning_rate": 4.328991439932003e-06,
      "log_odds_chosen": 0.6314505934715271,
      "log_odds_ratio": -0.4288046658039093,
      "logits/chosen": -0.29215654730796814,
      "logits/rejected": -1.5209310054779053,
      "logps/chosen": -1.486976981163025,
      "logps/rejected": -2.0075106620788574,
      "loss": 1.6116,
      "nll_loss": 1.568747878074646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14869770407676697,
      "rewards/margins": 0.05205334722995758,
      "rewards/rejected": -0.20075105130672455,
      "step": 344
    },
    {
      "epoch": 0.9527096996893338,
      "grad_norm": 0.20573773980140686,
      "learning_rate": 4.323518422566586e-06,
      "log_odds_chosen": 0.7072029113769531,
      "log_odds_ratio": -0.40426695346832275,
      "logits/chosen": -0.23872965574264526,
      "logits/rejected": -1.5439509153366089,
      "logps/chosen": -1.5577975511550903,
      "logps/rejected": -2.149085760116577,
      "loss": 1.6903,
      "nll_loss": 1.649876594543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15577976405620575,
      "rewards/margins": 0.059128809720277786,
      "rewards/rejected": -0.21490855515003204,
      "step": 345
    },
    {
      "epoch": 0.9554711770797376,
      "grad_norm": 0.1889713704586029,
      "learning_rate": 4.318026665727993e-06,
      "log_odds_chosen": 0.6957321166992188,
      "log_odds_ratio": -0.411748468875885,
      "logits/chosen": -0.3132311701774597,
      "logits/rejected": -1.7383476495742798,
      "logps/chosen": -1.4288955926895142,
      "logps/rejected": -1.9945143461227417,
      "loss": 1.5422,
      "nll_loss": 1.501028299331665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14288955926895142,
      "rewards/margins": 0.056561872363090515,
      "rewards/rejected": -0.19945143163204193,
      "step": 346
    },
    {
      "epoch": 0.9582326544701415,
      "grad_norm": 0.21370814740657806,
      "learning_rate": 4.3125162258525265e-06,
      "log_odds_chosen": 0.37931889295578003,
      "log_odds_ratio": -0.5239338874816895,
      "logits/chosen": -0.34714582562446594,
      "logits/rejected": -1.3666698932647705,
      "logps/chosen": -1.6313103437423706,
      "logps/rejected": -1.9475239515304565,
      "loss": 1.7528,
      "nll_loss": 1.7004497051239014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16313102841377258,
      "rewards/margins": 0.031621355563402176,
      "rewards/rejected": -0.19475241005420685,
      "step": 347
    },
    {
      "epoch": 0.9609941318605454,
      "grad_norm": 0.20202378928661346,
      "learning_rate": 4.3069871595684795e-06,
      "log_odds_chosen": 0.5231636762619019,
      "log_odds_ratio": -0.4718437194824219,
      "logits/chosen": -0.3397434949874878,
      "logits/rejected": -1.5151662826538086,
      "logps/chosen": -1.4959080219268799,
      "logps/rejected": -1.9243054389953613,
      "loss": 1.6233,
      "nll_loss": 1.5760971307754517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14959080517292023,
      "rewards/margins": 0.04283975064754486,
      "rewards/rejected": -0.1924305558204651,
      "step": 348
    },
    {
      "epoch": 0.9637556092509493,
      "grad_norm": 0.1996474266052246,
      "learning_rate": 4.3014395236955635e-06,
      "log_odds_chosen": 0.6398702263832092,
      "log_odds_ratio": -0.4294753670692444,
      "logits/chosen": -0.2723667621612549,
      "logits/rejected": -1.6896021366119385,
      "logps/chosen": -1.4915456771850586,
      "logps/rejected": -2.0213863849639893,
      "loss": 1.6075,
      "nll_loss": 1.5645424127578735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14915457367897034,
      "rewards/margins": 0.0529840886592865,
      "rewards/rejected": -0.20213866233825684,
      "step": 349
    },
    {
      "epoch": 0.9665170866413532,
      "grad_norm": 0.18923406302928925,
      "learning_rate": 4.295873375244319e-06,
      "log_odds_chosen": 0.5657058358192444,
      "log_odds_ratio": -0.4569021463394165,
      "logits/chosen": -0.2960559129714966,
      "logits/rejected": -1.5297155380249023,
      "logps/chosen": -1.446962833404541,
      "logps/rejected": -1.9055999517440796,
      "loss": 1.5727,
      "nll_loss": 1.527005672454834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14469628036022186,
      "rewards/margins": 0.045863717794418335,
      "rewards/rejected": -0.1905599981546402,
      "step": 350
    },
    {
      "epoch": 0.9692785640317569,
      "grad_norm": 0.2020733803510666,
      "learning_rate": 4.290288771415536e-06,
      "log_odds_chosen": 0.47521698474884033,
      "log_odds_ratio": -0.48997536301612854,
      "logits/chosen": -0.2961908280849457,
      "logits/rejected": -1.738245964050293,
      "logps/chosen": -1.6070820093154907,
      "logps/rejected": -2.003466844558716,
      "loss": 1.7209,
      "nll_loss": 1.6719499826431274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16070818901062012,
      "rewards/margins": 0.039638496935367584,
      "rewards/rejected": -0.2003466784954071,
      "step": 351
    },
    {
      "epoch": 0.9720400414221608,
      "grad_norm": 0.19740743935108185,
      "learning_rate": 4.284685769599658e-06,
      "log_odds_chosen": 0.5527662038803101,
      "log_odds_ratio": -0.4617147445678711,
      "logits/chosen": -0.3190363645553589,
      "logits/rejected": -1.6859633922576904,
      "logps/chosen": -1.5147120952606201,
      "logps/rejected": -1.9768071174621582,
      "loss": 1.6243,
      "nll_loss": 1.5781550407409668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15147122740745544,
      "rewards/margins": 0.04620949178934097,
      "rewards/rejected": -0.19768072664737701,
      "step": 352
    },
    {
      "epoch": 0.9748015188125647,
      "grad_norm": 0.19644393026828766,
      "learning_rate": 4.279064427376199e-06,
      "log_odds_chosen": 0.5512232780456543,
      "log_odds_ratio": -0.45938748121261597,
      "logits/chosen": -0.2991534173488617,
      "logits/rejected": -1.5584678649902344,
      "logps/chosen": -1.5441019535064697,
      "logps/rejected": -2.002403736114502,
      "loss": 1.6453,
      "nll_loss": 1.5993587970733643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544102132320404,
      "rewards/margins": 0.045830175280570984,
      "rewards/rejected": -0.2002403736114502,
      "step": 353
    },
    {
      "epoch": 0.9775629962029686,
      "grad_norm": 0.19680863618850708,
      "learning_rate": 4.273424802513145e-06,
      "log_odds_chosen": 0.5857518911361694,
      "log_odds_ratio": -0.4452309012413025,
      "logits/chosen": -0.2941315174102783,
      "logits/rejected": -1.6820390224456787,
      "logps/chosen": -1.5470666885375977,
      "logps/rejected": -2.031536102294922,
      "loss": 1.6621,
      "nll_loss": 1.6175867319107056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1547066569328308,
      "rewards/margins": 0.04844695329666138,
      "rewards/rejected": -0.2031536102294922,
      "step": 354
    },
    {
      "epoch": 0.9803244735933725,
      "grad_norm": 0.18272706866264343,
      "learning_rate": 4.267766952966369e-06,
      "log_odds_chosen": 0.7239515781402588,
      "log_odds_ratio": -0.4105169177055359,
      "logits/chosen": -0.23595450818538666,
      "logits/rejected": -1.5807809829711914,
      "logps/chosen": -1.3758444786071777,
      "logps/rejected": -1.9615696668624878,
      "loss": 1.4949,
      "nll_loss": 1.4538719654083252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13758444786071777,
      "rewards/margins": 0.058572523295879364,
      "rewards/rejected": -0.19615697860717773,
      "step": 355
    },
    {
      "epoch": 0.9830859509837763,
      "grad_norm": 0.19880905747413635,
      "learning_rate": 4.26209093687903e-06,
      "log_odds_chosen": 0.4604690670967102,
      "log_odds_ratio": -0.4963679313659668,
      "logits/chosen": -0.30030739307403564,
      "logits/rejected": -1.8233280181884766,
      "logps/chosen": -1.5396702289581299,
      "logps/rejected": -1.922170639038086,
      "loss": 1.6588,
      "nll_loss": 1.6091995239257812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.153967022895813,
      "rewards/margins": 0.038250040262937546,
      "rewards/rejected": -0.19221706688404083,
      "step": 356
    },
    {
      "epoch": 0.9858474283741802,
      "grad_norm": 0.19747234880924225,
      "learning_rate": 4.2563968125809734e-06,
      "log_odds_chosen": 0.5938950777053833,
      "log_odds_ratio": -0.44322288036346436,
      "logits/chosen": -0.17701445519924164,
      "logits/rejected": -1.6618235111236572,
      "logps/chosen": -1.60313880443573,
      "logps/rejected": -2.101062774658203,
      "loss": 1.7054,
      "nll_loss": 1.6611095666885376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16031388938426971,
      "rewards/margins": 0.04979238659143448,
      "rewards/rejected": -0.2101062536239624,
      "step": 357
    },
    {
      "epoch": 0.988608905764584,
      "grad_norm": 0.19918540120124817,
      "learning_rate": 4.2506846385881375e-06,
      "log_odds_chosen": 0.6773942708969116,
      "log_odds_ratio": -0.4168775975704193,
      "logits/chosen": -0.4877479672431946,
      "logits/rejected": -1.6255730390548706,
      "logps/chosen": -1.409053087234497,
      "logps/rejected": -1.9528459310531616,
      "loss": 1.5337,
      "nll_loss": 1.491982340812683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14090532064437866,
      "rewards/margins": 0.05437929555773735,
      "rewards/rejected": -0.19528460502624512,
      "step": 358
    },
    {
      "epoch": 0.9913703831549879,
      "grad_norm": 0.1901620328426361,
      "learning_rate": 4.2449544736019486e-06,
      "log_odds_chosen": 0.5646210312843323,
      "log_odds_ratio": -0.45523136854171753,
      "logits/chosen": -0.22285765409469604,
      "logits/rejected": -1.4501441717147827,
      "logps/chosen": -1.479009985923767,
      "logps/rejected": -1.9385974407196045,
      "loss": 1.5958,
      "nll_loss": 1.5503125190734863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479010134935379,
      "rewards/margins": 0.045958735048770905,
      "rewards/rejected": -0.1938597410917282,
      "step": 359
    },
    {
      "epoch": 0.9941318605453918,
      "grad_norm": 0.18198014795780182,
      "learning_rate": 4.239206376508716e-06,
      "log_odds_chosen": 0.7485941648483276,
      "log_odds_ratio": -0.39402005076408386,
      "logits/chosen": -0.25095412135124207,
      "logits/rejected": -1.9625945091247559,
      "logps/chosen": -1.4998773336410522,
      "logps/rejected": -2.127000570297241,
      "loss": 1.6257,
      "nll_loss": 1.5863466262817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14998774230480194,
      "rewards/margins": 0.06271231919527054,
      "rewards/rejected": -0.21270006895065308,
      "step": 360
    },
    {
      "epoch": 0.9968933379357956,
      "grad_norm": 0.18113547563552856,
      "learning_rate": 4.233440406379032e-06,
      "log_odds_chosen": 0.560468852519989,
      "log_odds_ratio": -0.45691755414009094,
      "logits/chosen": -0.35995978116989136,
      "logits/rejected": -1.630347490310669,
      "logps/chosen": -1.4223926067352295,
      "logps/rejected": -1.871058702468872,
      "loss": 1.5472,
      "nll_loss": 1.501501202583313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1422392576932907,
      "rewards/margins": 0.04486660659313202,
      "rewards/rejected": -0.18710586428642273,
      "step": 361
    },
    {
      "epoch": 0.9996548153261995,
      "grad_norm": 0.1977638155221939,
      "learning_rate": 4.227656622467162e-06,
      "log_odds_chosen": 0.6151151657104492,
      "log_odds_ratio": -0.4389611482620239,
      "logits/chosen": -0.34485432505607605,
      "logits/rejected": -1.5607213973999023,
      "logps/chosen": -1.528998851776123,
      "logps/rejected": -2.0421009063720703,
      "loss": 1.6442,
      "nll_loss": 1.6003334522247314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15289989113807678,
      "rewards/margins": 0.05131019651889801,
      "rewards/rejected": -0.2042100876569748,
      "step": 362
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.4796704351902008,
      "learning_rate": 4.221855084210433e-06,
      "log_odds_chosen": 0.42105579376220703,
      "log_odds_ratio": -0.5046184659004211,
      "logits/chosen": -0.522539496421814,
      "logits/rejected": -2.0535831451416016,
      "logps/chosen": -1.5670783519744873,
      "logps/rejected": -1.9137303829193115,
      "loss": 1.6859,
      "nll_loss": 1.635398030281067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15670783817768097,
      "rewards/margins": 0.034665197134017944,
      "rewards/rejected": -0.1913730353116989,
      "step": 363
    },
    {
      "epoch": 1.0027614773904039,
      "grad_norm": 0.2030143141746521,
      "learning_rate": 4.2160358512286266e-06,
      "log_odds_chosen": 0.5737169981002808,
      "log_odds_ratio": -0.45416390895843506,
      "logits/chosen": -0.2930692732334137,
      "logits/rejected": -1.6667262315750122,
      "logps/chosen": -1.5078061819076538,
      "logps/rejected": -1.9778721332550049,
      "loss": 1.6304,
      "nll_loss": 1.5849525928497314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1507806032896042,
      "rewards/margins": 0.04700660705566406,
      "rewards/rejected": -0.19778722524642944,
      "step": 364
    },
    {
      "epoch": 1.0055229547808078,
      "grad_norm": 0.18416902422904968,
      "learning_rate": 4.210198983323366e-06,
      "log_odds_chosen": 0.6264990568161011,
      "log_odds_ratio": -0.4338659346103668,
      "logits/chosen": -0.2520604133605957,
      "logits/rejected": -1.8486303091049194,
      "logps/chosen": -1.5509365797042847,
      "logps/rejected": -2.071831703186035,
      "loss": 1.6494,
      "nll_loss": 1.606053113937378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15509365499019623,
      "rewards/margins": 0.052089497447013855,
      "rewards/rejected": -0.20718316733837128,
      "step": 365
    },
    {
      "epoch": 1.0082844321712117,
      "grad_norm": 0.19729849696159363,
      "learning_rate": 4.204344540477499e-06,
      "log_odds_chosen": 0.6402697563171387,
      "log_odds_ratio": -0.429515540599823,
      "logits/chosen": -0.288376122713089,
      "logits/rejected": -1.6332948207855225,
      "logps/chosen": -1.5338780879974365,
      "logps/rejected": -2.068819046020508,
      "loss": 1.6531,
      "nll_loss": 1.6101782321929932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15338779985904694,
      "rewards/margins": 0.053494103252887726,
      "rewards/rejected": -0.20688191056251526,
      "step": 366
    },
    {
      "epoch": 1.0110459095616156,
      "grad_norm": 0.18270985782146454,
      "learning_rate": 4.1984725828544855e-06,
      "log_odds_chosen": 0.6716368198394775,
      "log_odds_ratio": -0.41607195138931274,
      "logits/chosen": -0.27237242460250854,
      "logits/rejected": -1.8720418214797974,
      "logps/chosen": -1.4672292470932007,
      "logps/rejected": -2.0190887451171875,
      "loss": 1.5786,
      "nll_loss": 1.5369707345962524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1467229127883911,
      "rewards/margins": 0.0551859587430954,
      "rewards/rejected": -0.2019088715314865,
      "step": 367
    },
    {
      "epoch": 1.0138073869520194,
      "grad_norm": 0.20039811730384827,
      "learning_rate": 4.192583170797775e-06,
      "log_odds_chosen": 0.5032901167869568,
      "log_odds_ratio": -0.4751865863800049,
      "logits/chosen": -0.3330274224281311,
      "logits/rejected": -1.4479092359542847,
      "logps/chosen": -1.5037407875061035,
      "logps/rejected": -1.9160493612289429,
      "loss": 1.6307,
      "nll_loss": 1.5831364393234253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15037408471107483,
      "rewards/margins": 0.041230857372283936,
      "rewards/rejected": -0.19160494208335876,
      "step": 368
    },
    {
      "epoch": 1.016568864342423,
      "grad_norm": 0.1855594515800476,
      "learning_rate": 4.186676364830187e-06,
      "log_odds_chosen": 0.5586062669754028,
      "log_odds_ratio": -0.45506125688552856,
      "logits/chosen": -0.23738795518875122,
      "logits/rejected": -1.6253269910812378,
      "logps/chosen": -1.566367745399475,
      "logps/rejected": -2.030472755432129,
      "loss": 1.6675,
      "nll_loss": 1.6219737529754639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1566367745399475,
      "rewards/margins": 0.04641049727797508,
      "rewards/rejected": -0.20304730534553528,
      "step": 369
    },
    {
      "epoch": 1.019330341732827,
      "grad_norm": 0.2150687575340271,
      "learning_rate": 4.1807522256532925e-06,
      "log_odds_chosen": 0.6354942321777344,
      "log_odds_ratio": -0.43560659885406494,
      "logits/chosen": -0.29759910702705383,
      "logits/rejected": -1.7936424016952515,
      "logps/chosen": -1.621231198310852,
      "logps/rejected": -2.1618242263793945,
      "loss": 1.7339,
      "nll_loss": 1.6903626918792725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16212311387062073,
      "rewards/margins": 0.054059334099292755,
      "rewards/rejected": -0.21618244051933289,
      "step": 370
    },
    {
      "epoch": 1.0220918191232309,
      "grad_norm": 0.18705667555332184,
      "learning_rate": 4.174810814146789e-06,
      "log_odds_chosen": 0.6377235651016235,
      "log_odds_ratio": -0.4334976375102997,
      "logits/chosen": -0.17882226407527924,
      "logits/rejected": -1.4699535369873047,
      "logps/chosen": -1.5574746131896973,
      "logps/rejected": -2.092029094696045,
      "loss": 1.6697,
      "nll_loss": 1.626399040222168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557474583387375,
      "rewards/margins": 0.05345546454191208,
      "rewards/rejected": -0.20920291543006897,
      "step": 371
    },
    {
      "epoch": 1.0248532965136348,
      "grad_norm": 0.18511676788330078,
      "learning_rate": 4.1688521913678706e-06,
      "log_odds_chosen": 0.7460950613021851,
      "log_odds_ratio": -0.39828044176101685,
      "logits/chosen": -0.24553045630455017,
      "logits/rejected": -1.5528843402862549,
      "logps/chosen": -1.506624460220337,
      "logps/rejected": -2.1312265396118164,
      "loss": 1.6125,
      "nll_loss": 1.5727205276489258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15066243708133698,
      "rewards/margins": 0.06246021389961243,
      "rewards/rejected": -0.2131226509809494,
      "step": 372
    },
    {
      "epoch": 1.0276147739040387,
      "grad_norm": 0.18531759083271027,
      "learning_rate": 4.162876418550606e-06,
      "log_odds_chosen": 0.5151762962341309,
      "log_odds_ratio": -0.47239360213279724,
      "logits/chosen": -0.3635602593421936,
      "logits/rejected": -1.655716061592102,
      "logps/chosen": -1.509564995765686,
      "logps/rejected": -1.9321736097335815,
      "loss": 1.6228,
      "nll_loss": 1.5756025314331055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15095649659633636,
      "rewards/margins": 0.042260847985744476,
      "rewards/rejected": -0.19321735203266144,
      "step": 373
    },
    {
      "epoch": 1.0303762512944425,
      "grad_norm": 0.19160981476306915,
      "learning_rate": 4.156883557105308e-06,
      "log_odds_chosen": 0.537889301776886,
      "log_odds_ratio": -0.46930158138275146,
      "logits/chosen": -0.3415728509426117,
      "logits/rejected": -1.7609002590179443,
      "logps/chosen": -1.4833260774612427,
      "logps/rejected": -1.925763726234436,
      "loss": 1.5902,
      "nll_loss": 1.5432225465774536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1483326107263565,
      "rewards/margins": 0.044243764132261276,
      "rewards/rejected": -0.1925763636827469,
      "step": 374
    },
    {
      "epoch": 1.0331377286848464,
      "grad_norm": 0.2014252096414566,
      "learning_rate": 4.150873668617899e-06,
      "log_odds_chosen": 0.5346254110336304,
      "log_odds_ratio": -0.4631119966506958,
      "logits/chosen": -0.4020942449569702,
      "logits/rejected": -1.6286017894744873,
      "logps/chosen": -1.554813265800476,
      "logps/rejected": -1.9967365264892578,
      "loss": 1.6587,
      "nll_loss": 1.6124264001846313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15548132359981537,
      "rewards/margins": 0.0441923588514328,
      "rewards/rejected": -0.19967366755008698,
      "step": 375
    },
    {
      "epoch": 1.0358992060752503,
      "grad_norm": 0.1909962296485901,
      "learning_rate": 4.144846814849282e-06,
      "log_odds_chosen": 0.4685186743736267,
      "log_odds_ratio": -0.4881165623664856,
      "logits/chosen": -0.19894683361053467,
      "logits/rejected": -1.4885890483856201,
      "logps/chosen": -1.5590081214904785,
      "logps/rejected": -1.9445595741271973,
      "loss": 1.6834,
      "nll_loss": 1.634606957435608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15590080618858337,
      "rewards/margins": 0.03855516016483307,
      "rewards/rejected": -0.19445598125457764,
      "step": 376
    },
    {
      "epoch": 1.0386606834656542,
      "grad_norm": 0.1877821981906891,
      "learning_rate": 4.138803057734705e-06,
      "log_odds_chosen": 0.7200895547866821,
      "log_odds_ratio": -0.4008093476295471,
      "logits/chosen": -0.25993677973747253,
      "logits/rejected": -1.574484944343567,
      "logps/chosen": -1.456033706665039,
      "logps/rejected": -2.0489182472229004,
      "loss": 1.5752,
      "nll_loss": 1.5350782871246338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14560337364673615,
      "rewards/margins": 0.05928843468427658,
      "rewards/rejected": -0.20489181578159332,
      "step": 377
    },
    {
      "epoch": 1.041422160856058,
      "grad_norm": 0.17346148192882538,
      "learning_rate": 4.132742459383122e-06,
      "log_odds_chosen": 0.7830832004547119,
      "log_odds_ratio": -0.3833864629268646,
      "logits/chosen": -0.27740198373794556,
      "logits/rejected": -1.7669141292572021,
      "logps/chosen": -1.380853533744812,
      "logps/rejected": -2.0105137825012207,
      "loss": 1.4985,
      "nll_loss": 1.4602102041244507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13808535039424896,
      "rewards/margins": 0.06296603381633759,
      "rewards/rejected": -0.20105136930942535,
      "step": 378
    },
    {
      "epoch": 1.0441836382464618,
      "grad_norm": 0.183350071310997,
      "learning_rate": 4.126665082076559e-06,
      "log_odds_chosen": 0.5126218795776367,
      "log_odds_ratio": -0.47373971343040466,
      "logits/chosen": -0.3141680955886841,
      "logits/rejected": -1.3418911695480347,
      "logps/chosen": -1.516774296760559,
      "logps/rejected": -1.933377742767334,
      "loss": 1.6442,
      "nll_loss": 1.5968701839447021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1516774296760559,
      "rewards/margins": 0.04166034609079361,
      "rewards/rejected": -0.19333778321743011,
      "step": 379
    },
    {
      "epoch": 1.0469451156368657,
      "grad_norm": 0.18558627367019653,
      "learning_rate": 4.120570988269472e-06,
      "log_odds_chosen": 0.651531994342804,
      "log_odds_ratio": -0.423582524061203,
      "logits/chosen": -0.38820552825927734,
      "logits/rejected": -1.826667070388794,
      "logps/chosen": -1.5054341554641724,
      "logps/rejected": -2.0457534790039062,
      "loss": 1.5926,
      "nll_loss": 1.5502351522445679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1505434215068817,
      "rewards/margins": 0.05403192341327667,
      "rewards/rejected": -0.20457535982131958,
      "step": 380
    },
    {
      "epoch": 1.0497065930272695,
      "grad_norm": 0.19876988232135773,
      "learning_rate": 4.114460240588101e-06,
      "log_odds_chosen": 0.7609922885894775,
      "log_odds_ratio": -0.3903619050979614,
      "logits/chosen": -0.28415167331695557,
      "logits/rejected": -1.509800910949707,
      "logps/chosen": -1.5427088737487793,
      "logps/rejected": -2.184783458709717,
      "loss": 1.6545,
      "nll_loss": 1.6154515743255615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15427090227603912,
      "rewards/margins": 0.06420743465423584,
      "rewards/rejected": -0.21847833693027496,
      "step": 381
    },
    {
      "epoch": 1.0524680704176734,
      "grad_norm": 0.17985652387142181,
      "learning_rate": 4.1083329018298356e-06,
      "log_odds_chosen": 0.7214679718017578,
      "log_odds_ratio": -0.401907354593277,
      "logits/chosen": -0.2644188404083252,
      "logits/rejected": -1.569467544555664,
      "logps/chosen": -1.4487037658691406,
      "logps/rejected": -2.043267250061035,
      "loss": 1.5525,
      "nll_loss": 1.5123515129089355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14487037062644958,
      "rewards/margins": 0.05945635586977005,
      "rewards/rejected": -0.20432673394680023,
      "step": 382
    },
    {
      "epoch": 1.0552295478080773,
      "grad_norm": 0.19042105972766876,
      "learning_rate": 4.102189034962561e-06,
      "log_odds_chosen": 0.6154542565345764,
      "log_odds_ratio": -0.4352980852127075,
      "logits/chosen": -0.21143808960914612,
      "logits/rejected": -1.4588818550109863,
      "logps/chosen": -1.583728551864624,
      "logps/rejected": -2.1008310317993164,
      "loss": 1.6812,
      "nll_loss": 1.6376571655273438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15837284922599792,
      "rewards/margins": 0.051710255444049835,
      "rewards/rejected": -0.21008309721946716,
      "step": 383
    },
    {
      "epoch": 1.0579910251984812,
      "grad_norm": 0.202115997672081,
      "learning_rate": 4.096028703124014e-06,
      "log_odds_chosen": 0.6282423138618469,
      "log_odds_ratio": -0.43798086047172546,
      "logits/chosen": -0.3399620056152344,
      "logits/rejected": -1.302710771560669,
      "logps/chosen": -1.5835597515106201,
      "logps/rejected": -2.1131136417388916,
      "loss": 1.67,
      "nll_loss": 1.6261749267578125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1583559811115265,
      "rewards/margins": 0.05295538902282715,
      "rewards/rejected": -0.21131137013435364,
      "step": 384
    },
    {
      "epoch": 1.060752502588885,
      "grad_norm": 0.18484216928482056,
      "learning_rate": 4.089851969621138e-06,
      "log_odds_chosen": 0.7135899662971497,
      "log_odds_ratio": -0.41620948910713196,
      "logits/chosen": -0.28658849000930786,
      "logits/rejected": -1.5913586616516113,
      "logps/chosen": -1.425489902496338,
      "logps/rejected": -2.015927791595459,
      "loss": 1.5551,
      "nll_loss": 1.5135202407836914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14254900813102722,
      "rewards/margins": 0.059043798595666885,
      "rewards/rejected": -0.20159278810024261,
      "step": 385
    },
    {
      "epoch": 1.063513979979289,
      "grad_norm": 0.20954498648643494,
      "learning_rate": 4.083658897929425e-06,
      "log_odds_chosen": 0.6113450527191162,
      "log_odds_ratio": -0.4348878562450409,
      "logits/chosen": -0.3510580360889435,
      "logits/rejected": -1.5097743272781372,
      "logps/chosen": -1.5490520000457764,
      "logps/rejected": -2.056283950805664,
      "loss": 1.6599,
      "nll_loss": 1.616385817527771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15490520000457764,
      "rewards/margins": 0.050723206251859665,
      "rewards/rejected": -0.2056283950805664,
      "step": 386
    },
    {
      "epoch": 1.0662754573696929,
      "grad_norm": 0.19627171754837036,
      "learning_rate": 4.077449551692268e-06,
      "log_odds_chosen": 0.5390780568122864,
      "log_odds_ratio": -0.46470367908477783,
      "logits/chosen": -0.23739197850227356,
      "logits/rejected": -1.621540904045105,
      "logps/chosen": -1.555280089378357,
      "logps/rejected": -2.001728057861328,
      "loss": 1.6682,
      "nll_loss": 1.6217542886734009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1555280089378357,
      "rewards/margins": 0.044644795358181,
      "rewards/rejected": -0.20017282664775848,
      "step": 387
    },
    {
      "epoch": 1.0690369347600965,
      "grad_norm": 0.1885172724723816,
      "learning_rate": 4.071223994720309e-06,
      "log_odds_chosen": 0.7198303937911987,
      "log_odds_ratio": -0.40047964453697205,
      "logits/chosen": -0.38902541995048523,
      "logits/rejected": -1.71836519241333,
      "logps/chosen": -1.4929300546646118,
      "logps/rejected": -2.0929007530212402,
      "loss": 1.6048,
      "nll_loss": 1.5647118091583252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14929300546646118,
      "rewards/margins": 0.0599970780313015,
      "rewards/rejected": -0.20929010212421417,
      "step": 388
    },
    {
      "epoch": 1.0717984121505004,
      "grad_norm": 0.20288851857185364,
      "learning_rate": 4.064982290990777e-06,
      "log_odds_chosen": 0.5762845277786255,
      "log_odds_ratio": -0.45192721486091614,
      "logits/chosen": -0.3292975425720215,
      "logits/rejected": -1.4449775218963623,
      "logps/chosen": -1.4430099725723267,
      "logps/rejected": -1.9106184244155884,
      "loss": 1.5566,
      "nll_loss": 1.511430263519287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14430099725723267,
      "rewards/margins": 0.04676084965467453,
      "rewards/rejected": -0.1910618394613266,
      "step": 389
    },
    {
      "epoch": 1.0745598895409043,
      "grad_norm": 0.18857373297214508,
      "learning_rate": 4.058724504646834e-06,
      "log_odds_chosen": 0.698142945766449,
      "log_odds_ratio": -0.41126978397369385,
      "logits/chosen": -0.3490840494632721,
      "logits/rejected": -1.5612983703613281,
      "logps/chosen": -1.4951398372650146,
      "logps/rejected": -2.07562255859375,
      "loss": 1.6046,
      "nll_loss": 1.563432216644287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14951398968696594,
      "rewards/margins": 0.058048274368047714,
      "rewards/rejected": -0.20756226778030396,
      "step": 390
    },
    {
      "epoch": 1.0773213669313082,
      "grad_norm": 0.18817338347434998,
      "learning_rate": 4.0524506999969185e-06,
      "log_odds_chosen": 0.5495921969413757,
      "log_odds_ratio": -0.46190890669822693,
      "logits/chosen": -0.32165098190307617,
      "logits/rejected": -1.7347975969314575,
      "logps/chosen": -1.5509448051452637,
      "logps/rejected": -2.0019354820251465,
      "loss": 1.6579,
      "nll_loss": 1.6116652488708496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15509448945522308,
      "rewards/margins": 0.045099057257175446,
      "rewards/rejected": -0.20019352436065674,
      "step": 391
    },
    {
      "epoch": 1.080082844321712,
      "grad_norm": 0.18304969370365143,
      "learning_rate": 4.046160941514079e-06,
      "log_odds_chosen": 0.7243356704711914,
      "log_odds_ratio": -0.4019787311553955,
      "logits/chosen": -0.20833127200603485,
      "logits/rejected": -1.3766155242919922,
      "logps/chosen": -1.4449529647827148,
      "logps/rejected": -2.0372161865234375,
      "loss": 1.5421,
      "nll_loss": 1.5019174814224243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14449530839920044,
      "rewards/margins": 0.05922630429267883,
      "rewards/rejected": -0.20372159779071808,
      "step": 392
    },
    {
      "epoch": 1.082844321712116,
      "grad_norm": 0.23034730553627014,
      "learning_rate": 4.039855293835316e-06,
      "log_odds_chosen": 0.6121989488601685,
      "log_odds_ratio": -0.4352457523345947,
      "logits/chosen": -0.37496262788772583,
      "logits/rejected": -1.6959614753723145,
      "logps/chosen": -1.4758626222610474,
      "logps/rejected": -1.9751558303833008,
      "loss": 1.5737,
      "nll_loss": 1.530150294303894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14758625626564026,
      "rewards/margins": 0.04992932081222534,
      "rewards/rejected": -0.1975155770778656,
      "step": 393
    },
    {
      "epoch": 1.0856057991025199,
      "grad_norm": 0.17257992923259735,
      "learning_rate": 4.033533821760917e-06,
      "log_odds_chosen": 0.7732821702957153,
      "log_odds_ratio": -0.3887802064418793,
      "logits/chosen": -0.3277055025100708,
      "logits/rejected": -1.868857741355896,
      "logps/chosen": -1.4420204162597656,
      "logps/rejected": -2.0812883377075195,
      "loss": 1.5466,
      "nll_loss": 1.5076910257339478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14420203864574432,
      "rewards/margins": 0.06392678618431091,
      "rewards/rejected": -0.20812883973121643,
      "step": 394
    },
    {
      "epoch": 1.0883672764929238,
      "grad_norm": 0.1858333796262741,
      "learning_rate": 4.027196590253786e-06,
      "log_odds_chosen": 0.4944222569465637,
      "log_odds_ratio": -0.4831688404083252,
      "logits/chosen": -0.28738462924957275,
      "logits/rejected": -1.3927885293960571,
      "logps/chosen": -1.4958670139312744,
      "logps/rejected": -1.8963196277618408,
      "loss": 1.6172,
      "nll_loss": 1.5688387155532837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14958669245243073,
      "rewards/margins": 0.04004526883363724,
      "rewards/rejected": -0.18963195383548737,
      "step": 395
    },
    {
      "epoch": 1.0911287538833276,
      "grad_norm": 0.1886051446199417,
      "learning_rate": 4.020843664438783e-06,
      "log_odds_chosen": 0.6189723610877991,
      "log_odds_ratio": -0.4430191218852997,
      "logits/chosen": -0.30028706789016724,
      "logits/rejected": -1.7108986377716064,
      "logps/chosen": -1.446290135383606,
      "logps/rejected": -1.9520014524459839,
      "loss": 1.5704,
      "nll_loss": 1.5261311531066895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14462901651859283,
      "rewards/margins": 0.05057113245129585,
      "rewards/rejected": -0.1952001452445984,
      "step": 396
    },
    {
      "epoch": 1.0938902312737315,
      "grad_norm": 0.189390167593956,
      "learning_rate": 4.01447510960205e-06,
      "log_odds_chosen": 0.7106426358222961,
      "log_odds_ratio": -0.40562015771865845,
      "logits/chosen": -0.4158485531806946,
      "logits/rejected": -1.7321726083755493,
      "logps/chosen": -1.5334806442260742,
      "logps/rejected": -2.1250576972961426,
      "loss": 1.6332,
      "nll_loss": 1.592668890953064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15334807336330414,
      "rewards/margins": 0.059157684445381165,
      "rewards/rejected": -0.2125057429075241,
      "step": 397
    },
    {
      "epoch": 1.0966517086641354,
      "grad_norm": 0.19324593245983124,
      "learning_rate": 4.008090991190341e-06,
      "log_odds_chosen": 0.49737095832824707,
      "log_odds_ratio": -0.47871023416519165,
      "logits/chosen": -0.20496344566345215,
      "logits/rejected": -1.5982781648635864,
      "logps/chosen": -1.570819616317749,
      "logps/rejected": -1.9830747842788696,
      "loss": 1.6702,
      "nll_loss": 1.622377634048462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1570819765329361,
      "rewards/margins": 0.04122550040483475,
      "rewards/rejected": -0.19830746948719025,
      "step": 398
    },
    {
      "epoch": 1.099413186054539,
      "grad_norm": 0.17873111367225647,
      "learning_rate": 4.001691374810352e-06,
      "log_odds_chosen": 0.7614402770996094,
      "log_odds_ratio": -0.39445751905441284,
      "logits/chosen": -0.27544835209846497,
      "logits/rejected": -1.992354154586792,
      "logps/chosen": -1.5202884674072266,
      "logps/rejected": -2.1587905883789062,
      "loss": 1.6232,
      "nll_loss": 1.5837651491165161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15202882885932922,
      "rewards/margins": 0.06385020911693573,
      "rewards/rejected": -0.21587903797626495,
      "step": 399
    },
    {
      "epoch": 1.102174663444943,
      "grad_norm": 0.17042513191699982,
      "learning_rate": 3.99527632622804e-06,
      "log_odds_chosen": 0.6560631990432739,
      "log_odds_ratio": -0.4252350628376007,
      "logits/chosen": -0.3314496576786041,
      "logits/rejected": -1.559665322303772,
      "logps/chosen": -1.4409160614013672,
      "logps/rejected": -1.9774514436721802,
      "loss": 1.5645,
      "nll_loss": 1.5220184326171875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14409160614013672,
      "rewards/margins": 0.05365355312824249,
      "rewards/rejected": -0.19774514436721802,
      "step": 400
    },
    {
      "epoch": 1.1049361408353469,
      "grad_norm": 0.1929718255996704,
      "learning_rate": 3.988845911367957e-06,
      "log_odds_chosen": 0.6236564517021179,
      "log_odds_ratio": -0.4364378750324249,
      "logits/chosen": -0.3177638649940491,
      "logits/rejected": -1.5883958339691162,
      "logps/chosen": -1.598113775253296,
      "logps/rejected": -2.1222128868103027,
      "loss": 1.6877,
      "nll_loss": 1.644063949584961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1598113626241684,
      "rewards/margins": 0.052409932017326355,
      "rewards/rejected": -0.21222129464149475,
      "step": 401
    },
    {
      "epoch": 1.1076976182257507,
      "grad_norm": 0.17265060544013977,
      "learning_rate": 3.982400196312565e-06,
      "log_odds_chosen": 0.6908576488494873,
      "log_odds_ratio": -0.4076445698738098,
      "logits/chosen": -0.28831660747528076,
      "logits/rejected": -1.4703201055526733,
      "logps/chosen": -1.4102579355239868,
      "logps/rejected": -1.9662861824035645,
      "loss": 1.5263,
      "nll_loss": 1.4855272769927979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14102579653263092,
      "rewards/margins": 0.05560281127691269,
      "rewards/rejected": -0.1966286301612854,
      "step": 402
    },
    {
      "epoch": 1.1104590956161546,
      "grad_norm": 0.18258056044578552,
      "learning_rate": 3.975939247301558e-06,
      "log_odds_chosen": 0.6171894669532776,
      "log_odds_ratio": -0.43477940559387207,
      "logits/chosen": -0.34336021542549133,
      "logits/rejected": -1.718552589416504,
      "logps/chosen": -1.5480486154556274,
      "logps/rejected": -2.061230421066284,
      "loss": 1.6639,
      "nll_loss": 1.620387077331543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15480485558509827,
      "rewards/margins": 0.051318198442459106,
      "rewards/rejected": -0.20612303912639618,
      "step": 403
    },
    {
      "epoch": 1.1132205730065585,
      "grad_norm": 0.17894716560840607,
      "learning_rate": 3.969463130731183e-06,
      "log_odds_chosen": 0.601898729801178,
      "log_odds_ratio": -0.44408684968948364,
      "logits/chosen": -0.2595762610435486,
      "logits/rejected": -1.671970248222351,
      "logps/chosen": -1.480761170387268,
      "logps/rejected": -1.973817229270935,
      "loss": 1.5901,
      "nll_loss": 1.5457121133804321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.148076131939888,
      "rewards/margins": 0.049305595457553864,
      "rewards/rejected": -0.19738171994686127,
      "step": 404
    },
    {
      "epoch": 1.1159820503969624,
      "grad_norm": 0.182702898979187,
      "learning_rate": 3.9629719131535595e-06,
      "log_odds_chosen": 0.5538443922996521,
      "log_odds_ratio": -0.463712602853775,
      "logits/chosen": -0.3739258646965027,
      "logits/rejected": -1.5419493913650513,
      "logps/chosen": -1.4951484203338623,
      "logps/rejected": -1.9511198997497559,
      "loss": 1.6103,
      "nll_loss": 1.563896656036377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14951485395431519,
      "rewards/margins": 0.04559716209769249,
      "rewards/rejected": -0.19511200487613678,
      "step": 405
    },
    {
      "epoch": 1.1187435277873663,
      "grad_norm": 0.1713375300168991,
      "learning_rate": 3.9564656612759904e-06,
      "log_odds_chosen": 0.6431328058242798,
      "log_odds_ratio": -0.42764732241630554,
      "logits/chosen": -0.29896292090415955,
      "logits/rejected": -1.8772965669631958,
      "logps/chosen": -1.4999200105667114,
      "logps/rejected": -2.031973361968994,
      "loss": 1.6098,
      "nll_loss": 1.5670844316482544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14999200403690338,
      "rewards/margins": 0.05320533737540245,
      "rewards/rejected": -0.20319733023643494,
      "step": 406
    },
    {
      "epoch": 1.1215050051777702,
      "grad_norm": 0.1880570352077484,
      "learning_rate": 3.94994444196028e-06,
      "log_odds_chosen": 0.6853294968605042,
      "log_odds_ratio": -0.41755619645118713,
      "logits/chosen": -0.2672858238220215,
      "logits/rejected": -1.5635608434677124,
      "logps/chosen": -1.5350513458251953,
      "logps/rejected": -2.1063671112060547,
      "loss": 1.6214,
      "nll_loss": 1.579624056816101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1535051316022873,
      "rewards/margins": 0.0571315735578537,
      "rewards/rejected": -0.21063672006130219,
      "step": 407
    },
    {
      "epoch": 1.124266482568174,
      "grad_norm": 0.17640192806720734,
      "learning_rate": 3.943408322222049e-06,
      "log_odds_chosen": 0.6473301649093628,
      "log_odds_ratio": -0.430767297744751,
      "logits/chosen": -0.25322699546813965,
      "logits/rejected": -1.3362030982971191,
      "logps/chosen": -1.4611663818359375,
      "logps/rejected": -1.9962221384048462,
      "loss": 1.5796,
      "nll_loss": 1.5365506410598755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14611662924289703,
      "rewards/margins": 0.053505584597587585,
      "rewards/rejected": -0.19962221384048462,
      "step": 408
    },
    {
      "epoch": 1.127027959958578,
      "grad_norm": 0.1975967288017273,
      "learning_rate": 3.936857369230037e-06,
      "log_odds_chosen": 0.5013847351074219,
      "log_odds_ratio": -0.48179811239242554,
      "logits/chosen": -0.24628515541553497,
      "logits/rejected": -1.5175528526306152,
      "logps/chosen": -1.5307085514068604,
      "logps/rejected": -1.9439282417297363,
      "loss": 1.6331,
      "nll_loss": 1.584876537322998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.153070867061615,
      "rewards/margins": 0.04132195562124252,
      "rewards/rejected": -0.1943928301334381,
      "step": 409
    },
    {
      "epoch": 1.1297894373489816,
      "grad_norm": 0.19114039838314056,
      "learning_rate": 3.930291650305424e-06,
      "log_odds_chosen": 0.6988283395767212,
      "log_odds_ratio": -0.40597397089004517,
      "logits/chosen": -0.25875231623649597,
      "logits/rejected": -1.3210163116455078,
      "logps/chosen": -1.4922490119934082,
      "logps/rejected": -2.0712697505950928,
      "loss": 1.5831,
      "nll_loss": 1.5425041913986206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1492249071598053,
      "rewards/margins": 0.05790204927325249,
      "rewards/rejected": -0.20712696015834808,
      "step": 410
    },
    {
      "epoch": 1.1325509147393855,
      "grad_norm": 0.1878093183040619,
      "learning_rate": 3.92371123292113e-06,
      "log_odds_chosen": 0.6247770190238953,
      "log_odds_ratio": -0.43160274624824524,
      "logits/chosen": -0.2506449520587921,
      "logits/rejected": -1.691251516342163,
      "logps/chosen": -1.5277273654937744,
      "logps/rejected": -2.0446317195892334,
      "loss": 1.6273,
      "nll_loss": 1.5841726064682007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15277275443077087,
      "rewards/margins": 0.051690444350242615,
      "rewards/rejected": -0.2044631838798523,
      "step": 411
    },
    {
      "epoch": 1.1353123921297894,
      "grad_norm": 0.1842024326324463,
      "learning_rate": 3.917116184701125e-06,
      "log_odds_chosen": 0.7279251217842102,
      "log_odds_ratio": -0.4019698202610016,
      "logits/chosen": -0.2969970107078552,
      "logits/rejected": -1.9143730401992798,
      "logps/chosen": -1.4923046827316284,
      "logps/rejected": -2.099824905395508,
      "loss": 1.5904,
      "nll_loss": 1.5501734018325806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1492304652929306,
      "rewards/margins": 0.0607520155608654,
      "rewards/rejected": -0.2099824696779251,
      "step": 412
    },
    {
      "epoch": 1.1380738695201933,
      "grad_norm": 0.2017175853252411,
      "learning_rate": 3.910506573419734e-06,
      "log_odds_chosen": 0.8388389348983765,
      "log_odds_ratio": -0.3656638562679291,
      "logits/chosen": -0.27687397599220276,
      "logits/rejected": -1.5076192617416382,
      "logps/chosen": -1.4898481369018555,
      "logps/rejected": -2.190901279449463,
      "loss": 1.5889,
      "nll_loss": 1.5523051023483276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14898481965065002,
      "rewards/margins": 0.07010531425476074,
      "rewards/rejected": -0.21909013390541077,
      "step": 413
    },
    {
      "epoch": 1.1408353469105972,
      "grad_norm": 0.18677456676959991,
      "learning_rate": 3.903882467000938e-06,
      "log_odds_chosen": 0.789838433265686,
      "log_odds_ratio": -0.37885501980781555,
      "logits/chosen": -0.2613790035247803,
      "logits/rejected": -1.6607636213302612,
      "logps/chosen": -1.5320932865142822,
      "logps/rejected": -2.194924831390381,
      "loss": 1.6278,
      "nll_loss": 1.5898661613464355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15320934355258942,
      "rewards/margins": 0.06628316640853882,
      "rewards/rejected": -0.21949250996112823,
      "step": 414
    },
    {
      "epoch": 1.143596824301001,
      "grad_norm": 0.18295292556285858,
      "learning_rate": 3.897243933517679e-06,
      "log_odds_chosen": 0.697784423828125,
      "log_odds_ratio": -0.40981873869895935,
      "logits/chosen": -0.36293089389801025,
      "logits/rejected": -1.7186784744262695,
      "logps/chosen": -1.4446918964385986,
      "logps/rejected": -2.0167624950408936,
      "loss": 1.5485,
      "nll_loss": 1.50748872756958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14446918666362762,
      "rewards/margins": 0.05720707029104233,
      "rewards/rejected": -0.20167624950408936,
      "step": 415
    },
    {
      "epoch": 1.146358301691405,
      "grad_norm": 0.18878361582756042,
      "learning_rate": 3.890591041191162e-06,
      "log_odds_chosen": 0.7591959834098816,
      "log_odds_ratio": -0.39290934801101685,
      "logits/chosen": -0.2790437340736389,
      "logits/rejected": -1.4805063009262085,
      "logps/chosen": -1.4139214754104614,
      "logps/rejected": -2.03341007232666,
      "loss": 1.5176,
      "nll_loss": 1.478288173675537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14139214158058167,
      "rewards/margins": 0.06194887310266495,
      "rewards/rejected": -0.20334100723266602,
      "step": 416
    },
    {
      "epoch": 1.1491197790818088,
      "grad_norm": 0.2020062804222107,
      "learning_rate": 3.883923858390149e-06,
      "log_odds_chosen": 0.75853431224823,
      "log_odds_ratio": -0.38675639033317566,
      "logits/chosen": -0.3220999538898468,
      "logits/rejected": -1.7704015970230103,
      "logps/chosen": -1.5146489143371582,
      "logps/rejected": -2.149966239929199,
      "loss": 1.6306,
      "nll_loss": 1.5919427871704102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15146489441394806,
      "rewards/margins": 0.06353174149990082,
      "rewards/rejected": -0.21499663591384888,
      "step": 417
    },
    {
      "epoch": 1.1518812564722127,
      "grad_norm": 0.17892806231975555,
      "learning_rate": 3.8772424536302565e-06,
      "log_odds_chosen": 0.5580451488494873,
      "log_odds_ratio": -0.4568861126899719,
      "logits/chosen": -0.3027711510658264,
      "logits/rejected": -1.707384467124939,
      "logps/chosen": -1.5017491579055786,
      "logps/rejected": -1.9595609903335571,
      "loss": 1.5943,
      "nll_loss": 1.5486559867858887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15017491579055786,
      "rewards/margins": 0.04578119143843651,
      "rewards/rejected": -0.19595609605312347,
      "step": 418
    },
    {
      "epoch": 1.1546427338626164,
      "grad_norm": 0.19901612401008606,
      "learning_rate": 3.870546895573258e-06,
      "log_odds_chosen": 0.6981452703475952,
      "log_odds_ratio": -0.4115220010280609,
      "logits/chosen": -0.3455246686935425,
      "logits/rejected": -1.3123235702514648,
      "logps/chosen": -1.479371428489685,
      "logps/rejected": -2.0550196170806885,
      "loss": 1.5787,
      "nll_loss": 1.5375878810882568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14793714880943298,
      "rewards/margins": 0.05756482481956482,
      "rewards/rejected": -0.2055019587278366,
      "step": 419
    },
    {
      "epoch": 1.1574042112530203,
      "grad_norm": 0.1906924694776535,
      "learning_rate": 3.863837253026372e-06,
      "log_odds_chosen": 0.7406031489372253,
      "log_odds_ratio": -0.40153640508651733,
      "logits/chosen": -0.37284234166145325,
      "logits/rejected": -1.6314030885696411,
      "logps/chosen": -1.4350590705871582,
      "logps/rejected": -2.0481374263763428,
      "loss": 1.5474,
      "nll_loss": 1.507211446762085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14350590109825134,
      "rewards/margins": 0.061307840049266815,
      "rewards/rejected": -0.20481374859809875,
      "step": 420
    },
    {
      "epoch": 1.1601656886434242,
      "grad_norm": 0.19268250465393066,
      "learning_rate": 3.857113594941556e-06,
      "log_odds_chosen": 0.5946828126907349,
      "log_odds_ratio": -0.44611191749572754,
      "logits/chosen": -0.3333417475223541,
      "logits/rejected": -1.820539116859436,
      "logps/chosen": -1.5357310771942139,
      "logps/rejected": -2.030311346054077,
      "loss": 1.6374,
      "nll_loss": 1.5927873849868774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15357311069965363,
      "rewards/margins": 0.04945802688598633,
      "rewards/rejected": -0.20303113758563995,
      "step": 421
    },
    {
      "epoch": 1.162927166033828,
      "grad_norm": 0.1852002888917923,
      "learning_rate": 3.8503759904148005e-06,
      "log_odds_chosen": 0.753965437412262,
      "log_odds_ratio": -0.39210668206214905,
      "logits/chosen": -0.2807456851005554,
      "logits/rejected": -1.7219178676605225,
      "logps/chosen": -1.4659725427627563,
      "logps/rejected": -2.0921542644500732,
      "loss": 1.5699,
      "nll_loss": 1.5306528806686401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1465972512960434,
      "rewards/margins": 0.0626181811094284,
      "rewards/rejected": -0.209215447306633,
      "step": 422
    },
    {
      "epoch": 1.165688643424232,
      "grad_norm": 0.18899178504943848,
      "learning_rate": 3.843624508685416e-06,
      "log_odds_chosen": 0.6049898266792297,
      "log_odds_ratio": -0.43925538659095764,
      "logits/chosen": -0.3452371060848236,
      "logits/rejected": -1.7807867527008057,
      "logps/chosen": -1.5395519733428955,
      "logps/rejected": -2.040437698364258,
      "loss": 1.6476,
      "nll_loss": 1.603638768196106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15395519137382507,
      "rewards/margins": 0.05008859187364578,
      "rewards/rejected": -0.20404377579689026,
      "step": 423
    },
    {
      "epoch": 1.1684501208146358,
      "grad_norm": 0.1812516748905182,
      "learning_rate": 3.8368592191353246e-06,
      "log_odds_chosen": 0.6994820833206177,
      "log_odds_ratio": -0.412604957818985,
      "logits/chosen": -0.32768282294273376,
      "logits/rejected": -1.7750557661056519,
      "logps/chosen": -1.4714299440383911,
      "logps/rejected": -2.0467658042907715,
      "loss": 1.5632,
      "nll_loss": 1.5219401121139526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14714302122592926,
      "rewards/margins": 0.05753358453512192,
      "rewards/rejected": -0.20467659831047058,
      "step": 424
    },
    {
      "epoch": 1.1712115982050397,
      "grad_norm": 0.18136382102966309,
      "learning_rate": 3.830080191288342e-06,
      "log_odds_chosen": 0.6253668665885925,
      "log_odds_ratio": -0.42983123660087585,
      "logits/chosen": -0.2867172360420227,
      "logits/rejected": -1.5802618265151978,
      "logps/chosen": -1.5029292106628418,
      "logps/rejected": -2.018980026245117,
      "loss": 1.6134,
      "nll_loss": 1.5704439878463745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15029294788837433,
      "rewards/margins": 0.05160506069660187,
      "rewards/rejected": -0.2018979787826538,
      "step": 425
    },
    {
      "epoch": 1.1739730755954436,
      "grad_norm": 0.19747108221054077,
      "learning_rate": 3.823287494809469e-06,
      "log_odds_chosen": 0.7677461504936218,
      "log_odds_ratio": -0.38692528009414673,
      "logits/chosen": -0.38328537344932556,
      "logits/rejected": -1.6817901134490967,
      "logps/chosen": -1.4731775522232056,
      "logps/rejected": -2.1067628860473633,
      "loss": 1.5885,
      "nll_loss": 1.5498508214950562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14731775224208832,
      "rewards/margins": 0.06335853040218353,
      "rewards/rejected": -0.21067628264427185,
      "step": 426
    },
    {
      "epoch": 1.1767345529858475,
      "grad_norm": 0.185506671667099,
      "learning_rate": 3.816481199504171e-06,
      "log_odds_chosen": 0.6367388963699341,
      "log_odds_ratio": -0.4264675974845886,
      "logits/chosen": -0.3647797703742981,
      "logits/rejected": -1.5863122940063477,
      "logps/chosen": -1.5056264400482178,
      "logps/rejected": -2.031801462173462,
      "loss": 1.6257,
      "nll_loss": 1.5830763578414917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15056264400482178,
      "rewards/margins": 0.052617497742176056,
      "rewards/rejected": -0.20318013429641724,
      "step": 427
    },
    {
      "epoch": 1.1794960303762512,
      "grad_norm": 0.1755794882774353,
      "learning_rate": 3.8096613753176635e-06,
      "log_odds_chosen": 0.584825873374939,
      "log_odds_ratio": -0.4493084251880646,
      "logits/chosen": -0.27612411975860596,
      "logits/rejected": -1.4378138780593872,
      "logps/chosen": -1.3924418687820435,
      "logps/rejected": -1.8575388193130493,
      "loss": 1.5104,
      "nll_loss": 1.4654783010482788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1392441838979721,
      "rewards/margins": 0.046509698033332825,
      "rewards/rejected": -0.18575388193130493,
      "step": 428
    },
    {
      "epoch": 1.1822575077666553,
      "grad_norm": 0.19249552488327026,
      "learning_rate": 3.8028280923341927e-06,
      "log_odds_chosen": 0.6473675966262817,
      "log_odds_ratio": -0.4286215901374817,
      "logits/chosen": -0.384939968585968,
      "logits/rejected": -1.5598773956298828,
      "logps/chosen": -1.4334121942520142,
      "logps/rejected": -1.9639126062393188,
      "loss": 1.5432,
      "nll_loss": 1.500383973121643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14334121346473694,
      "rewards/margins": 0.05305003747344017,
      "rewards/rejected": -0.1963912546634674,
      "step": 429
    },
    {
      "epoch": 1.185018985157059,
      "grad_norm": 0.19077420234680176,
      "learning_rate": 3.7959814207763134e-06,
      "log_odds_chosen": 0.7534008026123047,
      "log_odds_ratio": -0.3935543894767761,
      "logits/chosen": -0.33575281500816345,
      "logits/rejected": -1.7560871839523315,
      "logps/chosen": -1.5460617542266846,
      "logps/rejected": -2.17948579788208,
      "loss": 1.6361,
      "nll_loss": 1.5967589616775513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1546061784029007,
      "rewards/margins": 0.0633423924446106,
      "rewards/rejected": -0.2179485559463501,
      "step": 430
    },
    {
      "epoch": 1.1877804625474628,
      "grad_norm": 0.18546007573604584,
      "learning_rate": 3.789121431004168e-06,
      "log_odds_chosen": 0.6419503092765808,
      "log_odds_ratio": -0.42676353454589844,
      "logits/chosen": -0.36785295605659485,
      "logits/rejected": -1.6887683868408203,
      "logps/chosen": -1.5084519386291504,
      "logps/rejected": -2.036573886871338,
      "loss": 1.5997,
      "nll_loss": 1.5570096969604492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15084518492221832,
      "rewards/margins": 0.05281219631433487,
      "rewards/rejected": -0.2036573737859726,
      "step": 431
    },
    {
      "epoch": 1.1905419399378667,
      "grad_norm": 0.194391131401062,
      "learning_rate": 3.782248193514766e-06,
      "log_odds_chosen": 0.6094563007354736,
      "log_odds_ratio": -0.43929269909858704,
      "logits/chosen": -0.29947635531425476,
      "logits/rejected": -1.4955629110336304,
      "logps/chosen": -1.4138904809951782,
      "logps/rejected": -1.906821846961975,
      "loss": 1.5403,
      "nll_loss": 1.4963343143463135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14138904213905334,
      "rewards/margins": 0.049293152987957,
      "rewards/rejected": -0.19068220257759094,
      "step": 432
    },
    {
      "epoch": 1.1933034173282706,
      "grad_norm": 0.19166447222232819,
      "learning_rate": 3.775361778941257e-06,
      "log_odds_chosen": 0.7113713026046753,
      "log_odds_ratio": -0.403501957654953,
      "logits/chosen": -0.308038592338562,
      "logits/rejected": -1.6283587217330933,
      "logps/chosen": -1.501065731048584,
      "logps/rejected": -2.0941624641418457,
      "loss": 1.6079,
      "nll_loss": 1.567560076713562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15010657906532288,
      "rewards/margins": 0.05930966138839722,
      "rewards/rejected": -0.2094162404537201,
      "step": 433
    },
    {
      "epoch": 1.1960648947186745,
      "grad_norm": 0.18622690439224243,
      "learning_rate": 3.7684622580522057e-06,
      "log_odds_chosen": 0.42840951681137085,
      "log_odds_ratio": -0.5051460862159729,
      "logits/chosen": -0.2970636785030365,
      "logits/rejected": -1.6902334690093994,
      "logps/chosen": -1.4522030353546143,
      "logps/rejected": -1.7967474460601807,
      "loss": 1.5557,
      "nll_loss": 1.5051684379577637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452203243970871,
      "rewards/margins": 0.03445442020893097,
      "rewards/rejected": -0.17967472970485687,
      "step": 434
    },
    {
      "epoch": 1.1988263721090784,
      "grad_norm": 0.17388908565044403,
      "learning_rate": 3.761549701750865e-06,
      "log_odds_chosen": 0.7077789902687073,
      "log_odds_ratio": -0.41333314776420593,
      "logits/chosen": -0.3090924024581909,
      "logits/rejected": -1.843423843383789,
      "logps/chosen": -1.4652737379074097,
      "logps/rejected": -2.052661418914795,
      "loss": 1.5629,
      "nll_loss": 1.5216063261032104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14652739465236664,
      "rewards/margins": 0.058738768100738525,
      "rewards/rejected": -0.20526614785194397,
      "step": 435
    },
    {
      "epoch": 1.2015878494994823,
      "grad_norm": 0.17293590307235718,
      "learning_rate": 3.7546241810744444e-06,
      "log_odds_chosen": 0.5752092003822327,
      "log_odds_ratio": -0.4525831639766693,
      "logits/chosen": -0.32182741165161133,
      "logits/rejected": -1.3567001819610596,
      "logps/chosen": -1.4578838348388672,
      "logps/rejected": -1.9286342859268188,
      "loss": 1.5493,
      "nll_loss": 1.50407075881958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14578840136528015,
      "rewards/margins": 0.047075025737285614,
      "rewards/rejected": -0.19286341965198517,
      "step": 436
    },
    {
      "epoch": 1.2043493268898862,
      "grad_norm": 0.19718696177005768,
      "learning_rate": 3.747685767193385e-06,
      "log_odds_chosen": 0.5642775297164917,
      "log_odds_ratio": -0.4569299817085266,
      "logits/chosen": -0.3862457871437073,
      "logits/rejected": -1.4825284481048584,
      "logps/chosen": -1.5323418378829956,
      "logps/rejected": -1.9992895126342773,
      "loss": 1.6529,
      "nll_loss": 1.6072373390197754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15323419868946075,
      "rewards/margins": 0.046694785356521606,
      "rewards/rejected": -0.19992896914482117,
      "step": 437
    },
    {
      "epoch": 1.20711080428029,
      "grad_norm": 0.17915678024291992,
      "learning_rate": 3.740734531410626e-06,
      "log_odds_chosen": 0.7052585482597351,
      "log_odds_ratio": -0.4100540578365326,
      "logits/chosen": -0.3775138258934021,
      "logits/rejected": -1.859032154083252,
      "logps/chosen": -1.448523998260498,
      "logps/rejected": -2.021770477294922,
      "loss": 1.5652,
      "nll_loss": 1.5241587162017822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1448523849248886,
      "rewards/margins": 0.057324644178152084,
      "rewards/rejected": -0.202177032828331,
      "step": 438
    },
    {
      "epoch": 1.2098722816706937,
      "grad_norm": 0.19827017188072205,
      "learning_rate": 3.7337705451608676e-06,
      "log_odds_chosen": 0.859776496887207,
      "log_odds_ratio": -0.3666639029979706,
      "logits/chosen": -0.4492262601852417,
      "logits/rejected": -1.819311261177063,
      "logps/chosen": -1.3853495121002197,
      "logps/rejected": -2.082892894744873,
      "loss": 1.4954,
      "nll_loss": 1.4586937427520752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13853494822978973,
      "rewards/margins": 0.06975432485342026,
      "rewards/rejected": -0.2082892656326294,
      "step": 439
    },
    {
      "epoch": 1.2126337590610976,
      "grad_norm": 0.19302192330360413,
      "learning_rate": 3.7267938800098454e-06,
      "log_odds_chosen": 0.8500153422355652,
      "log_odds_ratio": -0.36537328362464905,
      "logits/chosen": -0.5428920984268188,
      "logits/rejected": -1.821568489074707,
      "logps/chosen": -1.3568768501281738,
      "logps/rejected": -2.052211046218872,
      "loss": 1.4693,
      "nll_loss": 1.4327150583267212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1356876939535141,
      "rewards/margins": 0.06953340768814087,
      "rewards/rejected": -0.20522108674049377,
      "step": 440
    },
    {
      "epoch": 1.2153952364515015,
      "grad_norm": 0.19974324107170105,
      "learning_rate": 3.7198046076535865e-06,
      "log_odds_chosen": 0.7102779746055603,
      "log_odds_ratio": -0.40437954664230347,
      "logits/chosen": -0.3492871820926666,
      "logits/rejected": -1.5553520917892456,
      "logps/chosen": -1.4951152801513672,
      "logps/rejected": -2.0831298828125,
      "loss": 1.6058,
      "nll_loss": 1.565347671508789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14951153099536896,
      "rewards/margins": 0.05880144238471985,
      "rewards/rejected": -0.20831298828125,
      "step": 441
    },
    {
      "epoch": 1.2181567138419054,
      "grad_norm": 0.18230760097503662,
      "learning_rate": 3.71280279991768e-06,
      "log_odds_chosen": 0.7506513595581055,
      "log_odds_ratio": -0.3923317492008209,
      "logits/chosen": -0.4152137339115143,
      "logits/rejected": -1.5510238409042358,
      "logps/chosen": -1.4440295696258545,
      "logps/rejected": -2.0645270347595215,
      "loss": 1.5654,
      "nll_loss": 1.5261952877044678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14440295100212097,
      "rewards/margins": 0.06204976886510849,
      "rewards/rejected": -0.20645272731781006,
      "step": 442
    },
    {
      "epoch": 1.2209181912323093,
      "grad_norm": 0.17220567166805267,
      "learning_rate": 3.705788528756533e-06,
      "log_odds_chosen": 0.7942907810211182,
      "log_odds_ratio": -0.37784573435783386,
      "logits/chosen": -0.43315792083740234,
      "logits/rejected": -1.8588192462921143,
      "logps/chosen": -1.4370851516723633,
      "logps/rejected": -2.0935044288635254,
      "loss": 1.5386,
      "nll_loss": 1.500786304473877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14370852708816528,
      "rewards/margins": 0.06564193964004517,
      "rewards/rejected": -0.20935045182704926,
      "step": 443
    },
    {
      "epoch": 1.2236796686227132,
      "grad_norm": 0.34615159034729004,
      "learning_rate": 3.698761866252635e-06,
      "log_odds_chosen": 0.6159027218818665,
      "log_odds_ratio": -0.44577932357788086,
      "logits/chosen": -0.4024880826473236,
      "logits/rejected": -1.7161859273910522,
      "logps/chosen": -1.533597707748413,
      "logps/rejected": -2.050471067428589,
      "loss": 1.6393,
      "nll_loss": 1.5947318077087402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1533597856760025,
      "rewards/margins": 0.0516873374581337,
      "rewards/rejected": -0.2050471305847168,
      "step": 444
    },
    {
      "epoch": 1.226441146013117,
      "grad_norm": 0.19309046864509583,
      "learning_rate": 3.691722884615814e-06,
      "log_odds_chosen": 0.6141983270645142,
      "log_odds_ratio": -0.43802952766418457,
      "logits/chosen": -0.2951904237270355,
      "logits/rejected": -1.1809048652648926,
      "logps/chosen": -1.4468181133270264,
      "logps/rejected": -1.9419705867767334,
      "loss": 1.5744,
      "nll_loss": 1.5305941104888916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14468181133270264,
      "rewards/margins": 0.04951523244380951,
      "rewards/rejected": -0.19419705867767334,
      "step": 445
    },
    {
      "epoch": 1.229202623403521,
      "grad_norm": 0.19239898025989532,
      "learning_rate": 3.684671656182497e-06,
      "log_odds_chosen": 0.7434544563293457,
      "log_odds_ratio": -0.39474281668663025,
      "logits/chosen": -0.4174625277519226,
      "logits/rejected": -1.7456690073013306,
      "logps/chosen": -1.4840799570083618,
      "logps/rejected": -2.0977742671966553,
      "loss": 1.5985,
      "nll_loss": 1.5590325593948364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14840799570083618,
      "rewards/margins": 0.06136942654848099,
      "rewards/rejected": -0.20977741479873657,
      "step": 446
    },
    {
      "epoch": 1.2319641007939248,
      "grad_norm": 0.19208678603172302,
      "learning_rate": 3.6776082534149664e-06,
      "log_odds_chosen": 0.7092225551605225,
      "log_odds_ratio": -0.40430110692977905,
      "logits/chosen": -0.4059605598449707,
      "logits/rejected": -1.8414146900177002,
      "logps/chosen": -1.5050946474075317,
      "logps/rejected": -2.0934860706329346,
      "loss": 1.5881,
      "nll_loss": 1.5477027893066406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15050947666168213,
      "rewards/margins": 0.05883914604783058,
      "rewards/rejected": -0.2093486189842224,
      "step": 447
    },
    {
      "epoch": 1.2347255781843287,
      "grad_norm": 0.18131853640079498,
      "learning_rate": 3.670532748900615e-06,
      "log_odds_chosen": 0.7535479664802551,
      "log_odds_ratio": -0.3983326554298401,
      "logits/chosen": -0.3855303227901459,
      "logits/rejected": -1.6423949003219604,
      "logps/chosen": -1.4195005893707275,
      "logps/rejected": -2.0328564643859863,
      "loss": 1.5252,
      "nll_loss": 1.4853616952896118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14195004105567932,
      "rewards/margins": 0.06133558601140976,
      "rewards/rejected": -0.20328564941883087,
      "step": 448
    },
    {
      "epoch": 1.2374870555747326,
      "grad_norm": 0.1773749440908432,
      "learning_rate": 3.663445215351198e-06,
      "log_odds_chosen": 0.794275164604187,
      "log_odds_ratio": -0.38321515917778015,
      "logits/chosen": -0.4386705160140991,
      "logits/rejected": -1.5580754280090332,
      "logps/chosen": -1.4094889163970947,
      "logps/rejected": -2.067706823348999,
      "loss": 1.5152,
      "nll_loss": 1.476863145828247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14094889163970947,
      "rewards/margins": 0.06582178175449371,
      "rewards/rejected": -0.20677067339420319,
      "step": 449
    },
    {
      "epoch": 1.2402485329651363,
      "grad_norm": 0.18508228659629822,
      "learning_rate": 3.656345725602089e-06,
      "log_odds_chosen": 0.8697155714035034,
      "log_odds_ratio": -0.35649657249450684,
      "logits/chosen": -0.36322081089019775,
      "logits/rejected": -1.8790079355239868,
      "logps/chosen": -1.5029995441436768,
      "logps/rejected": -2.235288143157959,
      "loss": 1.5982,
      "nll_loss": 1.5625407695770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15029993653297424,
      "rewards/margins": 0.0732288807630539,
      "rewards/rejected": -0.22352883219718933,
      "step": 450
    },
    {
      "epoch": 1.2430100103555402,
      "grad_norm": 0.19414876401424408,
      "learning_rate": 3.6492343526115292e-06,
      "log_odds_chosen": 0.6894505023956299,
      "log_odds_ratio": -0.4081804156303406,
      "logits/chosen": -0.26924291253089905,
      "logits/rejected": -1.8292944431304932,
      "logps/chosen": -1.5266273021697998,
      "logps/rejected": -2.100613594055176,
      "loss": 1.612,
      "nll_loss": 1.5711458921432495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526627391576767,
      "rewards/margins": 0.05739862471818924,
      "rewards/rejected": -0.21006137132644653,
      "step": 451
    },
    {
      "epoch": 1.245771487745944,
      "grad_norm": 0.18495413661003113,
      "learning_rate": 3.642111169459879e-06,
      "log_odds_chosen": 0.5452620983123779,
      "log_odds_ratio": -0.4608571529388428,
      "logits/chosen": -0.3355942666530609,
      "logits/rejected": -1.535355806350708,
      "logps/chosen": -1.489123821258545,
      "logps/rejected": -1.9351115226745605,
      "loss": 1.5796,
      "nll_loss": 1.533546805381775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14891238510608673,
      "rewards/margins": 0.044598765671253204,
      "rewards/rejected": -0.19351115822792053,
      "step": 452
    },
    {
      "epoch": 1.248532965136348,
      "grad_norm": 0.18209105730056763,
      "learning_rate": 3.634976249348867e-06,
      "log_odds_chosen": 0.6647239923477173,
      "log_odds_ratio": -0.42399975657463074,
      "logits/chosen": -0.3849475681781769,
      "logits/rejected": -1.8075506687164307,
      "logps/chosen": -1.4811009168624878,
      "logps/rejected": -2.028481960296631,
      "loss": 1.5873,
      "nll_loss": 1.5449340343475342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14811009168624878,
      "rewards/margins": 0.05473810061812401,
      "rewards/rejected": -0.2028481811285019,
      "step": 453
    },
    {
      "epoch": 1.2512944425267518,
      "grad_norm": 0.20314469933509827,
      "learning_rate": 3.6278296656008366e-06,
      "log_odds_chosen": 0.8694459199905396,
      "log_odds_ratio": -0.371795654296875,
      "logits/chosen": -0.4006415903568268,
      "logits/rejected": -1.7445772886276245,
      "logps/chosen": -1.3579434156417847,
      "logps/rejected": -2.065506935119629,
      "loss": 1.4782,
      "nll_loss": 1.4409888982772827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13579432666301727,
      "rewards/margins": 0.07075636833906174,
      "rewards/rejected": -0.2065506875514984,
      "step": 454
    },
    {
      "epoch": 1.2540559199171557,
      "grad_norm": 0.1716334968805313,
      "learning_rate": 3.6206714916579925e-06,
      "log_odds_chosen": 0.7703570127487183,
      "log_odds_ratio": -0.3888989984989166,
      "logits/chosen": -0.38477808237075806,
      "logits/rejected": -1.7767083644866943,
      "logps/chosen": -1.398410439491272,
      "logps/rejected": -2.027221441268921,
      "loss": 1.4987,
      "nll_loss": 1.4598515033721924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13984103500843048,
      "rewards/margins": 0.06288108229637146,
      "rewards/rejected": -0.20272211730480194,
      "step": 455
    },
    {
      "epoch": 1.2568173973075596,
      "grad_norm": 0.18591812252998352,
      "learning_rate": 3.613501801081648e-06,
      "log_odds_chosen": 0.617323637008667,
      "log_odds_ratio": -0.4372296929359436,
      "logits/chosen": -0.36923906207084656,
      "logits/rejected": -1.4284781217575073,
      "logps/chosen": -1.4494431018829346,
      "logps/rejected": -1.9530479907989502,
      "loss": 1.5618,
      "nll_loss": 1.5181188583374023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14494431018829346,
      "rewards/margins": 0.05036048963665962,
      "rewards/rejected": -0.19530481100082397,
      "step": 456
    },
    {
      "epoch": 1.2595788746979635,
      "grad_norm": 0.1984872668981552,
      "learning_rate": 3.606320667551466e-06,
      "log_odds_chosen": 0.7281745672225952,
      "log_odds_ratio": -0.4030551612377167,
      "logits/chosen": -0.36235833168029785,
      "logits/rejected": -1.7550745010375977,
      "logps/chosen": -1.49772310256958,
      "logps/rejected": -2.1043448448181152,
      "loss": 1.6083,
      "nll_loss": 1.5679997205734253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1497723013162613,
      "rewards/margins": 0.06066218018531799,
      "rewards/rejected": -0.21043448150157928,
      "step": 457
    },
    {
      "epoch": 1.2623403520883674,
      "grad_norm": 0.1827681064605713,
      "learning_rate": 3.599128164864706e-06,
      "log_odds_chosen": 0.8096928596496582,
      "log_odds_ratio": -0.37837105989456177,
      "logits/chosen": -0.45445504784584045,
      "logits/rejected": -1.770817756652832,
      "logps/chosen": -1.4516648054122925,
      "logps/rejected": -2.1250061988830566,
      "loss": 1.5451,
      "nll_loss": 1.5072699785232544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14516648650169373,
      "rewards/margins": 0.0673341453075409,
      "rewards/rejected": -0.21250061690807343,
      "step": 458
    },
    {
      "epoch": 1.265101829478771,
      "grad_norm": 0.19308075308799744,
      "learning_rate": 3.5919243669354585e-06,
      "log_odds_chosen": 0.6114473342895508,
      "log_odds_ratio": -0.4423988461494446,
      "logits/chosen": -0.405925452709198,
      "logits/rejected": -1.2670363187789917,
      "logps/chosen": -1.5176535844802856,
      "logps/rejected": -2.0253612995147705,
      "loss": 1.623,
      "nll_loss": 1.5787138938903809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517653614282608,
      "rewards/margins": 0.05077076703310013,
      "rewards/rejected": -0.20253612101078033,
      "step": 459
    },
    {
      "epoch": 1.2678633068691751,
      "grad_norm": 0.2073579728603363,
      "learning_rate": 3.5847093477938955e-06,
      "log_odds_chosen": 0.6397342681884766,
      "log_odds_ratio": -0.4318113625049591,
      "logits/chosen": -0.4677776098251343,
      "logits/rejected": -1.5898178815841675,
      "logps/chosen": -1.5188252925872803,
      "logps/rejected": -2.048469066619873,
      "loss": 1.6166,
      "nll_loss": 1.5734236240386963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15188252925872803,
      "rewards/margins": 0.05296438932418823,
      "rewards/rejected": -0.20484691858291626,
      "step": 460
    },
    {
      "epoch": 1.2706247842595788,
      "grad_norm": 0.18931885063648224,
      "learning_rate": 3.5774831815855017e-06,
      "log_odds_chosen": 0.8001710772514343,
      "log_odds_ratio": -0.3749019205570221,
      "logits/chosen": -0.4591723084449768,
      "logits/rejected": -1.6894080638885498,
      "logps/chosen": -1.4038267135620117,
      "logps/rejected": -2.0569498538970947,
      "loss": 1.5196,
      "nll_loss": 1.4821062088012695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14038267731666565,
      "rewards/margins": 0.06531231105327606,
      "rewards/rejected": -0.2056949883699417,
      "step": 461
    },
    {
      "epoch": 1.2733862616499827,
      "grad_norm": 0.17583203315734863,
      "learning_rate": 3.5702459425703146e-06,
      "log_odds_chosen": 0.8868230581283569,
      "log_odds_ratio": -0.3547811210155487,
      "logits/chosen": -0.43302029371261597,
      "logits/rejected": -1.9412630796432495,
      "logps/chosen": -1.508098840713501,
      "logps/rejected": -2.258622884750366,
      "loss": 1.6074,
      "nll_loss": 1.5718796253204346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1508098840713501,
      "rewards/margins": 0.0750524029135704,
      "rewards/rejected": -0.2258622944355011,
      "step": 462
    },
    {
      "epoch": 1.2761477390403866,
      "grad_norm": 0.18731163442134857,
      "learning_rate": 3.562997705122162e-06,
      "log_odds_chosen": 0.7847945094108582,
      "log_odds_ratio": -0.37928110361099243,
      "logits/chosen": -0.38437145948410034,
      "logits/rejected": -1.6156420707702637,
      "logps/chosen": -1.4260213375091553,
      "logps/rejected": -2.070751428604126,
      "loss": 1.5279,
      "nll_loss": 1.489925503730774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14260214567184448,
      "rewards/margins": 0.0644729733467102,
      "rewards/rejected": -0.20707513391971588,
      "step": 463
    },
    {
      "epoch": 1.2789092164307905,
      "grad_norm": 0.17475299537181854,
      "learning_rate": 3.5557385437279e-06,
      "log_odds_chosen": 0.838699221611023,
      "log_odds_ratio": -0.36731863021850586,
      "logits/chosen": -0.3592332601547241,
      "logits/rejected": -1.4209747314453125,
      "logps/chosen": -1.4949580430984497,
      "logps/rejected": -2.196079730987549,
      "loss": 1.5793,
      "nll_loss": 1.5425434112548828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14949579536914825,
      "rewards/margins": 0.07011217623949051,
      "rewards/rejected": -0.21960797905921936,
      "step": 464
    },
    {
      "epoch": 1.2816706938211944,
      "grad_norm": 0.19598983228206635,
      "learning_rate": 3.5484685329866424e-06,
      "log_odds_chosen": 0.5413783192634583,
      "log_odds_ratio": -0.4634351134300232,
      "logits/chosen": -0.3708513081073761,
      "logits/rejected": -1.4379222393035889,
      "logps/chosen": -1.5153756141662598,
      "logps/rejected": -1.9605016708374023,
      "loss": 1.6065,
      "nll_loss": 1.5601321458816528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15153755247592926,
      "rewards/margins": 0.044512614607810974,
      "rewards/rejected": -0.19605018198490143,
      "step": 465
    },
    {
      "epoch": 1.2844321712115983,
      "grad_norm": 0.18149885535240173,
      "learning_rate": 3.541187747608998e-06,
      "log_odds_chosen": 0.9340339303016663,
      "log_odds_ratio": -0.34465107321739197,
      "logits/chosen": -0.48461854457855225,
      "logits/rejected": -1.7797619104385376,
      "logps/chosen": -1.5739936828613281,
      "logps/rejected": -2.3727049827575684,
      "loss": 1.6472,
      "nll_loss": 1.6127065420150757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15739935636520386,
      "rewards/margins": 0.07987111061811447,
      "rewards/rejected": -0.23727048933506012,
      "step": 466
    },
    {
      "epoch": 1.2871936486020021,
      "grad_norm": 0.19978304207324982,
      "learning_rate": 3.533896262416302e-06,
      "log_odds_chosen": 0.7202765941619873,
      "log_odds_ratio": -0.4014014005661011,
      "logits/chosen": -0.4147559404373169,
      "logits/rejected": -1.5541059970855713,
      "logps/chosen": -1.4875589609146118,
      "logps/rejected": -2.0849876403808594,
      "loss": 1.5897,
      "nll_loss": 1.5495574474334717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14875589311122894,
      "rewards/margins": 0.05974285677075386,
      "rewards/rejected": -0.2084987461566925,
      "step": 467
    },
    {
      "epoch": 1.2899551259924058,
      "grad_norm": 0.21298463642597198,
      "learning_rate": 3.5265941523398455e-06,
      "log_odds_chosen": 0.8375830054283142,
      "log_odds_ratio": -0.36393651366233826,
      "logits/chosen": -0.42123696208000183,
      "logits/rejected": -1.5165983438491821,
      "logps/chosen": -1.5110690593719482,
      "logps/rejected": -2.2138826847076416,
      "loss": 1.6014,
      "nll_loss": 1.5650498867034912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15110690891742706,
      "rewards/margins": 0.07028134167194366,
      "rewards/rejected": -0.22138825058937073,
      "step": 468
    },
    {
      "epoch": 1.29271660338281,
      "grad_norm": 0.1818659007549286,
      "learning_rate": 3.519281492420108e-06,
      "log_odds_chosen": 0.5199939012527466,
      "log_odds_ratio": -0.47489961981773376,
      "logits/chosen": -0.4219110906124115,
      "logits/rejected": -1.4790453910827637,
      "logps/chosen": -1.5349076986312866,
      "logps/rejected": -1.9625968933105469,
      "loss": 1.6236,
      "nll_loss": 1.5760908126831055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15349076688289642,
      "rewards/margins": 0.042768917977809906,
      "rewards/rejected": -0.19625969231128693,
      "step": 469
    },
    {
      "epoch": 1.2954780807732136,
      "grad_norm": 0.19590230286121368,
      "learning_rate": 3.5119583578059845e-06,
      "log_odds_chosen": 0.6841270327568054,
      "log_odds_ratio": -0.43147027492523193,
      "logits/chosen": -0.4008702337741852,
      "logits/rejected": -1.6819429397583008,
      "logps/chosen": -1.6086608171463013,
      "logps/rejected": -2.1926984786987305,
      "loss": 1.6814,
      "nll_loss": 1.6382827758789062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16086608171463013,
      "rewards/margins": 0.05840376764535904,
      "rewards/rejected": -0.21926987171173096,
      "step": 470
    },
    {
      "epoch": 1.2982395581636175,
      "grad_norm": 0.17711324989795685,
      "learning_rate": 3.504624823754014e-06,
      "log_odds_chosen": 0.8761448860168457,
      "log_odds_ratio": -0.3517749607563019,
      "logits/chosen": -0.41212159395217896,
      "logits/rejected": -1.957320213317871,
      "logps/chosen": -1.4138109683990479,
      "logps/rejected": -2.131588935852051,
      "loss": 1.508,
      "nll_loss": 1.4727998971939087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14138111472129822,
      "rewards/margins": 0.07177779823541641,
      "rewards/rejected": -0.21315890550613403,
      "step": 471
    },
    {
      "epoch": 1.3010010355540214,
      "grad_norm": 0.19494958221912384,
      "learning_rate": 3.4972809656276047e-06,
      "log_odds_chosen": 0.8046627640724182,
      "log_odds_ratio": -0.3767206370830536,
      "logits/chosen": -0.36885958909988403,
      "logits/rejected": -1.8168095350265503,
      "logps/chosen": -1.5026159286499023,
      "logps/rejected": -2.176860809326172,
      "loss": 1.6015,
      "nll_loss": 1.5638599395751953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15026158094406128,
      "rewards/margins": 0.06742450594902039,
      "rewards/rejected": -0.21768608689308167,
      "step": 472
    },
    {
      "epoch": 1.3037625129444252,
      "grad_norm": 0.1766563355922699,
      "learning_rate": 3.4899268588962613e-06,
      "log_odds_chosen": 0.8576854467391968,
      "log_odds_ratio": -0.3584885597229004,
      "logits/chosen": -0.436603844165802,
      "logits/rejected": -1.6729352474212646,
      "logps/chosen": -1.3645590543746948,
      "logps/rejected": -2.0580813884735107,
      "loss": 1.4773,
      "nll_loss": 1.4414961338043213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13645590841770172,
      "rewards/margins": 0.06935223937034607,
      "rewards/rejected": -0.20580816268920898,
      "step": 473
    },
    {
      "epoch": 1.3065239903348291,
      "grad_norm": 0.18817074596881866,
      "learning_rate": 3.4825625791348093e-06,
      "log_odds_chosen": 0.8350617289543152,
      "log_odds_ratio": -0.3708168864250183,
      "logits/chosen": -0.3747791051864624,
      "logits/rejected": -1.553459882736206,
      "logps/chosen": -1.4124037027359009,
      "logps/rejected": -2.1011507511138916,
      "loss": 1.5145,
      "nll_loss": 1.477430820465088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14124037325382233,
      "rewards/margins": 0.06887470930814743,
      "rewards/rejected": -0.21011507511138916,
      "step": 474
    },
    {
      "epoch": 1.309285467725233,
      "grad_norm": 0.1850864142179489,
      "learning_rate": 3.4751882020226174e-06,
      "log_odds_chosen": 0.7951568365097046,
      "log_odds_ratio": -0.38151904940605164,
      "logits/chosen": -0.41285592317581177,
      "logits/rejected": -1.676164984703064,
      "logps/chosen": -1.519218921661377,
      "logps/rejected": -2.190464973449707,
      "loss": 1.6106,
      "nll_loss": 1.5724141597747803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15192189812660217,
      "rewards/margins": 0.06712460517883301,
      "rewards/rejected": -0.21904650330543518,
      "step": 475
    },
    {
      "epoch": 1.312046945115637,
      "grad_norm": 0.19581526517868042,
      "learning_rate": 3.467803803342821e-06,
      "log_odds_chosen": 0.7770808339118958,
      "log_odds_ratio": -0.38217130303382874,
      "logits/chosen": -0.3372090756893158,
      "logits/rejected": -1.6060659885406494,
      "logps/chosen": -1.4900453090667725,
      "logps/rejected": -2.1375834941864014,
      "loss": 1.5763,
      "nll_loss": 1.5380480289459229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14900454878807068,
      "rewards/margins": 0.06475377827882767,
      "rewards/rejected": -0.21375833451747894,
      "step": 476
    },
    {
      "epoch": 1.3148084225060408,
      "grad_norm": 0.19793154299259186,
      "learning_rate": 3.4604094589815402e-06,
      "log_odds_chosen": 0.6940678358078003,
      "log_odds_ratio": -0.40892940759658813,
      "logits/chosen": -0.42878496646881104,
      "logits/rejected": -1.5839377641677856,
      "logps/chosen": -1.4118523597717285,
      "logps/rejected": -1.9752473831176758,
      "loss": 1.5164,
      "nll_loss": 1.4755195379257202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411852389574051,
      "rewards/margins": 0.056339483708143234,
      "rewards/rejected": -0.19752474129199982,
      "step": 477
    },
    {
      "epoch": 1.3175698998964447,
      "grad_norm": 0.18727529048919678,
      "learning_rate": 3.4530052449271044e-06,
      "log_odds_chosen": 0.8616414666175842,
      "log_odds_ratio": -0.36362677812576294,
      "logits/chosen": -0.4634791910648346,
      "logits/rejected": -1.5600810050964355,
      "logps/chosen": -1.4240953922271729,
      "logps/rejected": -2.1343352794647217,
      "loss": 1.521,
      "nll_loss": 1.4846872091293335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.142409548163414,
      "rewards/margins": 0.07102398574352264,
      "rewards/rejected": -0.21343351900577545,
      "step": 478
    },
    {
      "epoch": 1.3203313772868484,
      "grad_norm": 0.19879932701587677,
      "learning_rate": 3.4455912372692696e-06,
      "log_odds_chosen": 0.5470461845397949,
      "log_odds_ratio": -0.46493563055992126,
      "logits/chosen": -0.3689558207988739,
      "logits/rejected": -1.5988726615905762,
      "logps/chosen": -1.4872140884399414,
      "logps/rejected": -1.9357192516326904,
      "loss": 1.5782,
      "nll_loss": 1.5316959619522095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14872139692306519,
      "rewards/margins": 0.04485052451491356,
      "rewards/rejected": -0.19357194006443024,
      "step": 479
    },
    {
      "epoch": 1.3230928546772525,
      "grad_norm": 0.19459912180900574,
      "learning_rate": 3.438167512198436e-06,
      "log_odds_chosen": 0.5693470239639282,
      "log_odds_ratio": -0.4518805146217346,
      "logits/chosen": -0.41612759232521057,
      "logits/rejected": -1.5004498958587646,
      "logps/chosen": -1.5251268148422241,
      "logps/rejected": -1.994539499282837,
      "loss": 1.6246,
      "nll_loss": 1.5794237852096558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15251268446445465,
      "rewards/margins": 0.046941258013248444,
      "rewards/rejected": -0.1994539499282837,
      "step": 480
    },
    {
      "epoch": 1.3258543320676561,
      "grad_norm": 0.1880672574043274,
      "learning_rate": 3.4307341460048633e-06,
      "log_odds_chosen": 0.7143114805221558,
      "log_odds_ratio": -0.40364253520965576,
      "logits/chosen": -0.4024220108985901,
      "logits/rejected": -1.5502820014953613,
      "logps/chosen": -1.5507985353469849,
      "logps/rejected": -2.1468398571014404,
      "loss": 1.6399,
      "nll_loss": 1.5995802879333496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15507985651493073,
      "rewards/margins": 0.059604134410619736,
      "rewards/rejected": -0.21468399465084076,
      "step": 481
    },
    {
      "epoch": 1.32861580945806,
      "grad_norm": 0.19784514605998993,
      "learning_rate": 3.4232912150778914e-06,
      "log_odds_chosen": 0.7461116909980774,
      "log_odds_ratio": -0.3941587507724762,
      "logits/chosen": -0.43998983502388,
      "logits/rejected": -1.7154864072799683,
      "logps/chosen": -1.4538161754608154,
      "logps/rejected": -2.0666849613189697,
      "loss": 1.5731,
      "nll_loss": 1.5336874723434448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1453816145658493,
      "rewards/margins": 0.06128688156604767,
      "rewards/rejected": -0.20666849613189697,
      "step": 482
    },
    {
      "epoch": 1.331377286848464,
      "grad_norm": 0.18405503034591675,
      "learning_rate": 3.415838795905151e-06,
      "log_odds_chosen": 0.9300730228424072,
      "log_odds_ratio": -0.34047600626945496,
      "logits/chosen": -0.5350120067596436,
      "logits/rejected": -1.9728281497955322,
      "logps/chosen": -1.4228546619415283,
      "logps/rejected": -2.1867175102233887,
      "loss": 1.5168,
      "nll_loss": 1.4828009605407715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14228546619415283,
      "rewards/margins": 0.07638627290725708,
      "rewards/rejected": -0.2186717540025711,
      "step": 483
    },
    {
      "epoch": 1.3341387642388678,
      "grad_norm": 0.19047723710536957,
      "learning_rate": 3.408376965071779e-06,
      "log_odds_chosen": 0.6742444634437561,
      "log_odds_ratio": -0.41640961170196533,
      "logits/chosen": -0.37510067224502563,
      "logits/rejected": -1.5923813581466675,
      "logps/chosen": -1.5178049802780151,
      "logps/rejected": -2.0749661922454834,
      "loss": 1.6055,
      "nll_loss": 1.5638338327407837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15178050100803375,
      "rewards/margins": 0.05571611970663071,
      "rewards/rejected": -0.20749662816524506,
      "step": 484
    },
    {
      "epoch": 1.3369002416292717,
      "grad_norm": 0.1780618578195572,
      "learning_rate": 3.400905799259634e-06,
      "log_odds_chosen": 0.653221607208252,
      "log_odds_ratio": -0.4260196387767792,
      "logits/chosen": -0.42139092087745667,
      "logits/rejected": -1.648254632949829,
      "logps/chosen": -1.4374747276306152,
      "logps/rejected": -1.9686298370361328,
      "loss": 1.5329,
      "nll_loss": 1.490256667137146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.143747478723526,
      "rewards/margins": 0.05311552435159683,
      "rewards/rejected": -0.19686299562454224,
      "step": 485
    },
    {
      "epoch": 1.3396617190196756,
      "grad_norm": 0.1804792881011963,
      "learning_rate": 3.393425375246503e-06,
      "log_odds_chosen": 0.6369755268096924,
      "log_odds_ratio": -0.4262813925743103,
      "logits/chosen": -0.4290909469127655,
      "logits/rejected": -1.7290550470352173,
      "logps/chosen": -1.4642930030822754,
      "logps/rejected": -1.98381507396698,
      "loss": 1.5519,
      "nll_loss": 1.509236454963684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14642930030822754,
      "rewards/margins": 0.05195220559835434,
      "rewards/rejected": -0.1983814835548401,
      "step": 486
    },
    {
      "epoch": 1.3424231964100795,
      "grad_norm": 0.19493341445922852,
      "learning_rate": 3.3859357699053165e-06,
      "log_odds_chosen": 0.7284951210021973,
      "log_odds_ratio": -0.40332838892936707,
      "logits/chosen": -0.40109413862228394,
      "logits/rejected": -1.4352595806121826,
      "logps/chosen": -1.402358055114746,
      "logps/rejected": -1.9987090826034546,
      "loss": 1.4967,
      "nll_loss": 1.456404447555542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1402358114719391,
      "rewards/margins": 0.059635113924741745,
      "rewards/rejected": -0.19987091422080994,
      "step": 487
    },
    {
      "epoch": 1.3451846738004831,
      "grad_norm": 0.19050319492816925,
      "learning_rate": 3.3784370602033572e-06,
      "log_odds_chosen": 0.8729457259178162,
      "log_odds_ratio": -0.35851162672042847,
      "logits/chosen": -0.4902225732803345,
      "logits/rejected": -1.7542881965637207,
      "logps/chosen": -1.4271174669265747,
      "logps/rejected": -2.151813507080078,
      "loss": 1.5277,
      "nll_loss": 1.4917997121810913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14271175861358643,
      "rewards/margins": 0.07246959954500198,
      "rewards/rejected": -0.2151813805103302,
      "step": 488
    },
    {
      "epoch": 1.3479461511908872,
      "grad_norm": 0.18820421397686005,
      "learning_rate": 3.3709293232014705e-06,
      "log_odds_chosen": 0.73007732629776,
      "log_odds_ratio": -0.40136751532554626,
      "logits/chosen": -0.44850000739097595,
      "logits/rejected": -2.0052490234375,
      "logps/chosen": -1.4974483251571655,
      "logps/rejected": -2.1092348098754883,
      "loss": 1.5884,
      "nll_loss": 1.5482368469238281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14974482357501984,
      "rewards/margins": 0.06117865815758705,
      "rewards/rejected": -0.21092349290847778,
      "step": 489
    },
    {
      "epoch": 1.350707628581291,
      "grad_norm": 0.18799753487110138,
      "learning_rate": 3.3634126360532694e-06,
      "log_odds_chosen": 0.8482156991958618,
      "log_odds_ratio": -0.36712414026260376,
      "logits/chosen": -0.32226884365081787,
      "logits/rejected": -1.778417944908142,
      "logps/chosen": -1.577460765838623,
      "logps/rejected": -2.302645206451416,
      "loss": 1.6644,
      "nll_loss": 1.6277295351028442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1577460616827011,
      "rewards/margins": 0.07251846790313721,
      "rewards/rejected": -0.23026452958583832,
      "step": 490
    },
    {
      "epoch": 1.3534691059716948,
      "grad_norm": 0.20365531742572784,
      "learning_rate": 3.355887076004345e-06,
      "log_odds_chosen": 0.7709956765174866,
      "log_odds_ratio": -0.383152574300766,
      "logits/chosen": -0.5297287702560425,
      "logits/rejected": -1.6290040016174316,
      "logps/chosen": -1.4873484373092651,
      "logps/rejected": -2.12845778465271,
      "loss": 1.5899,
      "nll_loss": 1.551561713218689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14873485267162323,
      "rewards/margins": 0.06411094218492508,
      "rewards/rejected": -0.21284577250480652,
      "step": 491
    },
    {
      "epoch": 1.3562305833620987,
      "grad_norm": 0.18781210482120514,
      "learning_rate": 3.3483527203914694e-06,
      "log_odds_chosen": 0.8746954798698425,
      "log_odds_ratio": -0.3559949994087219,
      "logits/chosen": -0.3871724307537079,
      "logits/rejected": -1.6837961673736572,
      "logps/chosen": -1.5181317329406738,
      "logps/rejected": -2.254382610321045,
      "loss": 1.613,
      "nll_loss": 1.5773670673370361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15181316435337067,
      "rewards/margins": 0.0736251100897789,
      "rewards/rejected": -0.22543828189373016,
      "step": 492
    },
    {
      "epoch": 1.3589920607525026,
      "grad_norm": 0.18413974344730377,
      "learning_rate": 3.340809646641805e-06,
      "log_odds_chosen": 0.6966791749000549,
      "log_odds_ratio": -0.4073706269264221,
      "logits/chosen": -0.32745176553726196,
      "logits/rejected": -1.734323501586914,
      "logps/chosen": -1.5864430665969849,
      "logps/rejected": -2.1757755279541016,
      "loss": 1.6634,
      "nll_loss": 1.6226191520690918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15864431858062744,
      "rewards/margins": 0.05893322825431824,
      "rewards/rejected": -0.21757756173610687,
      "step": 493
    },
    {
      "epoch": 1.3617535381429065,
      "grad_norm": 0.1795225441455841,
      "learning_rate": 3.333257932272105e-06,
      "log_odds_chosen": 0.7916382551193237,
      "log_odds_ratio": -0.3854186236858368,
      "logits/chosen": -0.3654117286205292,
      "logits/rejected": -1.731317400932312,
      "logps/chosen": -1.4387598037719727,
      "logps/rejected": -2.0977869033813477,
      "loss": 1.5401,
      "nll_loss": 1.5015188455581665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14387598633766174,
      "rewards/margins": 0.0659027174115181,
      "rewards/rejected": -0.20977871119976044,
      "step": 494
    },
    {
      "epoch": 1.3645150155333103,
      "grad_norm": 0.19810736179351807,
      "learning_rate": 3.3256976548879183e-06,
      "log_odds_chosen": 0.7499178647994995,
      "log_odds_ratio": -0.391695499420166,
      "logits/chosen": -0.4586635231971741,
      "logits/rejected": -1.5185637474060059,
      "logps/chosen": -1.494145393371582,
      "logps/rejected": -2.1175270080566406,
      "loss": 1.591,
      "nll_loss": 1.551873803138733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1494145393371582,
      "rewards/margins": 0.06233816593885422,
      "rewards/rejected": -0.21175269782543182,
      "step": 495
    },
    {
      "epoch": 1.3672764929237142,
      "grad_norm": 0.17777174711227417,
      "learning_rate": 3.3181288921827925e-06,
      "log_odds_chosen": 0.8323229551315308,
      "log_odds_ratio": -0.37165793776512146,
      "logits/chosen": -0.3486481308937073,
      "logits/rejected": -1.4729896783828735,
      "logps/chosen": -1.3969337940216064,
      "logps/rejected": -2.0784289836883545,
      "loss": 1.5068,
      "nll_loss": 1.4696803092956543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13969337940216064,
      "rewards/margins": 0.06814949959516525,
      "rewards/rejected": -0.2078428864479065,
      "step": 496
    },
    {
      "epoch": 1.3700379703141181,
      "grad_norm": 0.18033576011657715,
      "learning_rate": 3.310551721937475e-06,
      "log_odds_chosen": 0.7983404994010925,
      "log_odds_ratio": -0.3745984435081482,
      "logits/chosen": -0.3823201656341553,
      "logits/rejected": -1.5644809007644653,
      "logps/chosen": -1.4277198314666748,
      "logps/rejected": -2.0840868949890137,
      "loss": 1.5007,
      "nll_loss": 1.4631915092468262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14277197420597076,
      "rewards/margins": 0.06563669443130493,
      "rewards/rejected": -0.2084086835384369,
      "step": 497
    },
    {
      "epoch": 1.372799447704522,
      "grad_norm": 0.19242247939109802,
      "learning_rate": 3.3029662220191146e-06,
      "log_odds_chosen": 0.8101353645324707,
      "log_odds_ratio": -0.37228965759277344,
      "logits/chosen": -0.4509270191192627,
      "logits/rejected": -1.8056501150131226,
      "logps/chosen": -1.3381811380386353,
      "logps/rejected": -1.9927046298980713,
      "loss": 1.4395,
      "nll_loss": 1.4023054838180542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.133818119764328,
      "rewards/margins": 0.06545236706733704,
      "rewards/rejected": -0.19927047193050385,
      "step": 498
    },
    {
      "epoch": 1.3755609250949257,
      "grad_norm": 0.1753472536802292,
      "learning_rate": 3.2953724703804572e-06,
      "log_odds_chosen": 0.892001211643219,
      "log_odds_ratio": -0.35052934288978577,
      "logits/chosen": -0.4044141173362732,
      "logits/rejected": -1.8937467336654663,
      "logps/chosen": -1.407699704170227,
      "logps/rejected": -2.1476714611053467,
      "loss": 1.4972,
      "nll_loss": 1.4621860980987549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14076997339725494,
      "rewards/margins": 0.07399718463420868,
      "rewards/rejected": -0.21476714313030243,
      "step": 499
    },
    {
      "epoch": 1.3783224024853298,
      "grad_norm": 0.1958237588405609,
      "learning_rate": 3.2877705450590525e-06,
      "log_odds_chosen": 0.7319251298904419,
      "log_odds_ratio": -0.396656334400177,
      "logits/chosen": -0.3846544325351715,
      "logits/rejected": -1.6542454957962036,
      "logps/chosen": -1.4686310291290283,
      "logps/rejected": -2.0715951919555664,
      "loss": 1.5948,
      "nll_loss": 1.5551836490631104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14686310291290283,
      "rewards/margins": 0.06029640883207321,
      "rewards/rejected": -0.20715951919555664,
      "step": 500
    },
    {
      "epoch": 1.3810838798757334,
      "grad_norm": 0.17545589804649353,
      "learning_rate": 3.2801605241764432e-06,
      "log_odds_chosen": 0.788885235786438,
      "log_odds_ratio": -0.3837747573852539,
      "logits/chosen": -0.3873632252216339,
      "logits/rejected": -1.8536286354064941,
      "logps/chosen": -1.4469109773635864,
      "logps/rejected": -2.0962677001953125,
      "loss": 1.5495,
      "nll_loss": 1.5111393928527832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1446911245584488,
      "rewards/margins": 0.06493565440177917,
      "rewards/rejected": -0.20962676405906677,
      "step": 501
    },
    {
      "epoch": 1.3838453572661373,
      "grad_norm": 0.1889375001192093,
      "learning_rate": 3.272542485937369e-06,
      "log_odds_chosen": 0.7263686656951904,
      "log_odds_ratio": -0.4058024287223816,
      "logits/chosen": -0.37034204602241516,
      "logits/rejected": -1.4220876693725586,
      "logps/chosen": -1.4947741031646729,
      "logps/rejected": -2.097472906112671,
      "loss": 1.594,
      "nll_loss": 1.553464651107788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14947742223739624,
      "rewards/margins": 0.06026986241340637,
      "rewards/rejected": -0.2097472846508026,
      "step": 502
    },
    {
      "epoch": 1.3866068346565412,
      "grad_norm": 0.1781754493713379,
      "learning_rate": 3.2649165086289597e-06,
      "log_odds_chosen": 0.7348980903625488,
      "log_odds_ratio": -0.3946687579154968,
      "logits/chosen": -0.4843774437904358,
      "logits/rejected": -1.973357081413269,
      "logps/chosen": -1.4816904067993164,
      "logps/rejected": -2.091848373413086,
      "loss": 1.5609,
      "nll_loss": 1.5214791297912598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14816902577877045,
      "rewards/margins": 0.06101579964160919,
      "rewards/rejected": -0.20918481051921844,
      "step": 503
    },
    {
      "epoch": 1.389368312046945,
      "grad_norm": 0.1925283521413803,
      "learning_rate": 3.2572826706199304e-06,
      "log_odds_chosen": 0.9842827916145325,
      "log_odds_ratio": -0.32510116696357727,
      "logits/chosen": -0.4544733464717865,
      "logits/rejected": -1.6470658779144287,
      "logps/chosen": -1.365575909614563,
      "logps/rejected": -2.175246238708496,
      "loss": 1.4834,
      "nll_loss": 1.4508426189422607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13655760884284973,
      "rewards/margins": 0.08096704632043839,
      "rewards/rejected": -0.21752463281154633,
      "step": 504
    },
    {
      "epoch": 1.392129789437349,
      "grad_norm": 0.18144164979457855,
      "learning_rate": 3.249641050359779e-06,
      "log_odds_chosen": 0.7838388085365295,
      "log_odds_ratio": -0.3822609782218933,
      "logits/chosen": -0.3912584185600281,
      "logits/rejected": -1.5951449871063232,
      "logps/chosen": -1.5115033388137817,
      "logps/rejected": -2.16564679145813,
      "loss": 1.5972,
      "nll_loss": 1.5589358806610107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15115033090114594,
      "rewards/margins": 0.06541436165571213,
      "rewards/rejected": -0.21656471490859985,
      "step": 505
    },
    {
      "epoch": 1.3948912668277529,
      "grad_norm": 0.1973390281200409,
      "learning_rate": 3.2419917263779765e-06,
      "log_odds_chosen": 0.6922333836555481,
      "log_odds_ratio": -0.4096466302871704,
      "logits/chosen": -0.45658183097839355,
      "logits/rejected": -1.6782792806625366,
      "logps/chosen": -1.4805402755737305,
      "logps/rejected": -2.0515551567077637,
      "loss": 1.5777,
      "nll_loss": 1.5367363691329956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14805401861667633,
      "rewards/margins": 0.05710148066282272,
      "rewards/rejected": -0.20515549182891846,
      "step": 506
    },
    {
      "epoch": 1.3976527442181568,
      "grad_norm": 0.17680558562278748,
      "learning_rate": 3.234334777283162e-06,
      "log_odds_chosen": 0.9198102355003357,
      "log_odds_ratio": -0.34391117095947266,
      "logits/chosen": -0.5111406445503235,
      "logits/rejected": -1.8106690645217896,
      "logps/chosen": -1.4262008666992188,
      "logps/rejected": -2.1901590824127197,
      "loss": 1.5103,
      "nll_loss": 1.4758594036102295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14262010157108307,
      "rewards/margins": 0.07639580965042114,
      "rewards/rejected": -0.21901589632034302,
      "step": 507
    },
    {
      "epoch": 1.4004142216085607,
      "grad_norm": 0.19250674545764923,
      "learning_rate": 3.2266702817623348e-06,
      "log_odds_chosen": 0.6659774780273438,
      "log_odds_ratio": -0.418069452047348,
      "logits/chosen": -0.42007410526275635,
      "logits/rejected": -1.4803179502487183,
      "logps/chosen": -1.502759575843811,
      "logps/rejected": -2.0552127361297607,
      "loss": 1.6133,
      "nll_loss": 1.5714445114135742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15027597546577454,
      "rewards/margins": 0.05524532124400139,
      "rewards/rejected": -0.20552130043506622,
      "step": 508
    },
    {
      "epoch": 1.4031756989989645,
      "grad_norm": 0.1782425045967102,
      "learning_rate": 3.218998318580043e-06,
      "log_odds_chosen": 0.9149441719055176,
      "log_odds_ratio": -0.34030070900917053,
      "logits/chosen": -0.46657681465148926,
      "logits/rejected": -1.7872267961502075,
      "logps/chosen": -1.4161348342895508,
      "logps/rejected": -2.172861099243164,
      "loss": 1.5157,
      "nll_loss": 1.4816545248031616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14161348342895508,
      "rewards/margins": 0.07567264139652252,
      "rewards/rejected": -0.2172861099243164,
      "step": 509
    },
    {
      "epoch": 1.4059371763893682,
      "grad_norm": 0.18463608622550964,
      "learning_rate": 3.2113189665775812e-06,
      "log_odds_chosen": 0.7678059339523315,
      "log_odds_ratio": -0.391030490398407,
      "logits/chosen": -0.3615942597389221,
      "logits/rejected": -1.817467451095581,
      "logps/chosen": -1.511760950088501,
      "logps/rejected": -2.154329299926758,
      "loss": 1.6013,
      "nll_loss": 1.5622117519378662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15117612481117249,
      "rewards/margins": 0.06425681710243225,
      "rewards/rejected": -0.21543294191360474,
      "step": 510
    },
    {
      "epoch": 1.408698653779772,
      "grad_norm": 0.18353889882564545,
      "learning_rate": 3.203632304672172e-06,
      "log_odds_chosen": 0.8818862438201904,
      "log_odds_ratio": -0.35136285424232483,
      "logits/chosen": -0.3583129942417145,
      "logits/rejected": -1.9124095439910889,
      "logps/chosen": -1.446837306022644,
      "logps/rejected": -2.177347183227539,
      "loss": 1.5495,
      "nll_loss": 1.5143190622329712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14468373358249664,
      "rewards/margins": 0.07305099815130234,
      "rewards/rejected": -0.21773472428321838,
      "step": 511
    },
    {
      "epoch": 1.411460131170176,
      "grad_norm": 0.175659641623497,
      "learning_rate": 3.1959384118561587e-06,
      "log_odds_chosen": 1.1884584426879883,
      "log_odds_ratio": -0.302362859249115,
      "logits/chosen": -0.30318501591682434,
      "logits/rejected": -1.7051109075546265,
      "logps/chosen": -1.3106746673583984,
      "logps/rejected": -2.27245831489563,
      "loss": 1.432,
      "nll_loss": 1.4017256498336792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13106746971607208,
      "rewards/margins": 0.09617836773395538,
      "rewards/rejected": -0.22724585235118866,
      "step": 512
    },
    {
      "epoch": 1.4142216085605799,
      "grad_norm": 0.18372024595737457,
      "learning_rate": 3.188237367196194e-06,
      "log_odds_chosen": 0.8523167371749878,
      "log_odds_ratio": -0.3591596186161041,
      "logits/chosen": -0.4926550090312958,
      "logits/rejected": -1.6857538223266602,
      "logps/chosen": -1.4602830410003662,
      "logps/rejected": -2.1705353260040283,
      "loss": 1.5673,
      "nll_loss": 1.5313630104064941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460283100605011,
      "rewards/margins": 0.07102521508932114,
      "rewards/rejected": -0.21705353260040283,
      "step": 513
    },
    {
      "epoch": 1.4169830859509838,
      "grad_norm": 0.19358229637145996,
      "learning_rate": 3.180529249832428e-06,
      "log_odds_chosen": 0.8924736976623535,
      "log_odds_ratio": -0.3460920751094818,
      "logits/chosen": -0.40075597167015076,
      "logits/rejected": -1.7352858781814575,
      "logps/chosen": -1.4665447473526,
      "logps/rejected": -2.211902141571045,
      "loss": 1.5629,
      "nll_loss": 1.528282880783081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14665447175502777,
      "rewards/margins": 0.07453573495149612,
      "rewards/rejected": -0.2211902141571045,
      "step": 514
    },
    {
      "epoch": 1.4197445633413877,
      "grad_norm": 0.18455541133880615,
      "learning_rate": 3.172814138977692e-06,
      "log_odds_chosen": 0.6485744118690491,
      "log_odds_ratio": -0.4242747128009796,
      "logits/chosen": -0.40753471851348877,
      "logits/rejected": -1.4140284061431885,
      "logps/chosen": -1.449316143989563,
      "logps/rejected": -1.9814183712005615,
      "loss": 1.5443,
      "nll_loss": 1.5018267631530762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1449316143989563,
      "rewards/margins": 0.05321022868156433,
      "rewards/rejected": -0.19814184308052063,
      "step": 515
    },
    {
      "epoch": 1.4225060407317915,
      "grad_norm": 0.19965727627277374,
      "learning_rate": 3.165092113916688e-06,
      "log_odds_chosen": 0.9415782690048218,
      "log_odds_ratio": -0.3400614857673645,
      "logits/chosen": -0.37811753153800964,
      "logits/rejected": -1.5702593326568604,
      "logps/chosen": -1.4428731203079224,
      "logps/rejected": -2.229719877243042,
      "loss": 1.5402,
      "nll_loss": 1.5062329769134521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1442873179912567,
      "rewards/margins": 0.07868468761444092,
      "rewards/rejected": -0.22297202050685883,
      "step": 516
    },
    {
      "epoch": 1.4252675181221954,
      "grad_norm": 0.2037183940410614,
      "learning_rate": 3.1573632540051702e-06,
      "log_odds_chosen": 0.8820104598999023,
      "log_odds_ratio": -0.35525357723236084,
      "logits/chosen": -0.3366633355617523,
      "logits/rejected": -1.3813198804855347,
      "logps/chosen": -1.4109978675842285,
      "logps/rejected": -2.1356987953186035,
      "loss": 1.5272,
      "nll_loss": 1.4916839599609375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14109979569911957,
      "rewards/margins": 0.07247008383274078,
      "rewards/rejected": -0.21356987953186035,
      "step": 517
    },
    {
      "epoch": 1.4280289955125993,
      "grad_norm": 0.18941880762577057,
      "learning_rate": 3.1496276386691327e-06,
      "log_odds_chosen": 0.6363694071769714,
      "log_odds_ratio": -0.4300464987754822,
      "logits/chosen": -0.34271934628486633,
      "logits/rejected": -1.6024858951568604,
      "logps/chosen": -1.5349292755126953,
      "logps/rejected": -2.0657947063446045,
      "loss": 1.6149,
      "nll_loss": 1.5718772411346436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15349292755126953,
      "rewards/margins": 0.053086526691913605,
      "rewards/rejected": -0.20657946169376373,
      "step": 518
    },
    {
      "epoch": 1.430790472903003,
      "grad_norm": 0.19241963326931,
      "learning_rate": 3.1418853474039913e-06,
      "log_odds_chosen": 0.9222308993339539,
      "log_odds_ratio": -0.3464682102203369,
      "logits/chosen": -0.46934691071510315,
      "logits/rejected": -1.862980604171753,
      "logps/chosen": -1.4347989559173584,
      "logps/rejected": -2.1988108158111572,
      "loss": 1.51,
      "nll_loss": 1.4753470420837402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14347989857196808,
      "rewards/margins": 0.07640118896961212,
      "rewards/rejected": -0.219881072640419,
      "step": 519
    },
    {
      "epoch": 1.433551950293407,
      "grad_norm": 0.19518575072288513,
      "learning_rate": 3.1341364597737684e-06,
      "log_odds_chosen": 0.7466192245483398,
      "log_odds_ratio": -0.3926132619380951,
      "logits/chosen": -0.3003273606300354,
      "logits/rejected": -1.4642976522445679,
      "logps/chosen": -1.4427096843719482,
      "logps/rejected": -2.0578322410583496,
      "loss": 1.5336,
      "nll_loss": 1.49433434009552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14427097141742706,
      "rewards/margins": 0.06151225417852402,
      "rewards/rejected": -0.20578324794769287,
      "step": 520
    },
    {
      "epoch": 1.4363134276838108,
      "grad_norm": 0.19285590946674347,
      "learning_rate": 3.126381055410274e-06,
      "log_odds_chosen": 0.8940811157226562,
      "log_odds_ratio": -0.34637144207954407,
      "logits/chosen": -0.4038392901420593,
      "logits/rejected": -1.4152494668960571,
      "logps/chosen": -1.4457933902740479,
      "logps/rejected": -2.187410593032837,
      "loss": 1.5584,
      "nll_loss": 1.523801326751709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14457935094833374,
      "rewards/margins": 0.07416172325611115,
      "rewards/rejected": -0.2187410593032837,
      "step": 521
    },
    {
      "epoch": 1.4390749050742146,
      "grad_norm": 0.19499003887176514,
      "learning_rate": 3.1186192140122863e-06,
      "log_odds_chosen": 0.6345283389091492,
      "log_odds_ratio": -0.43169543147087097,
      "logits/chosen": -0.39155280590057373,
      "logits/rejected": -1.2614598274230957,
      "logps/chosen": -1.4829864501953125,
      "logps/rejected": -2.0023984909057617,
      "loss": 1.5886,
      "nll_loss": 1.5454603433609009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14829865097999573,
      "rewards/margins": 0.05194120854139328,
      "rewards/rejected": -0.20023983716964722,
      "step": 522
    },
    {
      "epoch": 1.4418363824646185,
      "grad_norm": 0.18813124299049377,
      "learning_rate": 3.1108510153447352e-06,
      "log_odds_chosen": 0.6801859140396118,
      "log_odds_ratio": -0.4208618998527527,
      "logits/chosen": -0.4520847797393799,
      "logits/rejected": -1.5769951343536377,
      "logps/chosen": -1.4872287511825562,
      "logps/rejected": -2.0551376342773438,
      "loss": 1.5849,
      "nll_loss": 1.5428580045700073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14872288703918457,
      "rewards/margins": 0.05679089203476906,
      "rewards/rejected": -0.20551377534866333,
      "step": 523
    },
    {
      "epoch": 1.4445978598550224,
      "grad_norm": 0.1975877583026886,
      "learning_rate": 3.1030765392378813e-06,
      "log_odds_chosen": 0.9773280620574951,
      "log_odds_ratio": -0.321841299533844,
      "logits/chosen": -0.4134189486503601,
      "logits/rejected": -1.5019309520721436,
      "logps/chosen": -1.4216653108596802,
      "logps/rejected": -2.2362189292907715,
      "loss": 1.4987,
      "nll_loss": 1.4664944410324097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14216652512550354,
      "rewards/margins": 0.08145537227392197,
      "rewards/rejected": -0.2236218899488449,
      "step": 524
    },
    {
      "epoch": 1.4473593372454263,
      "grad_norm": 0.19157834351062775,
      "learning_rate": 3.0952958655864957e-06,
      "log_odds_chosen": 0.8533503413200378,
      "log_odds_ratio": -0.3656317889690399,
      "logits/chosen": -0.4119202494621277,
      "logits/rejected": -1.60611093044281,
      "logps/chosen": -1.4607584476470947,
      "logps/rejected": -2.171645402908325,
      "loss": 1.5549,
      "nll_loss": 1.5183472633361816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14607584476470947,
      "rewards/margins": 0.07108870148658752,
      "rewards/rejected": -0.2171645313501358,
      "step": 525
    },
    {
      "epoch": 1.4501208146358302,
      "grad_norm": 0.18692122399806976,
      "learning_rate": 3.0875090743490383e-06,
      "log_odds_chosen": 1.008112907409668,
      "log_odds_ratio": -0.31326454877853394,
      "logits/chosen": -0.32677963376045227,
      "logits/rejected": -1.9253056049346924,
      "logps/chosen": -1.4773204326629639,
      "logps/rejected": -2.3293113708496094,
      "loss": 1.5654,
      "nll_loss": 1.5340484380722046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14773203432559967,
      "rewards/margins": 0.08519911766052246,
      "rewards/rejected": -0.23293115198612213,
      "step": 526
    },
    {
      "epoch": 1.452882292026234,
      "grad_norm": 0.20491987466812134,
      "learning_rate": 3.0797162455468367e-06,
      "log_odds_chosen": 0.9280487298965454,
      "log_odds_ratio": -0.33601540327072144,
      "logits/chosen": -0.3736734688282013,
      "logits/rejected": -1.8332206010818481,
      "logps/chosen": -1.4430752992630005,
      "logps/rejected": -2.2161500453948975,
      "loss": 1.5345,
      "nll_loss": 1.5009381771087646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14430750906467438,
      "rewards/margins": 0.07730748504400253,
      "rewards/rejected": -0.2216150015592575,
      "step": 527
    },
    {
      "epoch": 1.455643769416638,
      "grad_norm": 0.19581927359104156,
      "learning_rate": 3.071917459263264e-06,
      "log_odds_chosen": 0.8374117016792297,
      "log_odds_ratio": -0.36380189657211304,
      "logits/chosen": -0.40068864822387695,
      "logits/rejected": -1.559683084487915,
      "logps/chosen": -1.479543685913086,
      "logps/rejected": -2.176377296447754,
      "loss": 1.5825,
      "nll_loss": 1.546140432357788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14795437455177307,
      "rewards/margins": 0.06968337297439575,
      "rewards/rejected": -0.21763774752616882,
      "step": 528
    },
    {
      "epoch": 1.4584052468070419,
      "grad_norm": 0.18251369893550873,
      "learning_rate": 3.0641127956429157e-06,
      "log_odds_chosen": 1.0565541982650757,
      "log_odds_ratio": -0.31225889921188354,
      "logits/chosen": -0.3141689896583557,
      "logits/rejected": -1.755129337310791,
      "logps/chosen": -1.4867286682128906,
      "logps/rejected": -2.3898322582244873,
      "loss": 1.5731,
      "nll_loss": 1.541866660118103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14867284893989563,
      "rewards/margins": 0.09031039476394653,
      "rewards/rejected": -0.23898322880268097,
      "step": 529
    },
    {
      "epoch": 1.4611667241974455,
      "grad_norm": 0.18326324224472046,
      "learning_rate": 3.056302334890786e-06,
      "log_odds_chosen": 0.9464606046676636,
      "log_odds_ratio": -0.3374951481819153,
      "logits/chosen": -0.4323669373989105,
      "logits/rejected": -1.7882404327392578,
      "logps/chosen": -1.4064451456069946,
      "logps/rejected": -2.193542003631592,
      "loss": 1.4906,
      "nll_loss": 1.4568413496017456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14064452052116394,
      "rewards/margins": 0.0787096917629242,
      "rewards/rejected": -0.21935419738292694,
      "step": 530
    },
    {
      "epoch": 1.4639282015878494,
      "grad_norm": 0.18796159327030182,
      "learning_rate": 3.0484861572714446e-06,
      "log_odds_chosen": 1.0237690210342407,
      "log_odds_ratio": -0.30970078706741333,
      "logits/chosen": -0.4086054563522339,
      "logits/rejected": -1.8015973567962646,
      "logps/chosen": -1.4367576837539673,
      "logps/rejected": -2.2936997413635254,
      "loss": 1.5186,
      "nll_loss": 1.4875980615615845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1436757743358612,
      "rewards/margins": 0.08569422364234924,
      "rewards/rejected": -0.22937001287937164,
      "step": 531
    },
    {
      "epoch": 1.4666896789782533,
      "grad_norm": 0.22190004587173462,
      "learning_rate": 3.0406643431082088e-06,
      "log_odds_chosen": 1.0039647817611694,
      "log_odds_ratio": -0.3233025372028351,
      "logits/chosen": -0.4348459243774414,
      "logits/rejected": -1.7563586235046387,
      "logps/chosen": -1.418906331062317,
      "logps/rejected": -2.259153366088867,
      "loss": 1.5147,
      "nll_loss": 1.4823333024978638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14189063012599945,
      "rewards/margins": 0.08402469754219055,
      "rewards/rejected": -0.2259153425693512,
      "step": 532
    },
    {
      "epoch": 1.4694511563686572,
      "grad_norm": 0.19412145018577576,
      "learning_rate": 3.0328369727823216e-06,
      "log_odds_chosen": 0.7487189173698425,
      "log_odds_ratio": -0.3932346701622009,
      "logits/chosen": -0.3478846549987793,
      "logits/rejected": -1.6251699924468994,
      "logps/chosen": -1.4815884828567505,
      "logps/rejected": -2.1053826808929443,
      "loss": 1.5689,
      "nll_loss": 1.5295873880386353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14815884828567505,
      "rewards/margins": 0.06237940490245819,
      "rewards/rejected": -0.21053825318813324,
      "step": 533
    },
    {
      "epoch": 1.472212633759061,
      "grad_norm": 0.18353740870952606,
      "learning_rate": 3.0250041267321234e-06,
      "log_odds_chosen": 0.847536563873291,
      "log_odds_ratio": -0.360176146030426,
      "logits/chosen": -0.4132736623287201,
      "logits/rejected": -1.6758484840393066,
      "logps/chosen": -1.3828142881393433,
      "logps/rejected": -2.0753824710845947,
      "loss": 1.492,
      "nll_loss": 1.4560058116912842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1382814347743988,
      "rewards/margins": 0.06925684213638306,
      "rewards/rejected": -0.20753827691078186,
      "step": 534
    },
    {
      "epoch": 1.474974111149465,
      "grad_norm": 0.19951318204402924,
      "learning_rate": 3.0171658854522274e-06,
      "log_odds_chosen": 0.9957120418548584,
      "log_odds_ratio": -0.3261812925338745,
      "logits/chosen": -0.435792475938797,
      "logits/rejected": -1.6330525875091553,
      "logps/chosen": -1.4285228252410889,
      "logps/rejected": -2.266305446624756,
      "loss": 1.516,
      "nll_loss": 1.4834091663360596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1428522914648056,
      "rewards/margins": 0.0837782695889473,
      "rewards/rejected": -0.2266305685043335,
      "step": 535
    },
    {
      "epoch": 1.4777355885398689,
      "grad_norm": 0.1980207860469818,
      "learning_rate": 3.009322329492689e-06,
      "log_odds_chosen": 0.665631890296936,
      "log_odds_ratio": -0.41715607047080994,
      "logits/chosen": -0.46294739842414856,
      "logits/rejected": -1.6968400478363037,
      "logps/chosen": -1.5159837007522583,
      "logps/rejected": -2.067196846008301,
      "loss": 1.5875,
      "nll_loss": 1.5457911491394043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15159837901592255,
      "rewards/margins": 0.05512130260467529,
      "rewards/rejected": -0.20671968162059784,
      "step": 536
    },
    {
      "epoch": 1.4804970659302727,
      "grad_norm": 0.1900826096534729,
      "learning_rate": 3.0014735394581824e-06,
      "log_odds_chosen": 0.747369110584259,
      "log_odds_ratio": -0.3997749388217926,
      "logits/chosen": -0.4459250867366791,
      "logits/rejected": -1.5584537982940674,
      "logps/chosen": -1.4036555290222168,
      "logps/rejected": -2.0093021392822266,
      "loss": 1.5147,
      "nll_loss": 1.4746928215026855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14036555588245392,
      "rewards/margins": 0.06056464836001396,
      "rewards/rejected": -0.20093020796775818,
      "step": 537
    },
    {
      "epoch": 1.4832585433206766,
      "grad_norm": 0.18820805847644806,
      "learning_rate": 2.993619596007168e-06,
      "log_odds_chosen": 0.852586567401886,
      "log_odds_ratio": -0.36225834488868713,
      "logits/chosen": -0.37023186683654785,
      "logits/rejected": -1.8086068630218506,
      "logps/chosen": -1.5031466484069824,
      "logps/rejected": -2.218027114868164,
      "loss": 1.5854,
      "nll_loss": 1.5491464138031006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15031468868255615,
      "rewards/margins": 0.07148806005716324,
      "rewards/rejected": -0.2218027263879776,
      "step": 538
    },
    {
      "epoch": 1.4860200207110803,
      "grad_norm": 0.18898199498653412,
      "learning_rate": 2.985760579851068e-06,
      "log_odds_chosen": 1.0741615295410156,
      "log_odds_ratio": -0.3005737364292145,
      "logits/chosen": -0.3823893070220947,
      "logits/rejected": -1.7826778888702393,
      "logps/chosen": -1.5318934917449951,
      "logps/rejected": -2.454892158508301,
      "loss": 1.6065,
      "nll_loss": 1.5764073133468628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15318936109542847,
      "rewards/margins": 0.09229984879493713,
      "rewards/rejected": -0.2454892247915268,
      "step": 539
    },
    {
      "epoch": 1.4887814981014844,
      "grad_norm": 0.18205980956554413,
      "learning_rate": 2.9778965717534314e-06,
      "log_odds_chosen": 0.8840954303741455,
      "log_odds_ratio": -0.36301591992378235,
      "logits/chosen": -0.48036321997642517,
      "logits/rejected": -1.776585340499878,
      "logps/chosen": -1.3918758630752563,
      "logps/rejected": -2.1138408184051514,
      "loss": 1.4757,
      "nll_loss": 1.4393746852874756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13918758928775787,
      "rewards/margins": 0.07219650596380234,
      "rewards/rejected": -0.21138408780097961,
      "step": 540
    },
    {
      "epoch": 1.491542975491888,
      "grad_norm": 0.17650777101516724,
      "learning_rate": 2.9700276525291096e-06,
      "log_odds_chosen": 1.0778427124023438,
      "log_odds_ratio": -0.30152907967567444,
      "logits/chosen": -0.4420645833015442,
      "logits/rejected": -1.7818214893341064,
      "logps/chosen": -1.3953425884246826,
      "logps/rejected": -2.2949678897857666,
      "loss": 1.4933,
      "nll_loss": 1.4631969928741455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13953426480293274,
      "rewards/margins": 0.08996252715587616,
      "rewards/rejected": -0.2294968068599701,
      "step": 541
    },
    {
      "epoch": 1.494304452882292,
      "grad_norm": 0.18586601316928864,
      "learning_rate": 2.9621539030434223e-06,
      "log_odds_chosen": 1.0328391790390015,
      "log_odds_ratio": -0.3124905228614807,
      "logits/chosen": -0.5735373497009277,
      "logits/rejected": -1.8510777950286865,
      "logps/chosen": -1.4249876737594604,
      "logps/rejected": -2.2932486534118652,
      "loss": 1.5165,
      "nll_loss": 1.4852372407913208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14249876141548157,
      "rewards/margins": 0.08682610094547272,
      "rewards/rejected": -0.22932487726211548,
      "step": 542
    },
    {
      "epoch": 1.4970659302726959,
      "grad_norm": 0.18669597804546356,
      "learning_rate": 2.954275404211328e-06,
      "log_odds_chosen": 0.8634792566299438,
      "log_odds_ratio": -0.36653023958206177,
      "logits/chosen": -0.3841334581375122,
      "logits/rejected": -1.634179711341858,
      "logps/chosen": -1.3906664848327637,
      "logps/rejected": -2.0937647819519043,
      "loss": 1.4983,
      "nll_loss": 1.4616230726242065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1390666514635086,
      "rewards/margins": 0.0703098252415657,
      "rewards/rejected": -0.2093764841556549,
      "step": 543
    },
    {
      "epoch": 1.4998274076630997,
      "grad_norm": 0.19772256910800934,
      "learning_rate": 2.946392236996592e-06,
      "log_odds_chosen": 1.0479884147644043,
      "log_odds_ratio": -0.31629127264022827,
      "logits/chosen": -0.41537678241729736,
      "logits/rejected": -1.7293639183044434,
      "logps/chosen": -1.4252593517303467,
      "logps/rejected": -2.3072445392608643,
      "loss": 1.5216,
      "nll_loss": 1.4899686574935913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14252594113349915,
      "rewards/margins": 0.08819850534200668,
      "rewards/rejected": -0.23072443902492523,
      "step": 544
    },
    {
      "epoch": 1.5025888850535036,
      "grad_norm": 0.20116113126277924,
      "learning_rate": 2.9385044824109544e-06,
      "log_odds_chosen": 0.9986646175384521,
      "log_odds_ratio": -0.32834985852241516,
      "logits/chosen": -0.39595213532447815,
      "logits/rejected": -1.64156973361969,
      "logps/chosen": -1.4510791301727295,
      "logps/rejected": -2.2933764457702637,
      "loss": 1.5398,
      "nll_loss": 1.5069797039031982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1451079100370407,
      "rewards/margins": 0.0842297375202179,
      "rewards/rejected": -0.2293376624584198,
      "step": 545
    },
    {
      "epoch": 1.5053503624439075,
      "grad_norm": 0.1882292777299881,
      "learning_rate": 2.9306122215132974e-06,
      "log_odds_chosen": 0.9424600601196289,
      "log_odds_ratio": -0.33976250886917114,
      "logits/chosen": -0.3478313982486725,
      "logits/rejected": -1.7016404867172241,
      "logps/chosen": -1.4425792694091797,
      "logps/rejected": -2.232656478881836,
      "loss": 1.5428,
      "nll_loss": 1.5088294744491577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14425793290138245,
      "rewards/margins": 0.07900771498680115,
      "rewards/rejected": -0.2232656478881836,
      "step": 546
    },
    {
      "epoch": 1.5081118398343114,
      "grad_norm": 0.2000649869441986,
      "learning_rate": 2.9227155354088134e-06,
      "log_odds_chosen": 0.8382570743560791,
      "log_odds_ratio": -0.3624340891838074,
      "logits/chosen": -0.5286014080047607,
      "logits/rejected": -1.5262038707733154,
      "logps/chosen": -1.4778207540512085,
      "logps/rejected": -2.178725004196167,
      "loss": 1.5501,
      "nll_loss": 1.5138760805130005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477820873260498,
      "rewards/margins": 0.0700904130935669,
      "rewards/rejected": -0.2178725004196167,
      "step": 547
    },
    {
      "epoch": 1.510873317224715,
      "grad_norm": 0.19744566082954407,
      "learning_rate": 2.91481450524817e-06,
      "log_odds_chosen": 1.2025346755981445,
      "log_odds_ratio": -0.2766777575016022,
      "logits/chosen": -0.4703354239463806,
      "logits/rejected": -1.5989863872528076,
      "logps/chosen": -1.4106305837631226,
      "logps/rejected": -2.429147243499756,
      "loss": 1.4928,
      "nll_loss": 1.4651353359222412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14106306433677673,
      "rewards/margins": 0.10185164213180542,
      "rewards/rejected": -0.24291470646858215,
      "step": 548
    },
    {
      "epoch": 1.5136347946151192,
      "grad_norm": 0.18259872496128082,
      "learning_rate": 2.9069092122266758e-06,
      "log_odds_chosen": 1.020290732383728,
      "log_odds_ratio": -0.31102436780929565,
      "logits/chosen": -0.34419721364974976,
      "logits/rejected": -1.7346612215042114,
      "logps/chosen": -1.4083847999572754,
      "logps/rejected": -2.256067991256714,
      "loss": 1.4963,
      "nll_loss": 1.4652248620986938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14083848893642426,
      "rewards/margins": 0.08476831018924713,
      "rewards/rejected": -0.22560681402683258,
      "step": 549
    },
    {
      "epoch": 1.5163962720055228,
      "grad_norm": 0.18138575553894043,
      "learning_rate": 2.8989997375834485e-06,
      "log_odds_chosen": 1.027559518814087,
      "log_odds_ratio": -0.3187801241874695,
      "logits/chosen": -0.43066591024398804,
      "logits/rejected": -1.62416410446167,
      "logps/chosen": -1.412578821182251,
      "logps/rejected": -2.271681070327759,
      "loss": 1.5043,
      "nll_loss": 1.4724640846252441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412578821182251,
      "rewards/margins": 0.08591020107269287,
      "rewards/rejected": -0.22716809809207916,
      "step": 550
    },
    {
      "epoch": 1.519157749395927,
      "grad_norm": 0.18563157320022583,
      "learning_rate": 2.8910861626005774e-06,
      "log_odds_chosen": 0.9574222564697266,
      "log_odds_ratio": -0.3370060920715332,
      "logits/chosen": -0.4931245446205139,
      "logits/rejected": -1.7520966529846191,
      "logps/chosen": -1.4311273097991943,
      "logps/rejected": -2.229990243911743,
      "loss": 1.5245,
      "nll_loss": 1.4907807111740112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14311271905899048,
      "rewards/margins": 0.07988631725311279,
      "rewards/rejected": -0.22299903631210327,
      "step": 551
    },
    {
      "epoch": 1.5219192267863306,
      "grad_norm": 0.20305076241493225,
      "learning_rate": 2.8831685686022897e-06,
      "log_odds_chosen": 0.9472601413726807,
      "log_odds_ratio": -0.3367205262184143,
      "logits/chosen": -0.3746846616268158,
      "logits/rejected": -1.5865817070007324,
      "logps/chosen": -1.46114182472229,
      "logps/rejected": -2.255194664001465,
      "loss": 1.5544,
      "nll_loss": 1.5207619667053223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14611420035362244,
      "rewards/margins": 0.0794052854180336,
      "rewards/rejected": -0.22551947832107544,
      "step": 552
    },
    {
      "epoch": 1.5246807041767345,
      "grad_norm": 0.20478032529354095,
      "learning_rate": 2.8752470369541152e-06,
      "log_odds_chosen": 0.9435425400733948,
      "log_odds_ratio": -0.341013640165329,
      "logits/chosen": -0.47928714752197266,
      "logits/rejected": -1.607470154762268,
      "logps/chosen": -1.493147850036621,
      "logps/rejected": -2.2932393550872803,
      "loss": 1.585,
      "nll_loss": 1.550898790359497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14931480586528778,
      "rewards/margins": 0.08000914007425308,
      "rewards/rejected": -0.22932393848896027,
      "step": 553
    },
    {
      "epoch": 1.5274421815671384,
      "grad_norm": 0.18675510585308075,
      "learning_rate": 2.8673216490620453e-06,
      "log_odds_chosen": 0.8560038208961487,
      "log_odds_ratio": -0.3590989112854004,
      "logits/chosen": -0.3732084035873413,
      "logits/rejected": -1.5269240140914917,
      "logps/chosen": -1.4434525966644287,
      "logps/rejected": -2.1552090644836426,
      "loss": 1.5403,
      "nll_loss": 1.5043420791625977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1443452537059784,
      "rewards/margins": 0.07117564976215363,
      "rewards/rejected": -0.21552088856697083,
      "step": 554
    },
    {
      "epoch": 1.5302036589575423,
      "grad_norm": 0.19521136581897736,
      "learning_rate": 2.859392486371705e-06,
      "log_odds_chosen": 0.9180530905723572,
      "log_odds_ratio": -0.3567126989364624,
      "logits/chosen": -0.49841001629829407,
      "logits/rejected": -1.4869251251220703,
      "logps/chosen": -1.4458491802215576,
      "logps/rejected": -2.2207460403442383,
      "loss": 1.5457,
      "nll_loss": 1.5100224018096924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14458490908145905,
      "rewards/margins": 0.07748968154191971,
      "rewards/rejected": -0.22207459807395935,
      "step": 555
    },
    {
      "epoch": 1.5329651363479462,
      "grad_norm": 0.20385955274105072,
      "learning_rate": 2.8514596303675073e-06,
      "log_odds_chosen": 1.151139497756958,
      "log_odds_ratio": -0.2817905843257904,
      "logits/chosen": -0.43373167514801025,
      "logits/rejected": -1.7193479537963867,
      "logps/chosen": -1.4189940690994263,
      "logps/rejected": -2.3919098377227783,
      "loss": 1.5087,
      "nll_loss": 1.4805082082748413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14189940690994263,
      "rewards/margins": 0.09729158133268356,
      "rewards/rejected": -0.2391909807920456,
      "step": 556
    },
    {
      "epoch": 1.53572661373835,
      "grad_norm": 0.19316573441028595,
      "learning_rate": 2.8435231625718242e-06,
      "log_odds_chosen": 1.0120395421981812,
      "log_odds_ratio": -0.3164248764514923,
      "logits/chosen": -0.38262811303138733,
      "logits/rejected": -1.806386113166809,
      "logps/chosen": -1.4623085260391235,
      "logps/rejected": -2.3146116733551025,
      "loss": 1.5372,
      "nll_loss": 1.5055807828903198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14623086154460907,
      "rewards/margins": 0.08523032069206238,
      "rewards/rejected": -0.23146119713783264,
      "step": 557
    },
    {
      "epoch": 1.538488091128754,
      "grad_norm": 2.1178505420684814,
      "learning_rate": 2.835583164544139e-06,
      "log_odds_chosen": 0.9513680934906006,
      "log_odds_ratio": -0.33420196175575256,
      "logits/chosen": -0.33462485671043396,
      "logits/rejected": -1.6407803297042847,
      "logps/chosen": -1.4331406354904175,
      "logps/rejected": -2.228959798812866,
      "loss": 1.5297,
      "nll_loss": 1.4962306022644043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14331406354904175,
      "rewards/margins": 0.07958193868398666,
      "rewards/rejected": -0.22289599478244781,
      "step": 558
    },
    {
      "epoch": 1.5412495685191576,
      "grad_norm": 0.19349446892738342,
      "learning_rate": 2.827639717880218e-06,
      "log_odds_chosen": 0.8153694272041321,
      "log_odds_ratio": -0.3702642023563385,
      "logits/chosen": -0.41866424679756165,
      "logits/rejected": -1.768775463104248,
      "logps/chosen": -1.4800782203674316,
      "logps/rejected": -2.1566810607910156,
      "loss": 1.552,
      "nll_loss": 1.5150169134140015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1480078101158142,
      "rewards/margins": 0.06766027957201004,
      "rewards/rejected": -0.21566811203956604,
      "step": 559
    },
    {
      "epoch": 1.5440110459095617,
      "grad_norm": 0.20243015885353088,
      "learning_rate": 2.8196929042112652e-06,
      "log_odds_chosen": 0.7393404841423035,
      "log_odds_ratio": -0.3941328823566437,
      "logits/chosen": -0.3861313462257385,
      "logits/rejected": -1.5598360300064087,
      "logps/chosen": -1.514575481414795,
      "logps/rejected": -2.1315693855285645,
      "loss": 1.6067,
      "nll_loss": 1.5672829151153564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1514575481414795,
      "rewards/margins": 0.06169939786195755,
      "rewards/rejected": -0.21315695345401764,
      "step": 560
    },
    {
      "epoch": 1.5467725232999654,
      "grad_norm": 0.18742065131664276,
      "learning_rate": 2.811742805203087e-06,
      "log_odds_chosen": 1.0400229692459106,
      "log_odds_ratio": -0.3049144148826599,
      "logits/chosen": -0.44754305481910706,
      "logits/rejected": -1.7995396852493286,
      "logps/chosen": -1.401616096496582,
      "logps/rejected": -2.268768548965454,
      "loss": 1.4907,
      "nll_loss": 1.4601788520812988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14016160368919373,
      "rewards/margins": 0.0867152288556099,
      "rewards/rejected": -0.22687683999538422,
      "step": 561
    },
    {
      "epoch": 1.5495340006903695,
      "grad_norm": 0.1959799826145172,
      "learning_rate": 2.8037895025552513e-06,
      "log_odds_chosen": 0.782373309135437,
      "log_odds_ratio": -0.38005301356315613,
      "logits/chosen": -0.3823511302471161,
      "logits/rejected": -1.534692645072937,
      "logps/chosen": -1.5261653661727905,
      "logps/rejected": -2.181563377380371,
      "loss": 1.6025,
      "nll_loss": 1.5644484758377075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15261651575565338,
      "rewards/margins": 0.06553982943296432,
      "rewards/rejected": -0.2181563377380371,
      "step": 562
    },
    {
      "epoch": 1.5522954780807732,
      "grad_norm": 0.18624603748321533,
      "learning_rate": 2.7958330780002472e-06,
      "log_odds_chosen": 1.0219098329544067,
      "log_odds_ratio": -0.31670254468917847,
      "logits/chosen": -0.48139262199401855,
      "logits/rejected": -1.540400505065918,
      "logps/chosen": -1.4481538534164429,
      "logps/rejected": -2.306190013885498,
      "loss": 1.5315,
      "nll_loss": 1.4998620748519897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14481540024280548,
      "rewards/margins": 0.08580362796783447,
      "rewards/rejected": -0.23061901330947876,
      "step": 563
    },
    {
      "epoch": 1.555056955471177,
      "grad_norm": 0.21115663647651672,
      "learning_rate": 2.787873613302649e-06,
      "log_odds_chosen": 0.8930285573005676,
      "log_odds_ratio": -0.34711483120918274,
      "logits/chosen": -0.3769160807132721,
      "logits/rejected": -1.5557960271835327,
      "logps/chosen": -1.5821139812469482,
      "logps/rejected": -2.338306427001953,
      "loss": 1.6636,
      "nll_loss": 1.628933310508728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15821139514446259,
      "rewards/margins": 0.07561925053596497,
      "rewards/rejected": -0.23383064568042755,
      "step": 564
    },
    {
      "epoch": 1.557818432861581,
      "grad_norm": 0.19150473177433014,
      "learning_rate": 2.7799111902582697e-06,
      "log_odds_chosen": 1.0769083499908447,
      "log_odds_ratio": -0.29914119839668274,
      "logits/chosen": -0.4121115207672119,
      "logits/rejected": -1.5894091129302979,
      "logps/chosen": -1.4135100841522217,
      "logps/rejected": -2.3172075748443604,
      "loss": 1.4981,
      "nll_loss": 1.4682036638259888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14135099947452545,
      "rewards/margins": 0.09036976844072342,
      "rewards/rejected": -0.23172077536582947,
      "step": 565
    },
    {
      "epoch": 1.5605799102519848,
      "grad_norm": 0.20271562039852142,
      "learning_rate": 2.7719458906933277e-06,
      "log_odds_chosen": 0.9160252809524536,
      "log_odds_ratio": -0.34139731526374817,
      "logits/chosen": -0.4353451728820801,
      "logits/rejected": -1.7503840923309326,
      "logps/chosen": -1.4142577648162842,
      "logps/rejected": -2.1746983528137207,
      "loss": 1.5083,
      "nll_loss": 1.4741257429122925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14142578840255737,
      "rewards/margins": 0.07604406028985977,
      "rewards/rejected": -0.21746984124183655,
      "step": 566
    },
    {
      "epoch": 1.5633413876423887,
      "grad_norm": 0.18848182260990143,
      "learning_rate": 2.763977796463599e-06,
      "log_odds_chosen": 0.9602777361869812,
      "log_odds_ratio": -0.32593464851379395,
      "logits/chosen": -0.49435821175575256,
      "logits/rejected": -1.9882686138153076,
      "logps/chosen": -1.4518954753875732,
      "logps/rejected": -2.2552900314331055,
      "loss": 1.5211,
      "nll_loss": 1.488489031791687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.145189568400383,
      "rewards/margins": 0.08033942431211472,
      "rewards/rejected": -0.22552898526191711,
      "step": 567
    },
    {
      "epoch": 1.5661028650327924,
      "grad_norm": 0.21147844195365906,
      "learning_rate": 2.7560069894535783e-06,
      "log_odds_chosen": 0.8455137014389038,
      "log_odds_ratio": -0.3622473478317261,
      "logits/chosen": -0.3838009536266327,
      "logits/rejected": -1.5061184167861938,
      "logps/chosen": -1.4856162071228027,
      "logps/rejected": -2.1930487155914307,
      "loss": 1.5775,
      "nll_loss": 1.5413126945495605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14856162667274475,
      "rewards/margins": 0.07074324786663055,
      "rewards/rejected": -0.2193048745393753,
      "step": 568
    },
    {
      "epoch": 1.5688643424231965,
      "grad_norm": 0.2110675871372223,
      "learning_rate": 2.748033551575644e-06,
      "log_odds_chosen": 0.7732433676719666,
      "log_odds_ratio": -0.3829563856124878,
      "logits/chosen": -0.5417189598083496,
      "logits/rejected": -1.4143304824829102,
      "logps/chosen": -1.4991943836212158,
      "logps/rejected": -2.145965099334717,
      "loss": 1.5934,
      "nll_loss": 1.555059552192688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14991945028305054,
      "rewards/margins": 0.06467706710100174,
      "rewards/rejected": -0.21459650993347168,
      "step": 569
    },
    {
      "epoch": 1.5716258198136002,
      "grad_norm": 0.20080991089344025,
      "learning_rate": 2.7400575647692046e-06,
      "log_odds_chosen": 0.8514761924743652,
      "log_odds_ratio": -0.3591695725917816,
      "logits/chosen": -0.469234824180603,
      "logits/rejected": -1.8506807088851929,
      "logps/chosen": -1.5175936222076416,
      "logps/rejected": -2.235103130340576,
      "loss": 1.607,
      "nll_loss": 1.5710902214050293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15175937116146088,
      "rewards/margins": 0.0717509537935257,
      "rewards/rejected": -0.22351032495498657,
      "step": 570
    },
    {
      "epoch": 1.5743872972040043,
      "grad_norm": 0.19513045251369476,
      "learning_rate": 2.7320791109998655e-06,
      "log_odds_chosen": 0.9315704107284546,
      "log_odds_ratio": -0.3407260477542877,
      "logits/chosen": -0.620419442653656,
      "logits/rejected": -1.8977861404418945,
      "logps/chosen": -1.4148740768432617,
      "logps/rejected": -2.1817188262939453,
      "loss": 1.495,
      "nll_loss": 1.4609739780426025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14148743450641632,
      "rewards/margins": 0.07668447494506836,
      "rewards/rejected": -0.2181718945503235,
      "step": 571
    },
    {
      "epoch": 1.577148774594408,
      "grad_norm": 0.19687050580978394,
      "learning_rate": 2.724098272258584e-06,
      "log_odds_chosen": 0.9079073667526245,
      "log_odds_ratio": -0.34860873222351074,
      "logits/chosen": -0.48484548926353455,
      "logits/rejected": -1.6853313446044922,
      "logps/chosen": -1.4053500890731812,
      "logps/rejected": -2.1579606533050537,
      "loss": 1.4975,
      "nll_loss": 1.4626667499542236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14053499698638916,
      "rewards/margins": 0.07526105642318726,
      "rewards/rejected": -0.21579605340957642,
      "step": 572
    },
    {
      "epoch": 1.5799102519848118,
      "grad_norm": 0.1954527497291565,
      "learning_rate": 2.7161151305608258e-06,
      "log_odds_chosen": 0.9750263690948486,
      "log_odds_ratio": -0.3271198272705078,
      "logits/chosen": -0.51035475730896,
      "logits/rejected": -1.5407557487487793,
      "logps/chosen": -1.3590997457504272,
      "logps/rejected": -2.162524461746216,
      "loss": 1.4519,
      "nll_loss": 1.4191510677337646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13590997457504272,
      "rewards/margins": 0.08034246414899826,
      "rewards/rejected": -0.2162524312734604,
      "step": 573
    },
    {
      "epoch": 1.5826717293752157,
      "grad_norm": 0.19047684967517853,
      "learning_rate": 2.7081297679457238e-06,
      "log_odds_chosen": 0.9220460653305054,
      "log_odds_ratio": -0.3471813499927521,
      "logits/chosen": -0.5562020540237427,
      "logits/rejected": -1.592578411102295,
      "logps/chosen": -1.3342963457107544,
      "logps/rejected": -2.0851826667785645,
      "loss": 1.4261,
      "nll_loss": 1.3913562297821045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1334296315908432,
      "rewards/margins": 0.07508864253759384,
      "rewards/rejected": -0.20851828157901764,
      "step": 574
    },
    {
      "epoch": 1.5854332067656196,
      "grad_norm": 0.1876901239156723,
      "learning_rate": 2.7001422664752338e-06,
      "log_odds_chosen": 0.8865346908569336,
      "log_odds_ratio": -0.353354275226593,
      "logits/chosen": -0.46136918663978577,
      "logits/rejected": -1.9794909954071045,
      "logps/chosen": -1.4452863931655884,
      "logps/rejected": -2.18290376663208,
      "loss": 1.5254,
      "nll_loss": 1.4900963306427002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14452862739562988,
      "rewards/margins": 0.07376174628734589,
      "rewards/rejected": -0.21829037368297577,
      "step": 575
    },
    {
      "epoch": 1.5881946841560235,
      "grad_norm": 0.21001578867435455,
      "learning_rate": 2.692152708233292e-06,
      "log_odds_chosen": 0.9853892922401428,
      "log_odds_ratio": -0.32194337248802185,
      "logits/chosen": -0.4604001045227051,
      "logits/rejected": -1.8486415147781372,
      "logps/chosen": -1.4580106735229492,
      "logps/rejected": -2.287992477416992,
      "loss": 1.5497,
      "nll_loss": 1.5175025463104248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14580106735229492,
      "rewards/margins": 0.08299816399812698,
      "rewards/rejected": -0.2287992388010025,
      "step": 576
    },
    {
      "epoch": 1.5909561615464274,
      "grad_norm": 0.17950667440891266,
      "learning_rate": 2.684161175324971e-06,
      "log_odds_chosen": 1.089317798614502,
      "log_odds_ratio": -0.3076907992362976,
      "logits/chosen": -0.45762938261032104,
      "logits/rejected": -1.8938159942626953,
      "logps/chosen": -1.3867409229278564,
      "logps/rejected": -2.298753261566162,
      "loss": 1.4599,
      "nll_loss": 1.4291636943817139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13867409527301788,
      "rewards/margins": 0.09120123088359833,
      "rewards/rejected": -0.2298753261566162,
      "step": 577
    },
    {
      "epoch": 1.5937176389368313,
      "grad_norm": 0.19735175371170044,
      "learning_rate": 2.676167749875635e-06,
      "log_odds_chosen": 0.8711864948272705,
      "log_odds_ratio": -0.359083890914917,
      "logits/chosen": -0.46582692861557007,
      "logits/rejected": -1.7266442775726318,
      "logps/chosen": -1.4628636837005615,
      "logps/rejected": -2.1918888092041016,
      "loss": 1.5503,
      "nll_loss": 1.514415979385376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14628636837005615,
      "rewards/margins": 0.07290251553058624,
      "rewards/rejected": -0.2191888988018036,
      "step": 578
    },
    {
      "epoch": 1.596479116327235,
      "grad_norm": 0.20369943976402283,
      "learning_rate": 2.6681725140300995e-06,
      "log_odds_chosen": 1.0683999061584473,
      "log_odds_ratio": -0.3045715093612671,
      "logits/chosen": -0.5854548215866089,
      "logits/rejected": -1.6316256523132324,
      "logps/chosen": -1.365673542022705,
      "logps/rejected": -2.2548635005950928,
      "loss": 1.4494,
      "nll_loss": 1.4189079999923706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1365673542022705,
      "rewards/margins": 0.08891897648572922,
      "rewards/rejected": -0.22548633813858032,
      "step": 579
    },
    {
      "epoch": 1.599240593717639,
      "grad_norm": 0.19143253564834595,
      "learning_rate": 2.6601755499517826e-06,
      "log_odds_chosen": 0.8805921077728271,
      "log_odds_ratio": -0.35098567605018616,
      "logits/chosen": -0.46608656644821167,
      "logits/rejected": -1.950951337814331,
      "logps/chosen": -1.4723143577575684,
      "logps/rejected": -2.210609197616577,
      "loss": 1.5573,
      "nll_loss": 1.5221997499465942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14723142981529236,
      "rewards/margins": 0.07382948696613312,
      "rewards/rejected": -0.22106093168258667,
      "step": 580
    },
    {
      "epoch": 1.6020020711080427,
      "grad_norm": 0.20576678216457367,
      "learning_rate": 2.6521769398218635e-06,
      "log_odds_chosen": 1.0742186307907104,
      "log_odds_ratio": -0.2951492369174957,
      "logits/chosen": -0.45335662364959717,
      "logits/rejected": -1.8008517026901245,
      "logps/chosen": -1.462270736694336,
      "logps/rejected": -2.3689029216766357,
      "loss": 1.5473,
      "nll_loss": 1.5177663564682007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14622707664966583,
      "rewards/margins": 0.09066321700811386,
      "rewards/rejected": -0.2368902862071991,
      "step": 581
    },
    {
      "epoch": 1.6047635484984468,
      "grad_norm": 0.22372370958328247,
      "learning_rate": 2.6441767658384363e-06,
      "log_odds_chosen": 1.1760811805725098,
      "log_odds_ratio": -0.28239208459854126,
      "logits/chosen": -0.5110992789268494,
      "logits/rejected": -1.7087138891220093,
      "logps/chosen": -1.408003807067871,
      "logps/rejected": -2.3959457874298096,
      "loss": 1.4905,
      "nll_loss": 1.4622644186019897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1408003866672516,
      "rewards/margins": 0.09879419207572937,
      "rewards/rejected": -0.23959457874298096,
      "step": 582
    },
    {
      "epoch": 1.6075250258888505,
      "grad_norm": 0.19179660081863403,
      "learning_rate": 2.6361751102156673e-06,
      "log_odds_chosen": 1.0498160123825073,
      "log_odds_ratio": -0.3036152720451355,
      "logits/chosen": -0.5851718783378601,
      "logits/rejected": -1.7799510955810547,
      "logps/chosen": -1.4760764837265015,
      "logps/rejected": -2.3658981323242188,
      "loss": 1.5528,
      "nll_loss": 1.5223983526229858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14760765433311462,
      "rewards/margins": 0.08898216485977173,
      "rewards/rejected": -0.23658981919288635,
      "step": 583
    },
    {
      "epoch": 1.6102865032792544,
      "grad_norm": 0.19203205406665802,
      "learning_rate": 2.628172055182948e-06,
      "log_odds_chosen": 1.0640352964401245,
      "log_odds_ratio": -0.3011913001537323,
      "logits/chosen": -0.502800703048706,
      "logits/rejected": -1.6559865474700928,
      "logps/chosen": -1.411937952041626,
      "logps/rejected": -2.296353340148926,
      "loss": 1.4964,
      "nll_loss": 1.4662748575210571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14119380712509155,
      "rewards/margins": 0.08844152092933655,
      "rewards/rejected": -0.2296353280544281,
      "step": 584
    },
    {
      "epoch": 1.6130479806696583,
      "grad_norm": 0.21048317849636078,
      "learning_rate": 2.620167682984052e-06,
      "log_odds_chosen": 1.0453581809997559,
      "log_odds_ratio": -0.3030966818332672,
      "logits/chosen": -0.44269949197769165,
      "logits/rejected": -1.7648762464523315,
      "logps/chosen": -1.5006442070007324,
      "logps/rejected": -2.386173963546753,
      "loss": 1.5706,
      "nll_loss": 1.5402867794036865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15006442368030548,
      "rewards/margins": 0.08855296671390533,
      "rewards/rejected": -0.23861739039421082,
      "step": 585
    },
    {
      "epoch": 1.6158094580600622,
      "grad_norm": 0.202589213848114,
      "learning_rate": 2.6121620758762877e-06,
      "log_odds_chosen": 1.2401037216186523,
      "log_odds_ratio": -0.26773184537887573,
      "logits/chosen": -0.4954875409603119,
      "logits/rejected": -1.7728123664855957,
      "logps/chosen": -1.4029090404510498,
      "logps/rejected": -2.4527475833892822,
      "loss": 1.4836,
      "nll_loss": 1.456812858581543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14029091596603394,
      "rewards/margins": 0.10498383641242981,
      "rewards/rejected": -0.24527473747730255,
      "step": 586
    },
    {
      "epoch": 1.618570935450466,
      "grad_norm": 0.19367991387844086,
      "learning_rate": 2.604155316129654e-06,
      "log_odds_chosen": 1.1511355638504028,
      "log_odds_ratio": -0.2958333492279053,
      "logits/chosen": -0.41890662908554077,
      "logits/rejected": -1.7125511169433594,
      "logps/chosen": -1.370666742324829,
      "logps/rejected": -2.336446762084961,
      "loss": 1.4669,
      "nll_loss": 1.4373064041137695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13706667721271515,
      "rewards/margins": 0.09657800942659378,
      "rewards/rejected": -0.23364469408988953,
      "step": 587
    },
    {
      "epoch": 1.6213324128408697,
      "grad_norm": 0.20538587868213654,
      "learning_rate": 2.596147486025996e-06,
      "log_odds_chosen": 0.998070240020752,
      "log_odds_ratio": -0.32918909192085266,
      "logits/chosen": -0.4908704161643982,
      "logits/rejected": -1.7168567180633545,
      "logps/chosen": -1.4522796869277954,
      "logps/rejected": -2.2951455116271973,
      "loss": 1.542,
      "nll_loss": 1.5090749263763428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14522796869277954,
      "rewards/margins": 0.08428659290075302,
      "rewards/rejected": -0.22951453924179077,
      "step": 588
    },
    {
      "epoch": 1.6240938902312738,
      "grad_norm": 0.19525809586048126,
      "learning_rate": 2.5881386678581587e-06,
      "log_odds_chosen": 0.9631695747375488,
      "log_odds_ratio": -0.33941492438316345,
      "logits/chosen": -0.4752748906612396,
      "logits/rejected": -1.823403239250183,
      "logps/chosen": -1.4721821546554565,
      "logps/rejected": -2.2869250774383545,
      "loss": 1.5529,
      "nll_loss": 1.5189189910888672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14721821248531342,
      "rewards/margins": 0.08147431910037994,
      "rewards/rejected": -0.22869251668453217,
      "step": 589
    },
    {
      "epoch": 1.6268553676216775,
      "grad_norm": 0.18992260098457336,
      "learning_rate": 2.580128943929139e-06,
      "log_odds_chosen": 1.0529534816741943,
      "log_odds_ratio": -0.3118128478527069,
      "logits/chosen": -0.5909528732299805,
      "logits/rejected": -1.7769283056259155,
      "logps/chosen": -1.482077956199646,
      "logps/rejected": -2.3781533241271973,
      "loss": 1.5494,
      "nll_loss": 1.5182218551635742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14820778369903564,
      "rewards/margins": 0.08960752934217453,
      "rewards/rejected": -0.23781533539295197,
      "step": 590
    },
    {
      "epoch": 1.6296168450120816,
      "grad_norm": 0.19947484135627747,
      "learning_rate": 2.5721183965512424e-06,
      "log_odds_chosen": 0.9602290391921997,
      "log_odds_ratio": -0.33199790120124817,
      "logits/chosen": -0.4957779049873352,
      "logits/rejected": -1.8233221769332886,
      "logps/chosen": -1.4387155771255493,
      "logps/rejected": -2.243359327316284,
      "loss": 1.5226,
      "nll_loss": 1.4893605709075928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14387157559394836,
      "rewards/margins": 0.08046435564756393,
      "rewards/rejected": -0.2243359237909317,
      "step": 591
    },
    {
      "epoch": 1.6323783224024853,
      "grad_norm": 0.19560271501541138,
      "learning_rate": 2.564107108045239e-06,
      "log_odds_chosen": 0.9684425592422485,
      "log_odds_ratio": -0.33081498742103577,
      "logits/chosen": -0.390799880027771,
      "logits/rejected": -1.8621724843978882,
      "logps/chosen": -1.5033597946166992,
      "logps/rejected": -2.32216215133667,
      "loss": 1.5717,
      "nll_loss": 1.5385994911193848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15033598244190216,
      "rewards/margins": 0.08188023418188095,
      "rewards/rejected": -0.2322162240743637,
      "step": 592
    },
    {
      "epoch": 1.6351397997928891,
      "grad_norm": 0.21181620657444,
      "learning_rate": 2.556095160739513e-06,
      "log_odds_chosen": 1.1007102727890015,
      "log_odds_ratio": -0.29589590430259705,
      "logits/chosen": -0.4980851411819458,
      "logits/rejected": -1.648532748222351,
      "logps/chosen": -1.4862791299819946,
      "logps/rejected": -2.4272263050079346,
      "loss": 1.5887,
      "nll_loss": 1.5591251850128174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14862790703773499,
      "rewards/margins": 0.09409470856189728,
      "rewards/rejected": -0.24272264540195465,
      "step": 593
    },
    {
      "epoch": 1.637901277183293,
      "grad_norm": 0.20149773359298706,
      "learning_rate": 2.5480826369692178e-06,
      "log_odds_chosen": 0.94913649559021,
      "log_odds_ratio": -0.35477903485298157,
      "logits/chosen": -0.47597599029541016,
      "logits/rejected": -1.6722043752670288,
      "logps/chosen": -1.4268722534179688,
      "logps/rejected": -2.231581211090088,
      "loss": 1.521,
      "nll_loss": 1.485567331314087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14268723130226135,
      "rewards/margins": 0.08047088980674744,
      "rewards/rejected": -0.2231581062078476,
      "step": 594
    },
    {
      "epoch": 1.640662754573697,
      "grad_norm": 0.19771575927734375,
      "learning_rate": 2.5400696190754347e-06,
      "log_odds_chosen": 1.0618021488189697,
      "log_odds_ratio": -0.3072146475315094,
      "logits/chosen": -0.5567762851715088,
      "logits/rejected": -1.8684656620025635,
      "logps/chosen": -1.4032098054885864,
      "logps/rejected": -2.292144298553467,
      "loss": 1.4948,
      "nll_loss": 1.464051604270935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14032098650932312,
      "rewards/margins": 0.08889344334602356,
      "rewards/rejected": -0.22921442985534668,
      "step": 595
    },
    {
      "epoch": 1.6434242319641008,
      "grad_norm": 0.1921544373035431,
      "learning_rate": 2.532056189404318e-06,
      "log_odds_chosen": 1.0136642456054688,
      "log_odds_ratio": -0.31946948170661926,
      "logits/chosen": -0.5976563096046448,
      "logits/rejected": -1.6406235694885254,
      "logps/chosen": -1.3311948776245117,
      "logps/rejected": -2.164565086364746,
      "loss": 1.4056,
      "nll_loss": 1.3736519813537598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13311949372291565,
      "rewards/margins": 0.08333701640367508,
      "rewards/rejected": -0.21645651757717133,
      "step": 596
    },
    {
      "epoch": 1.6461857093545047,
      "grad_norm": 0.20298127830028534,
      "learning_rate": 2.524042430306258e-06,
      "log_odds_chosen": 1.0893070697784424,
      "log_odds_ratio": -0.29967525601387024,
      "logits/chosen": -0.5517418384552002,
      "logits/rejected": -1.6711257696151733,
      "logps/chosen": -1.431249976158142,
      "logps/rejected": -2.353149652481079,
      "loss": 1.5109,
      "nll_loss": 1.4809296131134033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431249976158142,
      "rewards/margins": 0.0921899825334549,
      "rewards/rejected": -0.2353149801492691,
      "step": 597
    },
    {
      "epoch": 1.6489471867449086,
      "grad_norm": 0.2075520008802414,
      "learning_rate": 2.516028424135028e-06,
      "log_odds_chosen": 0.9435156583786011,
      "log_odds_ratio": -0.331230103969574,
      "logits/chosen": -0.47791504859924316,
      "logits/rejected": -1.588948369026184,
      "logps/chosen": -1.4235913753509521,
      "logps/rejected": -2.206226110458374,
      "loss": 1.5238,
      "nll_loss": 1.490726351737976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14235913753509521,
      "rewards/margins": 0.07826346904039383,
      "rewards/rejected": -0.22062261402606964,
      "step": 598
    },
    {
      "epoch": 1.6517086641353123,
      "grad_norm": 0.3242340087890625,
      "learning_rate": 2.5080142532469405e-06,
      "log_odds_chosen": 0.8534775972366333,
      "log_odds_ratio": -0.35872241854667664,
      "logits/chosen": -0.5117841958999634,
      "logits/rejected": -1.6608119010925293,
      "logps/chosen": -1.383968710899353,
      "logps/rejected": -2.084062099456787,
      "loss": 1.4697,
      "nll_loss": 1.4338546991348267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13839687407016754,
      "rewards/margins": 0.07000933587551117,
      "rewards/rejected": -0.2084062248468399,
      "step": 599
    },
    {
      "epoch": 1.6544701415257164,
      "grad_norm": 0.18982619047164917,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 1.0374600887298584,
      "log_odds_ratio": -0.30684134364128113,
      "logits/chosen": -0.47200512886047363,
      "logits/rejected": -1.692277431488037,
      "logps/chosen": -1.3962559700012207,
      "logps/rejected": -2.260599374771118,
      "loss": 1.4793,
      "nll_loss": 1.448584794998169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13962560892105103,
      "rewards/margins": 0.08643434941768646,
      "rewards/rejected": -0.2260599583387375,
      "step": 600
    },
    {
      "epoch": 1.65723161891612,
      "grad_norm": 0.21345901489257812,
      "learning_rate": 2.4919857467530608e-06,
      "log_odds_chosen": 1.1221760511398315,
      "log_odds_ratio": -0.2836793065071106,
      "logits/chosen": -0.6063240766525269,
      "logits/rejected": -1.7040760517120361,
      "logps/chosen": -1.4183056354522705,
      "logps/rejected": -2.36080265045166,
      "loss": 1.5098,
      "nll_loss": 1.4814752340316772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14183056354522705,
      "rewards/margins": 0.09424971044063568,
      "rewards/rejected": -0.23608025908470154,
      "step": 601
    },
    {
      "epoch": 1.6599930963065241,
      "grad_norm": 0.21028882265090942,
      "learning_rate": 2.4839715758649724e-06,
      "log_odds_chosen": 0.9369592070579529,
      "log_odds_ratio": -0.3362049162387848,
      "logits/chosen": -0.45639631152153015,
      "logits/rejected": -1.7908605337142944,
      "logps/chosen": -1.5038936138153076,
      "logps/rejected": -2.2945337295532227,
      "loss": 1.5641,
      "nll_loss": 1.5304303169250488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15038937330245972,
      "rewards/margins": 0.07906400412321091,
      "rewards/rejected": -0.22945338487625122,
      "step": 602
    },
    {
      "epoch": 1.6627545736969278,
      "grad_norm": 0.20318543910980225,
      "learning_rate": 2.475957569693742e-06,
      "log_odds_chosen": 1.1263985633850098,
      "log_odds_ratio": -0.28691041469573975,
      "logits/chosen": -0.5157269835472107,
      "logits/rejected": -1.812491774559021,
      "logps/chosen": -1.4623472690582275,
      "logps/rejected": -2.419386625289917,
      "loss": 1.5437,
      "nll_loss": 1.515026330947876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14623473584651947,
      "rewards/margins": 0.09570392221212387,
      "rewards/rejected": -0.24193865060806274,
      "step": 603
    },
    {
      "epoch": 1.6655160510873317,
      "grad_norm": 0.2132967710494995,
      "learning_rate": 2.467943810595682e-06,
      "log_odds_chosen": 0.7572492361068726,
      "log_odds_ratio": -0.3912786543369293,
      "logits/chosen": -0.6214781999588013,
      "logits/rejected": -1.4917678833007812,
      "logps/chosen": -1.4525686502456665,
      "logps/rejected": -2.0807745456695557,
      "loss": 1.5376,
      "nll_loss": 1.4985002279281616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452568620443344,
      "rewards/margins": 0.06282058358192444,
      "rewards/rejected": -0.20807746052742004,
      "step": 604
    },
    {
      "epoch": 1.6682775284777356,
      "grad_norm": 0.19940869510173798,
      "learning_rate": 2.459930380924566e-06,
      "log_odds_chosen": 1.0550998449325562,
      "log_odds_ratio": -0.30236515402793884,
      "logits/chosen": -0.44110098481178284,
      "logits/rejected": -1.6045345067977905,
      "logps/chosen": -1.4446752071380615,
      "logps/rejected": -2.332336902618408,
      "loss": 1.5361,
      "nll_loss": 1.5058379173278809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1444675326347351,
      "rewards/margins": 0.0887661725282669,
      "rewards/rejected": -0.23323370516300201,
      "step": 605
    },
    {
      "epoch": 1.6710390058681395,
      "grad_norm": 0.2036145180463791,
      "learning_rate": 2.4519173630307826e-06,
      "log_odds_chosen": 0.9749938249588013,
      "log_odds_ratio": -0.33162128925323486,
      "logits/chosen": -0.5159994959831238,
      "logits/rejected": -1.5376782417297363,
      "logps/chosen": -1.3822424411773682,
      "logps/rejected": -2.1887948513031006,
      "loss": 1.463,
      "nll_loss": 1.4298853874206543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13822424411773682,
      "rewards/margins": 0.08065523952245712,
      "rewards/rejected": -0.21887947618961334,
      "step": 606
    },
    {
      "epoch": 1.6738004832585434,
      "grad_norm": 0.19285212457180023,
      "learning_rate": 2.443904839260488e-06,
      "log_odds_chosen": 1.0728142261505127,
      "log_odds_ratio": -0.2983938455581665,
      "logits/chosen": -0.48161622881889343,
      "logits/rejected": -1.8781613111495972,
      "logps/chosen": -1.4637552499771118,
      "logps/rejected": -2.369638442993164,
      "loss": 1.5399,
      "nll_loss": 1.5100862979888916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14637553691864014,
      "rewards/margins": 0.09058833122253418,
      "rewards/rejected": -0.23696383833885193,
      "step": 607
    },
    {
      "epoch": 1.676561960648947,
      "grad_norm": 0.213767871260643,
      "learning_rate": 2.4358928919547616e-06,
      "log_odds_chosen": 1.0425572395324707,
      "log_odds_ratio": -0.3122728168964386,
      "logits/chosen": -0.621905505657196,
      "logits/rejected": -1.3375020027160645,
      "logps/chosen": -1.4448013305664062,
      "logps/rejected": -2.3270554542541504,
      "loss": 1.5444,
      "nll_loss": 1.5131899118423462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1444801241159439,
      "rewards/margins": 0.08822542428970337,
      "rewards/rejected": -0.23270554840564728,
      "step": 608
    },
    {
      "epoch": 1.6793234380393511,
      "grad_norm": 0.20144887268543243,
      "learning_rate": 2.427881603448758e-06,
      "log_odds_chosen": 1.1905502080917358,
      "log_odds_ratio": -0.27711763978004456,
      "logits/chosen": -0.497713565826416,
      "logits/rejected": -1.6608542203903198,
      "logps/chosen": -1.3489593267440796,
      "logps/rejected": -2.3394763469696045,
      "loss": 1.4485,
      "nll_loss": 1.4207406044006348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1348959356546402,
      "rewards/margins": 0.09905169904232025,
      "rewards/rejected": -0.23394761979579926,
      "step": 609
    },
    {
      "epoch": 1.6820849154297548,
      "grad_norm": 0.17932826280593872,
      "learning_rate": 2.4198710560708623e-06,
      "log_odds_chosen": 1.1837692260742188,
      "log_odds_ratio": -0.2758199870586395,
      "logits/chosen": -0.5171345472335815,
      "logits/rejected": -1.9973483085632324,
      "logps/chosen": -1.4259376525878906,
      "logps/rejected": -2.424710512161255,
      "loss": 1.5165,
      "nll_loss": 1.4889655113220215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14259377121925354,
      "rewards/margins": 0.09987729787826538,
      "rewards/rejected": -0.24247105419635773,
      "step": 610
    },
    {
      "epoch": 1.684846392820159,
      "grad_norm": 0.2096761018037796,
      "learning_rate": 2.411861332141842e-06,
      "log_odds_chosen": 1.0929014682769775,
      "log_odds_ratio": -0.29848530888557434,
      "logits/chosen": -0.5515930652618408,
      "logits/rejected": -1.6768569946289062,
      "logps/chosen": -1.3467549085617065,
      "logps/rejected": -2.250361680984497,
      "loss": 1.4289,
      "nll_loss": 1.3990368843078613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13467547297477722,
      "rewards/margins": 0.09036068618297577,
      "rewards/rejected": -0.22503617405891418,
      "step": 611
    },
    {
      "epoch": 1.6876078702105626,
      "grad_norm": 0.23656167089939117,
      "learning_rate": 2.403852513974004e-06,
      "log_odds_chosen": 1.0888586044311523,
      "log_odds_ratio": -0.3083086609840393,
      "logits/chosen": -0.41853654384613037,
      "logits/rejected": -1.3835935592651367,
      "logps/chosen": -1.4978593587875366,
      "logps/rejected": -2.4035394191741943,
      "loss": 1.5535,
      "nll_loss": 1.5226693153381348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14978592097759247,
      "rewards/margins": 0.09056802093982697,
      "rewards/rejected": -0.24035394191741943,
      "step": 612
    },
    {
      "epoch": 1.6903693476009665,
      "grad_norm": 0.19909033179283142,
      "learning_rate": 2.3958446838703462e-06,
      "log_odds_chosen": 1.2349250316619873,
      "log_odds_ratio": -0.263581782579422,
      "logits/chosen": -0.44553297758102417,
      "logits/rejected": -1.8215895891189575,
      "logps/chosen": -1.4487204551696777,
      "logps/rejected": -2.501533031463623,
      "loss": 1.5126,
      "nll_loss": 1.4862422943115234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14487206935882568,
      "rewards/margins": 0.10528125613927841,
      "rewards/rejected": -0.2501533329486847,
      "step": 613
    },
    {
      "epoch": 1.6931308249913704,
      "grad_norm": 0.20478682219982147,
      "learning_rate": 2.3878379241237136e-06,
      "log_odds_chosen": 0.9483797550201416,
      "log_odds_ratio": -0.33403685688972473,
      "logits/chosen": -0.5537324547767639,
      "logits/rejected": -1.3671543598175049,
      "logps/chosen": -1.475398063659668,
      "logps/rejected": -2.2737462520599365,
      "loss": 1.5512,
      "nll_loss": 1.5178141593933105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14753982424736023,
      "rewards/margins": 0.07983479648828506,
      "rewards/rejected": -0.2273746132850647,
      "step": 614
    },
    {
      "epoch": 1.6958923023817742,
      "grad_norm": 0.19413712620735168,
      "learning_rate": 2.3798323170159487e-06,
      "log_odds_chosen": 0.9828821420669556,
      "log_odds_ratio": -0.32647275924682617,
      "logits/chosen": -0.5587603449821472,
      "logits/rejected": -1.868147373199463,
      "logps/chosen": -1.4277853965759277,
      "logps/rejected": -2.252943992614746,
      "loss": 1.5115,
      "nll_loss": 1.4788599014282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14277854561805725,
      "rewards/margins": 0.08251585066318512,
      "rewards/rejected": -0.22529438138008118,
      "step": 615
    },
    {
      "epoch": 1.6986537797721781,
      "grad_norm": 0.2020513415336609,
      "learning_rate": 2.3718279448170527e-06,
      "log_odds_chosen": 1.0720229148864746,
      "log_odds_ratio": -0.3052593767642975,
      "logits/chosen": -0.5639959573745728,
      "logits/rejected": -1.528878927230835,
      "logps/chosen": -1.4678881168365479,
      "logps/rejected": -2.3781814575195312,
      "loss": 1.5566,
      "nll_loss": 1.526060938835144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1467888206243515,
      "rewards/margins": 0.0910293310880661,
      "rewards/rejected": -0.2378181368112564,
      "step": 616
    },
    {
      "epoch": 1.701415257162582,
      "grad_norm": 0.18654049932956696,
      "learning_rate": 2.363824889784333e-06,
      "log_odds_chosen": 1.1250965595245361,
      "log_odds_ratio": -0.2894567847251892,
      "logits/chosen": -0.4636651277542114,
      "logits/rejected": -1.680492639541626,
      "logps/chosen": -1.379407286643982,
      "logps/rejected": -2.32011342048645,
      "loss": 1.4648,
      "nll_loss": 1.4358913898468018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13794073462486267,
      "rewards/margins": 0.09407063573598862,
      "rewards/rejected": -0.2320113629102707,
      "step": 617
    },
    {
      "epoch": 1.704176734552986,
      "grad_norm": 0.215307354927063,
      "learning_rate": 2.3558232341615645e-06,
      "log_odds_chosen": 0.9324149489402771,
      "log_odds_ratio": -0.3408413231372833,
      "logits/chosen": -0.4387376010417938,
      "logits/rejected": -1.6170055866241455,
      "logps/chosen": -1.4724270105361938,
      "logps/rejected": -2.2553932666778564,
      "loss": 1.569,
      "nll_loss": 1.5349595546722412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1472426950931549,
      "rewards/margins": 0.07829662412405014,
      "rewards/rejected": -0.22553932666778564,
      "step": 618
    },
    {
      "epoch": 1.7069382119433896,
      "grad_norm": 0.20065483450889587,
      "learning_rate": 2.3478230601781374e-06,
      "log_odds_chosen": 1.0297596454620361,
      "log_odds_ratio": -0.3112369775772095,
      "logits/chosen": -0.6553104519844055,
      "logits/rejected": -1.7943086624145508,
      "logps/chosen": -1.4568157196044922,
      "logps/rejected": -2.324397087097168,
      "loss": 1.5444,
      "nll_loss": 1.5132304430007935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14568158984184265,
      "rewards/margins": 0.08675812184810638,
      "rewards/rejected": -0.23243969678878784,
      "step": 619
    },
    {
      "epoch": 1.7096996893337937,
      "grad_norm": 0.19946393370628357,
      "learning_rate": 2.339824450048218e-06,
      "log_odds_chosen": 1.1597028970718384,
      "log_odds_ratio": -0.27503636479377747,
      "logits/chosen": -0.4450312554836273,
      "logits/rejected": -1.8078252077102661,
      "logps/chosen": -1.4679698944091797,
      "logps/rejected": -2.453463077545166,
      "loss": 1.5524,
      "nll_loss": 1.5249354839324951,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14679698646068573,
      "rewards/margins": 0.09854928404092789,
      "rewards/rejected": -0.24534626305103302,
      "step": 620
    },
    {
      "epoch": 1.7124611667241973,
      "grad_norm": 0.20729611814022064,
      "learning_rate": 2.331827485969901e-06,
      "log_odds_chosen": 1.084412932395935,
      "log_odds_ratio": -0.3078622817993164,
      "logits/chosen": -0.5357551574707031,
      "logits/rejected": -1.7995785474777222,
      "logps/chosen": -1.438812494277954,
      "logps/rejected": -2.353656768798828,
      "loss": 1.5192,
      "nll_loss": 1.4883934259414673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14388124644756317,
      "rewards/margins": 0.0914844498038292,
      "rewards/rejected": -0.23536570370197296,
      "step": 621
    },
    {
      "epoch": 1.7152226441146015,
      "grad_norm": 0.21530677378177643,
      "learning_rate": 2.323832250124365e-06,
      "log_odds_chosen": 1.1834465265274048,
      "log_odds_ratio": -0.2767082154750824,
      "logits/chosen": -0.5363863110542297,
      "logits/rejected": -1.811279535293579,
      "logps/chosen": -1.4377635717391968,
      "logps/rejected": -2.4433891773223877,
      "loss": 1.5118,
      "nll_loss": 1.4841231107711792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14377637207508087,
      "rewards/margins": 0.10056254267692566,
      "rewards/rejected": -0.24433891475200653,
      "step": 622
    },
    {
      "epoch": 1.7179841215050051,
      "grad_norm": 0.2050798088312149,
      "learning_rate": 2.3158388246750308e-06,
      "log_odds_chosen": 0.8693351745605469,
      "log_odds_ratio": -0.3585469126701355,
      "logits/chosen": -0.6070123314857483,
      "logits/rejected": -1.4305416345596313,
      "logps/chosen": -1.4386093616485596,
      "logps/rejected": -2.1597909927368164,
      "loss": 1.5264,
      "nll_loss": 1.4905424118041992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14386095106601715,
      "rewards/margins": 0.07211817055940628,
      "rewards/rejected": -0.21597909927368164,
      "step": 623
    },
    {
      "epoch": 1.720745598895409,
      "grad_norm": 0.19685739278793335,
      "learning_rate": 2.307847291766709e-06,
      "log_odds_chosen": 1.0625098943710327,
      "log_odds_ratio": -0.3041455149650574,
      "logits/chosen": -0.5462653636932373,
      "logits/rejected": -1.6785917282104492,
      "logps/chosen": -1.4387760162353516,
      "logps/rejected": -2.3338117599487305,
      "loss": 1.5194,
      "nll_loss": 1.4889512062072754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14387759566307068,
      "rewards/margins": 0.08950359374284744,
      "rewards/rejected": -0.23338119685649872,
      "step": 624
    },
    {
      "epoch": 1.723507076285813,
      "grad_norm": 0.19702772796154022,
      "learning_rate": 2.299857733524767e-06,
      "log_odds_chosen": 0.9161310195922852,
      "log_odds_ratio": -0.3448503613471985,
      "logits/chosen": -0.5154906511306763,
      "logits/rejected": -1.6805267333984375,
      "logps/chosen": -1.4076039791107178,
      "logps/rejected": -2.1692450046539307,
      "loss": 1.4916,
      "nll_loss": 1.4570910930633545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1407603770494461,
      "rewards/margins": 0.076164111495018,
      "rewards/rejected": -0.2169245034456253,
      "step": 625
    },
    {
      "epoch": 1.7262685536762168,
      "grad_norm": 0.19595174491405487,
      "learning_rate": 2.291870232054277e-06,
      "log_odds_chosen": 0.9098183512687683,
      "log_odds_ratio": -0.34384119510650635,
      "logits/chosen": -0.4463910162448883,
      "logits/rejected": -2.0581111907958984,
      "logps/chosen": -1.4506475925445557,
      "logps/rejected": -2.2079553604125977,
      "loss": 1.5414,
      "nll_loss": 1.5070030689239502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14506477117538452,
      "rewards/margins": 0.07573077827692032,
      "rewards/rejected": -0.22079554200172424,
      "step": 626
    },
    {
      "epoch": 1.7290300310666207,
      "grad_norm": 0.21787682175636292,
      "learning_rate": 2.283884869439175e-06,
      "log_odds_chosen": 1.1688117980957031,
      "log_odds_ratio": -0.2737486958503723,
      "logits/chosen": -0.5133581161499023,
      "logits/rejected": -1.7837564945220947,
      "logps/chosen": -1.5140570402145386,
      "logps/rejected": -2.5165603160858154,
      "loss": 1.5811,
      "nll_loss": 1.5537470579147339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1514057070016861,
      "rewards/margins": 0.10025034844875336,
      "rewards/rejected": -0.25165605545043945,
      "step": 627
    },
    {
      "epoch": 1.7317915084570243,
      "grad_norm": 0.19975106418132782,
      "learning_rate": 2.2759017277414165e-06,
      "log_odds_chosen": 1.3159109354019165,
      "log_odds_ratio": -0.24291640520095825,
      "logits/chosen": -0.5234307646751404,
      "logits/rejected": -1.8133102655410767,
      "logps/chosen": -1.3741402626037598,
      "logps/rejected": -2.484227418899536,
      "loss": 1.4488,
      "nll_loss": 1.424553632736206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13741403818130493,
      "rewards/margins": 0.11100868880748749,
      "rewards/rejected": -0.24842271208763123,
      "step": 628
    },
    {
      "epoch": 1.7345529858474285,
      "grad_norm": 0.20025824010372162,
      "learning_rate": 2.267920889000135e-06,
      "log_odds_chosen": 0.9150658845901489,
      "log_odds_ratio": -0.3410889804363251,
      "logits/chosen": -0.4876527786254883,
      "logits/rejected": -1.7472199201583862,
      "logps/chosen": -1.487480640411377,
      "logps/rejected": -2.258146047592163,
      "loss": 1.5642,
      "nll_loss": 1.530122995376587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14874805510044098,
      "rewards/margins": 0.07706654071807861,
      "rewards/rejected": -0.2258146107196808,
      "step": 629
    },
    {
      "epoch": 1.7373144632378321,
      "grad_norm": 0.21012412011623383,
      "learning_rate": 2.2599424352307958e-06,
      "log_odds_chosen": 1.1698055267333984,
      "log_odds_ratio": -0.2723722457885742,
      "logits/chosen": -0.5588301420211792,
      "logits/rejected": -1.7450660467147827,
      "logps/chosen": -1.2947266101837158,
      "logps/rejected": -2.2551028728485107,
      "loss": 1.3905,
      "nll_loss": 1.3633110523223877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12947267293930054,
      "rewards/margins": 0.09603763371706009,
      "rewards/rejected": -0.22551029920578003,
      "step": 630
    },
    {
      "epoch": 1.7400759406282362,
      "grad_norm": 0.20005680620670319,
      "learning_rate": 2.2519664484243564e-06,
      "log_odds_chosen": 1.027710199356079,
      "log_odds_ratio": -0.3135443925857544,
      "logits/chosen": -0.5663747191429138,
      "logits/rejected": -1.6136528253555298,
      "logps/chosen": -1.4354394674301147,
      "logps/rejected": -2.2970330715179443,
      "loss": 1.5057,
      "nll_loss": 1.4743560552597046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14354394376277924,
      "rewards/margins": 0.0861593633890152,
      "rewards/rejected": -0.22970330715179443,
      "step": 631
    },
    {
      "epoch": 1.74283741801864,
      "grad_norm": 0.1957002431154251,
      "learning_rate": 2.243993010546422e-06,
      "log_odds_chosen": 1.0878336429595947,
      "log_odds_ratio": -0.30391043424606323,
      "logits/chosen": -0.5153028964996338,
      "logits/rejected": -1.4119391441345215,
      "logps/chosen": -1.4629807472229004,
      "logps/rejected": -2.3840439319610596,
      "loss": 1.5377,
      "nll_loss": 1.507292628288269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14629808068275452,
      "rewards/margins": 0.09210632741451263,
      "rewards/rejected": -0.23840439319610596,
      "step": 632
    },
    {
      "epoch": 1.745598895409044,
      "grad_norm": 0.2143191397190094,
      "learning_rate": 2.2360222035364027e-06,
      "log_odds_chosen": 0.9642858505249023,
      "log_odds_ratio": -0.32829856872558594,
      "logits/chosen": -0.4845995604991913,
      "logits/rejected": -1.453553318977356,
      "logps/chosen": -1.440818190574646,
      "logps/rejected": -2.2478103637695312,
      "loss": 1.5264,
      "nll_loss": 1.4935935735702515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14408183097839355,
      "rewards/margins": 0.08069923520088196,
      "rewards/rejected": -0.22478105127811432,
      "step": 633
    },
    {
      "epoch": 1.7483603727994477,
      "grad_norm": 0.19890277087688446,
      "learning_rate": 2.228054109306673e-06,
      "log_odds_chosen": 1.1955161094665527,
      "log_odds_ratio": -0.27173271775245667,
      "logits/chosen": -0.5415982604026794,
      "logits/rejected": -2.0548179149627686,
      "logps/chosen": -1.4267183542251587,
      "logps/rejected": -2.4402973651885986,
      "loss": 1.5018,
      "nll_loss": 1.4746149778366089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1426718533039093,
      "rewards/margins": 0.10135789215564728,
      "rewards/rejected": -0.24402973055839539,
      "step": 634
    },
    {
      "epoch": 1.7511218501898516,
      "grad_norm": 0.21162277460098267,
      "learning_rate": 2.2200888097417308e-06,
      "log_odds_chosen": 1.1923596858978271,
      "log_odds_ratio": -0.2712979316711426,
      "logits/chosen": -0.5525784492492676,
      "logits/rejected": -1.8263614177703857,
      "logps/chosen": -1.4566607475280762,
      "logps/rejected": -2.4698026180267334,
      "loss": 1.5463,
      "nll_loss": 1.5191999673843384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14566607773303986,
      "rewards/margins": 0.10131418704986572,
      "rewards/rejected": -0.24698026478290558,
      "step": 635
    },
    {
      "epoch": 1.7538833275802554,
      "grad_norm": 0.21881026029586792,
      "learning_rate": 2.212126386697352e-06,
      "log_odds_chosen": 0.9788854122161865,
      "log_odds_ratio": -0.3270198404788971,
      "logits/chosen": -0.5637598633766174,
      "logits/rejected": -1.4680867195129395,
      "logps/chosen": -1.486203670501709,
      "logps/rejected": -2.3113152980804443,
      "loss": 1.5654,
      "nll_loss": 1.532711386680603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486203521490097,
      "rewards/margins": 0.08251115679740906,
      "rewards/rejected": -0.23113152384757996,
      "step": 636
    },
    {
      "epoch": 1.7566448049706593,
      "grad_norm": 0.1946457475423813,
      "learning_rate": 2.204166921999753e-06,
      "log_odds_chosen": 1.10805082321167,
      "log_odds_ratio": -0.28972306847572327,
      "logits/chosen": -0.5027114748954773,
      "logits/rejected": -1.73331618309021,
      "logps/chosen": -1.4216092824935913,
      "logps/rejected": -2.3447673320770264,
      "loss": 1.492,
      "nll_loss": 1.4630554914474487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14216093719005585,
      "rewards/margins": 0.09231581538915634,
      "rewards/rejected": -0.2344767451286316,
      "step": 637
    },
    {
      "epoch": 1.7594062823610632,
      "grad_norm": 0.2095964401960373,
      "learning_rate": 2.196210497444749e-06,
      "log_odds_chosen": 1.129142165184021,
      "log_odds_ratio": -0.2840039134025574,
      "logits/chosen": -0.6306648850440979,
      "logits/rejected": -1.8879787921905518,
      "logps/chosen": -1.4602231979370117,
      "logps/rejected": -2.418236255645752,
      "loss": 1.5277,
      "nll_loss": 1.4992913007736206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14602231979370117,
      "rewards/margins": 0.09580130875110626,
      "rewards/rejected": -0.24182362854480743,
      "step": 638
    },
    {
      "epoch": 1.7621677597514669,
      "grad_norm": 0.2068668156862259,
      "learning_rate": 2.1882571947969134e-06,
      "log_odds_chosen": 0.9739767909049988,
      "log_odds_ratio": -0.3297915458679199,
      "logits/chosen": -0.5247660875320435,
      "logits/rejected": -1.5939874649047852,
      "logps/chosen": -1.424277901649475,
      "logps/rejected": -2.237306833267212,
      "loss": 1.5071,
      "nll_loss": 1.4741605520248413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14242780208587646,
      "rewards/margins": 0.08130290359258652,
      "rewards/rejected": -0.22373069822788239,
      "step": 639
    },
    {
      "epoch": 1.764929237141871,
      "grad_norm": 0.2063615918159485,
      "learning_rate": 2.1803070957887348e-06,
      "log_odds_chosen": 0.9807276725769043,
      "log_odds_ratio": -0.32462048530578613,
      "logits/chosen": -0.5069320201873779,
      "logits/rejected": -1.6134123802185059,
      "logps/chosen": -1.391251802444458,
      "logps/rejected": -2.201903820037842,
      "loss": 1.4684,
      "nll_loss": 1.4358959197998047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13912518322467804,
      "rewards/margins": 0.08106519281864166,
      "rewards/rejected": -0.2201903760433197,
      "step": 640
    },
    {
      "epoch": 1.7676907145322747,
      "grad_norm": 0.20232875645160675,
      "learning_rate": 2.1723602821197835e-06,
      "log_odds_chosen": 1.0409268140792847,
      "log_odds_ratio": -0.310039222240448,
      "logits/chosen": -0.5773876905441284,
      "logits/rejected": -1.7172613143920898,
      "logps/chosen": -1.43693208694458,
      "logps/rejected": -2.3079919815063477,
      "loss": 1.5217,
      "nll_loss": 1.4906917810440063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.143693208694458,
      "rewards/margins": 0.08710600435733795,
      "rewards/rejected": -0.23079921305179596,
      "step": 641
    },
    {
      "epoch": 1.7704521919226788,
      "grad_norm": 0.22946524620056152,
      "learning_rate": 2.1644168354558623e-06,
      "log_odds_chosen": 1.0303146839141846,
      "log_odds_ratio": -0.31098419427871704,
      "logits/chosen": -0.5952135324478149,
      "logits/rejected": -1.6176903247833252,
      "logps/chosen": -1.502160906791687,
      "logps/rejected": -2.3773584365844727,
      "loss": 1.5832,
      "nll_loss": 1.5521173477172852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15021610260009766,
      "rewards/margins": 0.08751977235078812,
      "rewards/rejected": -0.2377358376979828,
      "step": 642
    },
    {
      "epoch": 1.7732136693130824,
      "grad_norm": 0.21178148686885834,
      "learning_rate": 2.1564768374281774e-06,
      "log_odds_chosen": 1.2231894731521606,
      "log_odds_ratio": -0.2707389295101166,
      "logits/chosen": -0.6701139807701111,
      "logits/rejected": -2.0402708053588867,
      "logps/chosen": -1.4372367858886719,
      "logps/rejected": -2.47658371925354,
      "loss": 1.5106,
      "nll_loss": 1.4835582971572876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14372369647026062,
      "rewards/margins": 0.10393467545509338,
      "rewards/rejected": -0.2476583570241928,
      "step": 643
    },
    {
      "epoch": 1.7759751467034863,
      "grad_norm": 0.20058633387088776,
      "learning_rate": 2.1485403696324935e-06,
      "log_odds_chosen": 0.9557737708091736,
      "log_odds_ratio": -0.3273327946662903,
      "logits/chosen": -0.5809505581855774,
      "logits/rejected": -1.6531572341918945,
      "logps/chosen": -1.4407601356506348,
      "logps/rejected": -2.23899245262146,
      "loss": 1.5311,
      "nll_loss": 1.4983248710632324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14407603442668915,
      "rewards/margins": 0.07982321083545685,
      "rewards/rejected": -0.2238992303609848,
      "step": 644
    },
    {
      "epoch": 1.7787366240938902,
      "grad_norm": 0.22395431995391846,
      "learning_rate": 2.140607513628296e-06,
      "log_odds_chosen": 1.0929580926895142,
      "log_odds_ratio": -0.3005385994911194,
      "logits/chosen": -0.5677430033683777,
      "logits/rejected": -1.6676310300827026,
      "logps/chosen": -1.491722583770752,
      "logps/rejected": -2.4220542907714844,
      "loss": 1.5717,
      "nll_loss": 1.541687250137329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14917226135730743,
      "rewards/margins": 0.09303317964076996,
      "rewards/rejected": -0.2422054558992386,
      "step": 645
    },
    {
      "epoch": 1.781498101484294,
      "grad_norm": 0.19423379004001617,
      "learning_rate": 2.1326783509379555e-06,
      "log_odds_chosen": 1.230663537979126,
      "log_odds_ratio": -0.27645426988601685,
      "logits/chosen": -0.6330420970916748,
      "logits/rejected": -1.7341974973678589,
      "logps/chosen": -1.33867347240448,
      "logps/rejected": -2.3714494705200195,
      "loss": 1.4287,
      "nll_loss": 1.401092290878296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13386735320091248,
      "rewards/margins": 0.10327759385108948,
      "rewards/rejected": -0.23714493215084076,
      "step": 646
    },
    {
      "epoch": 1.784259578874698,
      "grad_norm": 0.21576644480228424,
      "learning_rate": 2.1247529630458856e-06,
      "log_odds_chosen": 0.9605802297592163,
      "log_odds_ratio": -0.33712220191955566,
      "logits/chosen": -0.5994086265563965,
      "logits/rejected": -1.5157111883163452,
      "logps/chosen": -1.4738686084747314,
      "logps/rejected": -2.2864460945129395,
      "loss": 1.5531,
      "nll_loss": 1.5193928480148315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14738686382770538,
      "rewards/margins": 0.08125776797533035,
      "rewards/rejected": -0.22864462435245514,
      "step": 647
    },
    {
      "epoch": 1.7870210562651019,
      "grad_norm": 0.2162182778120041,
      "learning_rate": 2.11683143139771e-06,
      "log_odds_chosen": 1.0968060493469238,
      "log_odds_ratio": -0.3004920184612274,
      "logits/chosen": -0.5496225357055664,
      "logits/rejected": -1.5592927932739258,
      "logps/chosen": -1.4152264595031738,
      "logps/rejected": -2.3394081592559814,
      "loss": 1.5072,
      "nll_loss": 1.4771496057510376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14152266085147858,
      "rewards/margins": 0.09241816401481628,
      "rewards/rejected": -0.23394080996513367,
      "step": 648
    },
    {
      "epoch": 1.7897825336555058,
      "grad_norm": 0.20863664150238037,
      "learning_rate": 2.1089138373994226e-06,
      "log_odds_chosen": 1.085922122001648,
      "log_odds_ratio": -0.2969967722892761,
      "logits/chosen": -0.589851975440979,
      "logits/rejected": -1.8017020225524902,
      "logps/chosen": -1.4874114990234375,
      "logps/rejected": -2.4139440059661865,
      "loss": 1.5819,
      "nll_loss": 1.552212119102478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14874114096164703,
      "rewards/margins": 0.09265326708555222,
      "rewards/rejected": -0.24139441549777985,
      "step": 649
    },
    {
      "epoch": 1.7925440110459094,
      "grad_norm": 0.2144482284784317,
      "learning_rate": 2.1010002624165528e-06,
      "log_odds_chosen": 0.886896550655365,
      "log_odds_ratio": -0.3648533225059509,
      "logits/chosen": -0.5143107175827026,
      "logits/rejected": -1.4646867513656616,
      "logps/chosen": -1.5178825855255127,
      "logps/rejected": -2.2712976932525635,
      "loss": 1.5934,
      "nll_loss": 1.5568923950195312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15178826451301575,
      "rewards/margins": 0.07534152269363403,
      "rewards/rejected": -0.2271297723054886,
      "step": 650
    },
    {
      "epoch": 1.7953054884363135,
      "grad_norm": 0.19179891049861908,
      "learning_rate": 2.0930907877733255e-06,
      "log_odds_chosen": 1.1237578392028809,
      "log_odds_ratio": -0.2867814004421234,
      "logits/chosen": -0.43964463472366333,
      "logits/rejected": -1.8275145292282104,
      "logps/chosen": -1.4230607748031616,
      "logps/rejected": -2.369021415710449,
      "loss": 1.4899,
      "nll_loss": 1.4611868858337402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14230608940124512,
      "rewards/margins": 0.09459604322910309,
      "rewards/rejected": -0.2369021326303482,
      "step": 651
    },
    {
      "epoch": 1.7980669658267172,
      "grad_norm": 0.19488249719142914,
      "learning_rate": 2.085185494751831e-06,
      "log_odds_chosen": 1.1252477169036865,
      "log_odds_ratio": -0.2942362427711487,
      "logits/chosen": -0.526394248008728,
      "logits/rejected": -1.643446683883667,
      "logps/chosen": -1.3340688943862915,
      "logps/rejected": -2.267882823944092,
      "loss": 1.4353,
      "nll_loss": 1.4058688879013062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1334068924188614,
      "rewards/margins": 0.09338139742612839,
      "rewards/rejected": -0.22678828239440918,
      "step": 652
    },
    {
      "epoch": 1.8008284432171213,
      "grad_norm": 0.20605449378490448,
      "learning_rate": 2.077284464591187e-06,
      "log_odds_chosen": 1.0873126983642578,
      "log_odds_ratio": -0.3010372817516327,
      "logits/chosen": -0.47243842482566833,
      "logits/rejected": -1.7291725873947144,
      "logps/chosen": -1.4694960117340088,
      "logps/rejected": -2.3931758403778076,
      "loss": 1.5427,
      "nll_loss": 1.5125980377197266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14694960415363312,
      "rewards/margins": 0.092367984354496,
      "rewards/rejected": -0.23931758105754852,
      "step": 653
    },
    {
      "epoch": 1.803589920607525,
      "grad_norm": 0.201577290892601,
      "learning_rate": 2.069387778486703e-06,
      "log_odds_chosen": 1.1571605205535889,
      "log_odds_ratio": -0.28402575850486755,
      "logits/chosen": -0.5961681008338928,
      "logits/rejected": -1.9189343452453613,
      "logps/chosen": -1.5506150722503662,
      "logps/rejected": -2.54967999458313,
      "loss": 1.6219,
      "nll_loss": 1.5935449600219727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1550614982843399,
      "rewards/margins": 0.09990648925304413,
      "rewards/rejected": -0.25496798753738403,
      "step": 654
    },
    {
      "epoch": 1.8063513979979289,
      "grad_norm": 0.21721962094306946,
      "learning_rate": 2.0614955175890464e-06,
      "log_odds_chosen": 1.1083087921142578,
      "log_odds_ratio": -0.2993934750556946,
      "logits/chosen": -0.6008000373840332,
      "logits/rejected": -1.8069735765457153,
      "logps/chosen": -1.4827308654785156,
      "logps/rejected": -2.4276866912841797,
      "loss": 1.5456,
      "nll_loss": 1.515679955482483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14827308058738708,
      "rewards/margins": 0.09449561685323715,
      "rewards/rejected": -0.24276868999004364,
      "step": 655
    },
    {
      "epoch": 1.8091128753883328,
      "grad_norm": 0.19219523668289185,
      "learning_rate": 2.053607763003409e-06,
      "log_odds_chosen": 1.2417378425598145,
      "log_odds_ratio": -0.2833240032196045,
      "logits/chosen": -0.5437658429145813,
      "logits/rejected": -1.7058812379837036,
      "logps/chosen": -1.4310146570205688,
      "logps/rejected": -2.4930710792541504,
      "loss": 1.5176,
      "nll_loss": 1.4892582893371582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14310148358345032,
      "rewards/margins": 0.10620564222335815,
      "rewards/rejected": -0.24930711090564728,
      "step": 656
    },
    {
      "epoch": 1.8118743527787367,
      "grad_norm": 0.1976533681154251,
      "learning_rate": 2.045724595788673e-06,
      "log_odds_chosen": 1.2406878471374512,
      "log_odds_ratio": -0.2589613199234009,
      "logits/chosen": -0.5714414119720459,
      "logits/rejected": -1.9800139665603638,
      "logps/chosen": -1.4148036241531372,
      "logps/rejected": -2.4669976234436035,
      "loss": 1.4927,
      "nll_loss": 1.4667580127716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14148037135601044,
      "rewards/margins": 0.10521939396858215,
      "rewards/rejected": -0.2466997504234314,
      "step": 657
    },
    {
      "epoch": 1.8146358301691405,
      "grad_norm": 0.2066175788640976,
      "learning_rate": 2.037846096956578e-06,
      "log_odds_chosen": 1.1821521520614624,
      "log_odds_ratio": -0.27946531772613525,
      "logits/chosen": -0.5792191028594971,
      "logits/rejected": -1.5947271585464478,
      "logps/chosen": -1.4168436527252197,
      "logps/rejected": -2.4209742546081543,
      "loss": 1.5035,
      "nll_loss": 1.4755176305770874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1416843682527542,
      "rewards/margins": 0.10041306912899017,
      "rewards/rejected": -0.24209743738174438,
      "step": 658
    },
    {
      "epoch": 1.8173973075595442,
      "grad_norm": 0.19449689984321594,
      "learning_rate": 2.0299723474708917e-06,
      "log_odds_chosen": 1.0754530429840088,
      "log_odds_ratio": -0.30575478076934814,
      "logits/chosen": -0.522871732711792,
      "logits/rejected": -1.6677809953689575,
      "logps/chosen": -1.405239224433899,
      "logps/rejected": -2.306657075881958,
      "loss": 1.4803,
      "nll_loss": 1.4497466087341309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14052392542362213,
      "rewards/margins": 0.09014178067445755,
      "rewards/rejected": -0.23066571354866028,
      "step": 659
    },
    {
      "epoch": 1.8201587849499483,
      "grad_norm": 0.20105119049549103,
      "learning_rate": 2.02210342824657e-06,
      "log_odds_chosen": 1.2487144470214844,
      "log_odds_ratio": -0.26469725370407104,
      "logits/chosen": -0.6605066061019897,
      "logits/rejected": -1.9393763542175293,
      "logps/chosen": -1.4192674160003662,
      "logps/rejected": -2.4731574058532715,
      "loss": 1.5006,
      "nll_loss": 1.4741599559783936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14192675054073334,
      "rewards/margins": 0.10538898408412933,
      "rewards/rejected": -0.24731573462486267,
      "step": 660
    },
    {
      "epoch": 1.822920262340352,
      "grad_norm": 0.20803654193878174,
      "learning_rate": 2.0142394201489334e-06,
      "log_odds_chosen": 1.2532217502593994,
      "log_odds_ratio": -0.2540588676929474,
      "logits/chosen": -0.5191413164138794,
      "logits/rejected": -1.8599536418914795,
      "logps/chosen": -1.4484375715255737,
      "logps/rejected": -2.5184717178344727,
      "loss": 1.513,
      "nll_loss": 1.4876114130020142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14484377205371857,
      "rewards/margins": 0.10700343549251556,
      "rewards/rejected": -0.25184717774391174,
      "step": 661
    },
    {
      "epoch": 1.825681739730756,
      "grad_norm": 0.2015330046415329,
      "learning_rate": 2.0063804039928324e-06,
      "log_odds_chosen": 1.2455298900604248,
      "log_odds_ratio": -0.26516517996788025,
      "logits/chosen": -0.5362703204154968,
      "logits/rejected": -1.6895490884780884,
      "logps/chosen": -1.4490387439727783,
      "logps/rejected": -2.5178382396698,
      "loss": 1.5259,
      "nll_loss": 1.49942946434021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14490386843681335,
      "rewards/margins": 0.10687996447086334,
      "rewards/rejected": -0.2517838180065155,
      "step": 662
    },
    {
      "epoch": 1.8284432171211598,
      "grad_norm": 0.21101270616054535,
      "learning_rate": 1.9985264605418185e-06,
      "log_odds_chosen": 1.265969157218933,
      "log_odds_ratio": -0.26632457971572876,
      "logits/chosen": -0.624642550945282,
      "logits/rejected": -1.8359934091567993,
      "logps/chosen": -1.4265787601470947,
      "logps/rejected": -2.5020813941955566,
      "loss": 1.5126,
      "nll_loss": 1.4859540462493896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14265787601470947,
      "rewards/margins": 0.10755026340484619,
      "rewards/rejected": -0.25020813941955566,
      "step": 663
    },
    {
      "epoch": 1.8312046945115636,
      "grad_norm": 0.21531936526298523,
      "learning_rate": 1.9906776705073113e-06,
      "log_odds_chosen": 1.3287895917892456,
      "log_odds_ratio": -0.24583633244037628,
      "logits/chosen": -0.6203726530075073,
      "logits/rejected": -1.9401627779006958,
      "logps/chosen": -1.4545714855194092,
      "logps/rejected": -2.5981836318969727,
      "loss": 1.5272,
      "nll_loss": 1.5025845766067505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14545713365077972,
      "rewards/margins": 0.1143612340092659,
      "rewards/rejected": -0.2598183751106262,
      "step": 664
    },
    {
      "epoch": 1.8339661719019675,
      "grad_norm": 0.20089717209339142,
      "learning_rate": 1.982834114547773e-06,
      "log_odds_chosen": 1.1791274547576904,
      "log_odds_ratio": -0.27245157957077026,
      "logits/chosen": -0.6062506437301636,
      "logits/rejected": -1.8836218118667603,
      "logps/chosen": -1.440775752067566,
      "logps/rejected": -2.4392642974853516,
      "loss": 1.524,
      "nll_loss": 1.4968045949935913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1440775841474533,
      "rewards/margins": 0.09984884411096573,
      "rewards/rejected": -0.24392642080783844,
      "step": 665
    },
    {
      "epoch": 1.8367276492923714,
      "grad_norm": 0.21053946018218994,
      "learning_rate": 1.9749958732678766e-06,
      "log_odds_chosen": 1.1956483125686646,
      "log_odds_ratio": -0.2698996663093567,
      "logits/chosen": -0.6107760071754456,
      "logits/rejected": -1.7991489171981812,
      "logps/chosen": -1.4191172122955322,
      "logps/rejected": -2.4300425052642822,
      "loss": 1.5046,
      "nll_loss": 1.4775981903076172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14191171526908875,
      "rewards/margins": 0.10109255462884903,
      "rewards/rejected": -0.24300426244735718,
      "step": 666
    },
    {
      "epoch": 1.8394891266827753,
      "grad_norm": 0.23768320679664612,
      "learning_rate": 1.967163027217679e-06,
      "log_odds_chosen": 1.1234877109527588,
      "log_odds_ratio": -0.29781588912010193,
      "logits/chosen": -0.6074894666671753,
      "logits/rejected": -1.5648796558380127,
      "logps/chosen": -1.4478806257247925,
      "logps/rejected": -2.397280216217041,
      "loss": 1.5416,
      "nll_loss": 1.5118470191955566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14478805661201477,
      "rewards/margins": 0.0949399471282959,
      "rewards/rejected": -0.23972800374031067,
      "step": 667
    },
    {
      "epoch": 1.8422506040731792,
      "grad_norm": 0.22009487450122833,
      "learning_rate": 1.9593356568917916e-06,
      "log_odds_chosen": 1.0102051496505737,
      "log_odds_ratio": -0.3210904896259308,
      "logits/chosen": -0.5804159641265869,
      "logits/rejected": -1.5628560781478882,
      "logps/chosen": -1.5433801412582397,
      "logps/rejected": -2.4113738536834717,
      "loss": 1.6194,
      "nll_loss": 1.5873008966445923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543380171060562,
      "rewards/margins": 0.08679936826229095,
      "rewards/rejected": -0.24113738536834717,
      "step": 668
    },
    {
      "epoch": 1.845012081463583,
      "grad_norm": 0.20616894960403442,
      "learning_rate": 1.951513842728556e-06,
      "log_odds_chosen": 1.106505036354065,
      "log_odds_ratio": -0.2891699969768524,
      "logits/chosen": -0.5833257436752319,
      "logits/rejected": -1.9531350135803223,
      "logps/chosen": -1.4294037818908691,
      "logps/rejected": -2.3592071533203125,
      "loss": 1.5097,
      "nll_loss": 1.4808082580566406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14294040203094482,
      "rewards/margins": 0.09298031032085419,
      "rewards/rejected": -0.235920712351799,
      "step": 669
    },
    {
      "epoch": 1.8477735588539868,
      "grad_norm": 0.21930097043514252,
      "learning_rate": 1.9436976651092143e-06,
      "log_odds_chosen": 1.1857033967971802,
      "log_odds_ratio": -0.27113771438598633,
      "logits/chosen": -0.6338096261024475,
      "logits/rejected": -1.7122553586959839,
      "logps/chosen": -1.424892544746399,
      "logps/rejected": -2.4289939403533936,
      "loss": 1.5164,
      "nll_loss": 1.4892981052398682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1424892544746399,
      "rewards/margins": 0.10041015595197678,
      "rewards/rejected": -0.24289940297603607,
      "step": 670
    },
    {
      "epoch": 1.8505350362443909,
      "grad_norm": 0.20231999456882477,
      "learning_rate": 1.935887204357085e-06,
      "log_odds_chosen": 1.3218573331832886,
      "log_odds_ratio": -0.25989198684692383,
      "logits/chosen": -0.5880028009414673,
      "logits/rejected": -1.7775429487228394,
      "logps/chosen": -1.4524977207183838,
      "logps/rejected": -2.591127395629883,
      "loss": 1.5228,
      "nll_loss": 1.4968382120132446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14524978399276733,
      "rewards/margins": 0.11386297643184662,
      "rewards/rejected": -0.25911277532577515,
      "step": 671
    },
    {
      "epoch": 1.8532965136347945,
      "grad_norm": 0.2004518359899521,
      "learning_rate": 1.928082540736737e-06,
      "log_odds_chosen": 1.3334486484527588,
      "log_odds_ratio": -0.2543681859970093,
      "logits/chosen": -0.5984243750572205,
      "logits/rejected": -1.6906664371490479,
      "logps/chosen": -1.4101768732070923,
      "logps/rejected": -2.5547547340393066,
      "loss": 1.5006,
      "nll_loss": 1.4751571416854858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14101769030094147,
      "rewards/margins": 0.11445777118206024,
      "rewards/rejected": -0.2554754912853241,
      "step": 672
    },
    {
      "epoch": 1.8560579910251986,
      "grad_norm": 0.21709613502025604,
      "learning_rate": 1.920283754453164e-06,
      "log_odds_chosen": 1.3128209114074707,
      "log_odds_ratio": -0.2452673465013504,
      "logits/chosen": -0.5661304593086243,
      "logits/rejected": -1.7389955520629883,
      "logps/chosen": -1.3728487491607666,
      "logps/rejected": -2.478579044342041,
      "loss": 1.4466,
      "nll_loss": 1.422105312347412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13728488981723785,
      "rewards/margins": 0.11057302355766296,
      "rewards/rejected": -0.24785789847373962,
      "step": 673
    },
    {
      "epoch": 1.8588194684156023,
      "grad_norm": 0.2150663137435913,
      "learning_rate": 1.912490925650962e-06,
      "log_odds_chosen": 1.348259687423706,
      "log_odds_ratio": -0.23330648243427277,
      "logits/chosen": -0.6710218191146851,
      "logits/rejected": -1.9455617666244507,
      "logps/chosen": -1.4661921262741089,
      "logps/rejected": -2.6251721382141113,
      "loss": 1.5275,
      "nll_loss": 1.5041943788528442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14661921560764313,
      "rewards/margins": 0.1158980131149292,
      "rewards/rejected": -0.26251721382141113,
      "step": 674
    },
    {
      "epoch": 1.8615809458060062,
      "grad_norm": 0.2174362689256668,
      "learning_rate": 1.9047041344135045e-06,
      "log_odds_chosen": 1.003183126449585,
      "log_odds_ratio": -0.3182450830936432,
      "logits/chosen": -0.6303192973136902,
      "logits/rejected": -1.8845601081848145,
      "logps/chosen": -1.5408192873001099,
      "logps/rejected": -2.400413990020752,
      "loss": 1.6063,
      "nll_loss": 1.5744513273239136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15408192574977875,
      "rewards/margins": 0.08595946431159973,
      "rewards/rejected": -0.24004140496253967,
      "step": 675
    },
    {
      "epoch": 1.86434242319641,
      "grad_norm": 0.1918780356645584,
      "learning_rate": 1.8969234607621187e-06,
      "log_odds_chosen": 1.1956127882003784,
      "log_odds_ratio": -0.2732754349708557,
      "logits/chosen": -0.6226102113723755,
      "logits/rejected": -1.8827693462371826,
      "logps/chosen": -1.3975963592529297,
      "logps/rejected": -2.3993749618530273,
      "loss": 1.4784,
      "nll_loss": 1.4511014223098755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1397596299648285,
      "rewards/margins": 0.10017785429954529,
      "rewards/rejected": -0.23993748426437378,
      "step": 676
    },
    {
      "epoch": 1.867103900586814,
      "grad_norm": 0.19374889135360718,
      "learning_rate": 1.8891489846552645e-06,
      "log_odds_chosen": 1.1678223609924316,
      "log_odds_ratio": -0.2740553915500641,
      "logits/chosen": -0.5673727989196777,
      "logits/rejected": -1.970065951347351,
      "logps/chosen": -1.3401387929916382,
      "logps/rejected": -2.310602903366089,
      "loss": 1.4337,
      "nll_loss": 1.4062803983688354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13401387631893158,
      "rewards/margins": 0.09704641252756119,
      "rewards/rejected": -0.23106031119823456,
      "step": 677
    },
    {
      "epoch": 1.8698653779772179,
      "grad_norm": 0.20325230062007904,
      "learning_rate": 1.8813807859877147e-06,
      "log_odds_chosen": 1.0683099031448364,
      "log_odds_ratio": -0.30433735251426697,
      "logits/chosen": -0.5034862756729126,
      "logits/rejected": -1.5476434230804443,
      "logps/chosen": -1.4672025442123413,
      "logps/rejected": -2.3724498748779297,
      "loss": 1.549,
      "nll_loss": 1.5185987949371338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1467202603816986,
      "rewards/margins": 0.09052471071481705,
      "rewards/rejected": -0.23724496364593506,
      "step": 678
    },
    {
      "epoch": 1.8726268553676215,
      "grad_norm": 0.19582170248031616,
      "learning_rate": 1.8736189445897268e-06,
      "log_odds_chosen": 1.1498327255249023,
      "log_odds_ratio": -0.28108832240104675,
      "logits/chosen": -0.6858939528465271,
      "logits/rejected": -1.8512457609176636,
      "logps/chosen": -1.3199814558029175,
      "logps/rejected": -2.2606868743896484,
      "loss": 1.4063,
      "nll_loss": 1.3781564235687256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13199815154075623,
      "rewards/margins": 0.09407053887844086,
      "rewards/rejected": -0.22606870532035828,
      "step": 679
    },
    {
      "epoch": 1.8753883327580256,
      "grad_norm": 0.19537782669067383,
      "learning_rate": 1.865863540226232e-06,
      "log_odds_chosen": 1.2928959131240845,
      "log_odds_ratio": -0.24565257132053375,
      "logits/chosen": -0.5815032124519348,
      "logits/rejected": -1.7621716260910034,
      "logps/chosen": -1.3540948629379272,
      "logps/rejected": -2.4354796409606934,
      "loss": 1.4379,
      "nll_loss": 1.4132860898971558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13540948927402496,
      "rewards/margins": 0.10813847184181213,
      "rewards/rejected": -0.2435479760169983,
      "step": 680
    },
    {
      "epoch": 1.8781498101484293,
      "grad_norm": 0.20229066908359528,
      "learning_rate": 1.8581146525960093e-06,
      "log_odds_chosen": 1.1232048273086548,
      "log_odds_ratio": -0.2937353551387787,
      "logits/chosen": -0.637298583984375,
      "logits/rejected": -1.6448088884353638,
      "logps/chosen": -1.3534826040267944,
      "logps/rejected": -2.2935564517974854,
      "loss": 1.4484,
      "nll_loss": 1.4190561771392822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13534826040267944,
      "rewards/margins": 0.09400738775730133,
      "rewards/rejected": -0.22935563325881958,
      "step": 681
    },
    {
      "epoch": 1.8809112875388334,
      "grad_norm": 0.2125195562839508,
      "learning_rate": 1.8503723613308683e-06,
      "log_odds_chosen": 1.274790644645691,
      "log_odds_ratio": -0.26285573840141296,
      "logits/chosen": -0.5560697317123413,
      "logits/rejected": -1.9034526348114014,
      "logps/chosen": -1.442688226699829,
      "logps/rejected": -2.5351686477661133,
      "loss": 1.5013,
      "nll_loss": 1.4750303030014038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14426881074905396,
      "rewards/margins": 0.1092480719089508,
      "rewards/rejected": -0.25351691246032715,
      "step": 682
    },
    {
      "epoch": 1.883672764929237,
      "grad_norm": 0.20290443301200867,
      "learning_rate": 1.8426367459948306e-06,
      "log_odds_chosen": 1.2297403812408447,
      "log_odds_ratio": -0.2659325897693634,
      "logits/chosen": -0.5875911712646484,
      "logits/rejected": -1.781616449356079,
      "logps/chosen": -1.4287062883377075,
      "logps/rejected": -2.475780487060547,
      "loss": 1.507,
      "nll_loss": 1.4804446697235107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14287063479423523,
      "rewards/margins": 0.10470742732286453,
      "rewards/rejected": -0.24757805466651917,
      "step": 683
    },
    {
      "epoch": 1.886434242319641,
      "grad_norm": 0.22338153421878815,
      "learning_rate": 1.8349078860833125e-06,
      "log_odds_chosen": 1.1323693990707397,
      "log_odds_ratio": -0.28367865085601807,
      "logits/chosen": -0.6488045454025269,
      "logits/rejected": -1.7948299646377563,
      "logps/chosen": -1.4490909576416016,
      "logps/rejected": -2.4043657779693604,
      "loss": 1.5199,
      "nll_loss": 1.491485595703125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1449090987443924,
      "rewards/margins": 0.09552749991416931,
      "rewards/rejected": -0.2404365837574005,
      "step": 684
    },
    {
      "epoch": 1.8891957197100449,
      "grad_norm": 0.21708610653877258,
      "learning_rate": 1.827185861022308e-06,
      "log_odds_chosen": 1.321728229522705,
      "log_odds_ratio": -0.2453889697790146,
      "logits/chosen": -0.5850996375083923,
      "logits/rejected": -1.7692553997039795,
      "logps/chosen": -1.4336585998535156,
      "logps/rejected": -2.565013885498047,
      "loss": 1.4925,
      "nll_loss": 1.467954158782959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14336584508419037,
      "rewards/margins": 0.11313553899526596,
      "rewards/rejected": -0.2565014064311981,
      "step": 685
    },
    {
      "epoch": 1.8919571971004487,
      "grad_norm": 0.20613998174667358,
      "learning_rate": 1.8194707501675724e-06,
      "log_odds_chosen": 1.4058480262756348,
      "log_odds_ratio": -0.2284797877073288,
      "logits/chosen": -0.688378632068634,
      "logits/rejected": -1.8188039064407349,
      "logps/chosen": -1.377896785736084,
      "logps/rejected": -2.5687503814697266,
      "loss": 1.4495,
      "nll_loss": 1.4266870021820068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13778966665267944,
      "rewards/margins": 0.119085393846035,
      "rewards/rejected": -0.25687506794929504,
      "step": 686
    },
    {
      "epoch": 1.8947186744908526,
      "grad_norm": 0.22078323364257812,
      "learning_rate": 1.8117626328038069e-06,
      "log_odds_chosen": 1.1295044422149658,
      "log_odds_ratio": -0.28533411026000977,
      "logits/chosen": -0.5322979688644409,
      "logits/rejected": -1.77653968334198,
      "logps/chosen": -1.4119873046875,
      "logps/rejected": -2.3596487045288086,
      "loss": 1.4847,
      "nll_loss": 1.4561336040496826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14119873940944672,
      "rewards/margins": 0.09476611763238907,
      "rewards/rejected": -0.23596486449241638,
      "step": 687
    },
    {
      "epoch": 1.8974801518812565,
      "grad_norm": 0.20481730997562408,
      "learning_rate": 1.8040615881438425e-06,
      "log_odds_chosen": 1.02912175655365,
      "log_odds_ratio": -0.3101516664028168,
      "logits/chosen": -0.6844128370285034,
      "logits/rejected": -1.748498797416687,
      "logps/chosen": -1.4500445127487183,
      "logps/rejected": -2.318002700805664,
      "loss": 1.5398,
      "nll_loss": 1.5087554454803467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14500445127487183,
      "rewards/margins": 0.08679580688476562,
      "rewards/rejected": -0.23180025815963745,
      "step": 688
    },
    {
      "epoch": 1.9002416292716604,
      "grad_norm": 0.21134057641029358,
      "learning_rate": 1.7963676953278292e-06,
      "log_odds_chosen": 1.2188888788223267,
      "log_odds_ratio": -0.26270338892936707,
      "logits/chosen": -0.6463155150413513,
      "logits/rejected": -1.7863259315490723,
      "logps/chosen": -1.4500489234924316,
      "logps/rejected": -2.4897103309631348,
      "loss": 1.5305,
      "nll_loss": 1.5042182207107544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14500491321086884,
      "rewards/margins": 0.1039661318063736,
      "rewards/rejected": -0.24897105991840363,
      "step": 689
    },
    {
      "epoch": 1.903003106662064,
      "grad_norm": 0.21317671239376068,
      "learning_rate": 1.7886810334224192e-06,
      "log_odds_chosen": 1.2021043300628662,
      "log_odds_ratio": -0.2723836600780487,
      "logits/chosen": -0.6922492980957031,
      "logits/rejected": -1.89365816116333,
      "logps/chosen": -1.414392352104187,
      "logps/rejected": -2.432486057281494,
      "loss": 1.4898,
      "nll_loss": 1.4626052379608154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14143922924995422,
      "rewards/margins": 0.10180937498807907,
      "rewards/rejected": -0.2432485967874527,
      "step": 690
    },
    {
      "epoch": 1.9057645840524682,
      "grad_norm": 0.1928180307149887,
      "learning_rate": 1.781001681419957e-06,
      "log_odds_chosen": 1.4509155750274658,
      "log_odds_ratio": -0.24097022414207458,
      "logits/chosen": -0.5879383087158203,
      "logits/rejected": -1.8242533206939697,
      "logps/chosen": -1.3567006587982178,
      "logps/rejected": -2.600719451904297,
      "loss": 1.4364,
      "nll_loss": 1.4122999906539917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13567008078098297,
      "rewards/margins": 0.12440188229084015,
      "rewards/rejected": -0.2600719630718231,
      "step": 691
    },
    {
      "epoch": 1.9085260614428718,
      "grad_norm": 0.2123909443616867,
      "learning_rate": 1.7733297182376663e-06,
      "log_odds_chosen": 1.194056510925293,
      "log_odds_ratio": -0.2803875803947449,
      "logits/chosen": -0.7072063088417053,
      "logits/rejected": -1.670470118522644,
      "logps/chosen": -1.3613109588623047,
      "logps/rejected": -2.3637614250183105,
      "loss": 1.4595,
      "nll_loss": 1.431486964225769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13613110780715942,
      "rewards/margins": 0.10024504363536835,
      "rewards/rejected": -0.23637613654136658,
      "step": 692
    },
    {
      "epoch": 1.911287538833276,
      "grad_norm": 0.2346637099981308,
      "learning_rate": 1.7656652227168384e-06,
      "log_odds_chosen": 1.2246406078338623,
      "log_odds_ratio": -0.2636195421218872,
      "logits/chosen": -0.6246435046195984,
      "logits/rejected": -1.6411041021347046,
      "logps/chosen": -1.4360047578811646,
      "logps/rejected": -2.4795889854431152,
      "loss": 1.5046,
      "nll_loss": 1.4782655239105225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1436004787683487,
      "rewards/margins": 0.10435841977596283,
      "rewards/rejected": -0.24795889854431152,
      "step": 693
    },
    {
      "epoch": 1.9140490162236796,
      "grad_norm": 0.21563811600208282,
      "learning_rate": 1.7580082736220235e-06,
      "log_odds_chosen": 1.351541519165039,
      "log_odds_ratio": -0.24304015934467316,
      "logits/chosen": -0.6352071762084961,
      "logits/rejected": -1.7275444269180298,
      "logps/chosen": -1.4102025032043457,
      "logps/rejected": -2.5561411380767822,
      "loss": 1.4874,
      "nll_loss": 1.4630484580993652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1410202533006668,
      "rewards/margins": 0.11459386348724365,
      "rewards/rejected": -0.25561410188674927,
      "step": 694
    },
    {
      "epoch": 1.9168104936140835,
      "grad_norm": 0.23292036354541779,
      "learning_rate": 1.750358949640221e-06,
      "log_odds_chosen": 1.0779083967208862,
      "log_odds_ratio": -0.3038214147090912,
      "logits/chosen": -0.6787149906158447,
      "logits/rejected": -1.33951997756958,
      "logps/chosen": -1.389949083328247,
      "logps/rejected": -2.295311212539673,
      "loss": 1.4843,
      "nll_loss": 1.453916072845459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13899490237236023,
      "rewards/margins": 0.0905362069606781,
      "rewards/rejected": -0.22953110933303833,
      "step": 695
    },
    {
      "epoch": 1.9195719710044874,
      "grad_norm": 0.20314742624759674,
      "learning_rate": 1.74271732938007e-06,
      "log_odds_chosen": 1.121297836303711,
      "log_odds_ratio": -0.2867622375488281,
      "logits/chosen": -0.6550911068916321,
      "logits/rejected": -1.6976155042648315,
      "logps/chosen": -1.3656046390533447,
      "logps/rejected": -2.2984426021575928,
      "loss": 1.4571,
      "nll_loss": 1.428386926651001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13656045496463776,
      "rewards/margins": 0.09328380227088928,
      "rewards/rejected": -0.22984427213668823,
      "step": 696
    },
    {
      "epoch": 1.9223334483948913,
      "grad_norm": 0.22091306746006012,
      "learning_rate": 1.7350834913710414e-06,
      "log_odds_chosen": 1.366137146949768,
      "log_odds_ratio": -0.23213300108909607,
      "logits/chosen": -0.7029794454574585,
      "logits/rejected": -1.7516682147979736,
      "logps/chosen": -1.3843116760253906,
      "logps/rejected": -2.5412652492523193,
      "loss": 1.4653,
      "nll_loss": 1.4420487880706787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13843117654323578,
      "rewards/margins": 0.11569535732269287,
      "rewards/rejected": -0.25412654876708984,
      "step": 697
    },
    {
      "epoch": 1.9250949257852952,
      "grad_norm": 0.21280437707901,
      "learning_rate": 1.7274575140626318e-06,
      "log_odds_chosen": 1.4392701387405396,
      "log_odds_ratio": -0.21989382803440094,
      "logits/chosen": -0.6601370573043823,
      "logits/rejected": -1.899329423904419,
      "logps/chosen": -1.3539754152297974,
      "logps/rejected": -2.572890281677246,
      "loss": 1.4257,
      "nll_loss": 1.403670072555542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1353975534439087,
      "rewards/margins": 0.12189144641160965,
      "rewards/rejected": -0.25728899240493774,
      "step": 698
    },
    {
      "epoch": 1.9278564031756988,
      "grad_norm": 0.22868065536022186,
      "learning_rate": 1.7198394758235576e-06,
      "log_odds_chosen": 1.0325840711593628,
      "log_odds_ratio": -0.3077024519443512,
      "logits/chosen": -0.7413440346717834,
      "logits/rejected": -1.7055625915527344,
      "logps/chosen": -1.4271225929260254,
      "logps/rejected": -2.2928249835968018,
      "loss": 1.5059,
      "nll_loss": 1.4751147031784058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1427122801542282,
      "rewards/margins": 0.08657021820545197,
      "rewards/rejected": -0.2292824685573578,
      "step": 699
    },
    {
      "epoch": 1.930617880566103,
      "grad_norm": 0.1975845843553543,
      "learning_rate": 1.7122294549409486e-06,
      "log_odds_chosen": 1.1576435565948486,
      "log_odds_ratio": -0.2774282693862915,
      "logits/chosen": -0.6693485975265503,
      "logits/rejected": -1.845128059387207,
      "logps/chosen": -1.4216663837432861,
      "logps/rejected": -2.3981292247772217,
      "loss": 1.4897,
      "nll_loss": 1.4619686603546143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1421666294336319,
      "rewards/margins": 0.0976463034749031,
      "rewards/rejected": -0.2398129254579544,
      "step": 700
    },
    {
      "epoch": 1.9333793579565066,
      "grad_norm": 0.2156144380569458,
      "learning_rate": 1.704627529619543e-06,
      "log_odds_chosen": 1.0374083518981934,
      "log_odds_ratio": -0.313325971364975,
      "logits/chosen": -0.6931014060974121,
      "logits/rejected": -1.77117919921875,
      "logps/chosen": -1.4780950546264648,
      "logps/rejected": -2.358729839324951,
      "loss": 1.5628,
      "nll_loss": 1.531511902809143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14780952036380768,
      "rewards/margins": 0.08806345611810684,
      "rewards/rejected": -0.23587298393249512,
      "step": 701
    },
    {
      "epoch": 1.9361408353469107,
      "grad_norm": 0.20054619014263153,
      "learning_rate": 1.6970337779808862e-06,
      "log_odds_chosen": 1.087822437286377,
      "log_odds_ratio": -0.2988903224468231,
      "logits/chosen": -0.6482985615730286,
      "logits/rejected": -1.7110782861709595,
      "logps/chosen": -1.4709477424621582,
      "logps/rejected": -2.3889198303222656,
      "loss": 1.5488,
      "nll_loss": 1.5189554691314697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14709478616714478,
      "rewards/margins": 0.09179720282554626,
      "rewards/rejected": -0.23889197409152985,
      "step": 702
    },
    {
      "epoch": 1.9389023127373144,
      "grad_norm": 0.2022159993648529,
      "learning_rate": 1.689448278062525e-06,
      "log_odds_chosen": 1.2595505714416504,
      "log_odds_ratio": -0.25268253684043884,
      "logits/chosen": -0.6973377466201782,
      "logits/rejected": -1.8542219400405884,
      "logps/chosen": -1.4063862562179565,
      "logps/rejected": -2.4727578163146973,
      "loss": 1.4933,
      "nll_loss": 1.4680652618408203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14063863456249237,
      "rewards/margins": 0.10663717240095139,
      "rewards/rejected": -0.24727579951286316,
      "step": 703
    },
    {
      "epoch": 1.9416637901277183,
      "grad_norm": 0.768374502658844,
      "learning_rate": 1.681871107817208e-06,
      "log_odds_chosen": 1.2967448234558105,
      "log_odds_ratio": -0.2469194382429123,
      "logits/chosen": -0.6639657616615295,
      "logits/rejected": -1.750392198562622,
      "logps/chosen": -1.3959019184112549,
      "logps/rejected": -2.4921858310699463,
      "loss": 1.4819,
      "nll_loss": 1.4571783542633057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13959020376205444,
      "rewards/margins": 0.10962840914726257,
      "rewards/rejected": -0.24921859800815582,
      "step": 704
    },
    {
      "epoch": 1.9444252675181222,
      "grad_norm": 0.21738582849502563,
      "learning_rate": 1.6743023451120831e-06,
      "log_odds_chosen": 1.179488182067871,
      "log_odds_ratio": -0.2750599980354309,
      "logits/chosen": -0.5516219139099121,
      "logits/rejected": -1.8831249475479126,
      "logps/chosen": -1.524298071861267,
      "logps/rejected": -2.542219638824463,
      "loss": 1.5971,
      "nll_loss": 1.569570541381836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15242981910705566,
      "rewards/margins": 0.10179214924573898,
      "rewards/rejected": -0.25422194600105286,
      "step": 705
    },
    {
      "epoch": 1.947186744908526,
      "grad_norm": 0.1983586996793747,
      "learning_rate": 1.6667420677278962e-06,
      "log_odds_chosen": 1.2816970348358154,
      "log_odds_ratio": -0.2563754618167877,
      "logits/chosen": -0.5835578441619873,
      "logits/rejected": -1.8975870609283447,
      "logps/chosen": -1.4354478120803833,
      "logps/rejected": -2.5307862758636475,
      "loss": 1.5051,
      "nll_loss": 1.4794516563415527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14354479312896729,
      "rewards/margins": 0.10953383892774582,
      "rewards/rejected": -0.2530786395072937,
      "step": 706
    },
    {
      "epoch": 1.94994822229893,
      "grad_norm": 0.21526382863521576,
      "learning_rate": 1.659190353358196e-06,
      "log_odds_chosen": 1.4398778676986694,
      "log_odds_ratio": -0.22124187648296356,
      "logits/chosen": -0.6108973026275635,
      "logits/rejected": -1.833961844444275,
      "logps/chosen": -1.3309154510498047,
      "logps/rejected": -2.544623851776123,
      "loss": 1.4061,
      "nll_loss": 1.383970022201538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13309155404567719,
      "rewards/margins": 0.1213708445429802,
      "rewards/rejected": -0.2544623911380768,
      "step": 707
    },
    {
      "epoch": 1.9527096996893338,
      "grad_norm": 0.21251551806926727,
      "learning_rate": 1.6516472796085314e-06,
      "log_odds_chosen": 1.4610146284103394,
      "log_odds_ratio": -0.2106262743473053,
      "logits/chosen": -0.716134786605835,
      "logits/rejected": -2.0480661392211914,
      "logps/chosen": -1.3701903820037842,
      "logps/rejected": -2.611903190612793,
      "loss": 1.4403,
      "nll_loss": 1.419216275215149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13701903820037842,
      "rewards/margins": 0.12417127192020416,
      "rewards/rejected": -0.2611902952194214,
      "step": 708
    },
    {
      "epoch": 1.9554711770797377,
      "grad_norm": 0.21623145043849945,
      "learning_rate": 1.644112923995656e-06,
      "log_odds_chosen": 1.4195404052734375,
      "log_odds_ratio": -0.2321530133485794,
      "logits/chosen": -0.6861832737922668,
      "logits/rejected": -1.6703221797943115,
      "logps/chosen": -1.414649248123169,
      "logps/rejected": -2.63264536857605,
      "loss": 1.4826,
      "nll_loss": 1.4593451023101807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1414649337530136,
      "rewards/margins": 0.12179961055517197,
      "rewards/rejected": -0.263264536857605,
      "step": 709
    },
    {
      "epoch": 1.9582326544701414,
      "grad_norm": 0.19821126759052277,
      "learning_rate": 1.6365873639467314e-06,
      "log_odds_chosen": 1.1184158325195312,
      "log_odds_ratio": -0.2991623878479004,
      "logits/chosen": -0.5886229276657104,
      "logits/rejected": -1.7395761013031006,
      "logps/chosen": -1.3078004121780396,
      "logps/rejected": -2.224254608154297,
      "loss": 1.401,
      "nll_loss": 1.371075987815857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13078004121780396,
      "rewards/margins": 0.09164540469646454,
      "rewards/rejected": -0.2224254608154297,
      "step": 710
    },
    {
      "epoch": 1.9609941318605455,
      "grad_norm": 0.20421788096427917,
      "learning_rate": 1.6290706767985299e-06,
      "log_odds_chosen": 1.2795296907424927,
      "log_odds_ratio": -0.2593962252140045,
      "logits/chosen": -0.6316720843315125,
      "logits/rejected": -1.9121975898742676,
      "logps/chosen": -1.3871182203292847,
      "logps/rejected": -2.4652185440063477,
      "loss": 1.4544,
      "nll_loss": 1.4284642934799194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1387118101119995,
      "rewards/margins": 0.10781005024909973,
      "rewards/rejected": -0.24652186036109924,
      "step": 711
    },
    {
      "epoch": 1.9637556092509492,
      "grad_norm": 0.23917905986309052,
      "learning_rate": 1.6215629397966432e-06,
      "log_odds_chosen": 1.2976970672607422,
      "log_odds_ratio": -0.24504350125789642,
      "logits/chosen": -0.6498063802719116,
      "logits/rejected": -1.8320040702819824,
      "logps/chosen": -1.3817360401153564,
      "logps/rejected": -2.479696273803711,
      "loss": 1.461,
      "nll_loss": 1.436539888381958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13817362487316132,
      "rewards/margins": 0.10979600250720978,
      "rewards/rejected": -0.2479696273803711,
      "step": 712
    },
    {
      "epoch": 1.9665170866413533,
      "grad_norm": 0.21537664532661438,
      "learning_rate": 1.614064230094684e-06,
      "log_odds_chosen": 1.2931197881698608,
      "log_odds_ratio": -0.24594642221927643,
      "logits/chosen": -0.7107862830162048,
      "logits/rejected": -1.7718734741210938,
      "logps/chosen": -1.439354658126831,
      "logps/rejected": -2.5453765392303467,
      "loss": 1.5165,
      "nll_loss": 1.4919335842132568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14393548667430878,
      "rewards/margins": 0.11060215532779694,
      "rewards/rejected": -0.2545376420021057,
      "step": 713
    },
    {
      "epoch": 1.969278564031757,
      "grad_norm": 0.20869500935077667,
      "learning_rate": 1.6065746247534986e-06,
      "log_odds_chosen": 1.3867168426513672,
      "log_odds_ratio": -0.23219045996665955,
      "logits/chosen": -0.6156042218208313,
      "logits/rejected": -1.8871080875396729,
      "logps/chosen": -1.3930268287658691,
      "logps/rejected": -2.572889804840088,
      "loss": 1.4852,
      "nll_loss": 1.4620187282562256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13930267095565796,
      "rewards/margins": 0.11798630654811859,
      "rewards/rejected": -0.25728896260261536,
      "step": 714
    },
    {
      "epoch": 1.9720400414221608,
      "grad_norm": 0.22620144486427307,
      "learning_rate": 1.599094200740367e-06,
      "log_odds_chosen": 1.1833250522613525,
      "log_odds_ratio": -0.2710687816143036,
      "logits/chosen": -0.6892810463905334,
      "logits/rejected": -1.728080153465271,
      "logps/chosen": -1.4800595045089722,
      "logps/rejected": -2.4893577098846436,
      "loss": 1.557,
      "nll_loss": 1.5298923254013062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14800597727298737,
      "rewards/margins": 0.10092981159687042,
      "rewards/rejected": -0.2489357888698578,
      "step": 715
    },
    {
      "epoch": 1.9748015188125647,
      "grad_norm": 0.2466319352388382,
      "learning_rate": 1.5916230349282215e-06,
      "log_odds_chosen": 1.4189105033874512,
      "log_odds_ratio": -0.23354926705360413,
      "logits/chosen": -0.759655237197876,
      "logits/rejected": -1.570073127746582,
      "logps/chosen": -1.3400826454162598,
      "logps/rejected": -2.537966728210449,
      "loss": 1.431,
      "nll_loss": 1.407649278640747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1340082734823227,
      "rewards/margins": 0.11978840827941895,
      "rewards/rejected": -0.25379669666290283,
      "step": 716
    },
    {
      "epoch": 1.9775629962029686,
      "grad_norm": 0.21501441299915314,
      "learning_rate": 1.58416120409485e-06,
      "log_odds_chosen": 1.2824525833129883,
      "log_odds_ratio": -0.24649572372436523,
      "logits/chosen": -0.6352999806404114,
      "logits/rejected": -2.0760064125061035,
      "logps/chosen": -1.4632048606872559,
      "logps/rejected": -2.5624969005584717,
      "loss": 1.5135,
      "nll_loss": 1.4888546466827393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14632049202919006,
      "rewards/margins": 0.10992921143770218,
      "rewards/rejected": -0.25624969601631165,
      "step": 717
    },
    {
      "epoch": 1.9803244735933725,
      "grad_norm": 0.20214088261127472,
      "learning_rate": 1.5767087849221097e-06,
      "log_odds_chosen": 1.2076809406280518,
      "log_odds_ratio": -0.2692188620567322,
      "logits/chosen": -0.6633406281471252,
      "logits/rejected": -1.7296428680419922,
      "logps/chosen": -1.385692834854126,
      "logps/rejected": -2.4033093452453613,
      "loss": 1.4713,
      "nll_loss": 1.4444183111190796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13856931030750275,
      "rewards/margins": 0.10176165401935577,
      "rewards/rejected": -0.24033094942569733,
      "step": 718
    },
    {
      "epoch": 1.9830859509837762,
      "grad_norm": 0.21768531203269958,
      "learning_rate": 1.5692658539951371e-06,
      "log_odds_chosen": 1.3379472494125366,
      "log_odds_ratio": -0.23937927186489105,
      "logits/chosen": -0.7326225638389587,
      "logits/rejected": -1.9709231853485107,
      "logps/chosen": -1.4037171602249146,
      "logps/rejected": -2.5365142822265625,
      "loss": 1.4892,
      "nll_loss": 1.4652434587478638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14037172496318817,
      "rewards/margins": 0.11327968537807465,
      "rewards/rejected": -0.2536514103412628,
      "step": 719
    },
    {
      "epoch": 1.9858474283741803,
      "grad_norm": 0.21629256010055542,
      "learning_rate": 1.561832487801565e-06,
      "log_odds_chosen": 1.3326436281204224,
      "log_odds_ratio": -0.2489023208618164,
      "logits/chosen": -0.6759160161018372,
      "logits/rejected": -1.9447509050369263,
      "logps/chosen": -1.413867473602295,
      "logps/rejected": -2.552180528640747,
      "loss": 1.4818,
      "nll_loss": 1.456876277923584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1413867473602295,
      "rewards/margins": 0.1138312891125679,
      "rewards/rejected": -0.2552180588245392,
      "step": 720
    },
    {
      "epoch": 1.988608905764584,
      "grad_norm": 0.2140151709318161,
      "learning_rate": 1.5544087627307308e-06,
      "log_odds_chosen": 1.4864577054977417,
      "log_odds_ratio": -0.20958803594112396,
      "logits/chosen": -0.7287192940711975,
      "logits/rejected": -2.011051893234253,
      "logps/chosen": -1.403037190437317,
      "logps/rejected": -2.680103302001953,
      "loss": 1.4696,
      "nll_loss": 1.4486826658248901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14030373096466064,
      "rewards/margins": 0.1277066171169281,
      "rewards/rejected": -0.26801031827926636,
      "step": 721
    },
    {
      "epoch": 1.991370383154988,
      "grad_norm": 0.21654756367206573,
      "learning_rate": 1.546994755072896e-06,
      "log_odds_chosen": 1.4832570552825928,
      "log_odds_ratio": -0.21186988055706024,
      "logits/chosen": -0.6306319832801819,
      "logits/rejected": -1.948667287826538,
      "logps/chosen": -1.3662739992141724,
      "logps/rejected": -2.6316754817962646,
      "loss": 1.4518,
      "nll_loss": 1.4306564331054688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13662739098072052,
      "rewards/margins": 0.1265401393175125,
      "rewards/rejected": -0.2631675601005554,
      "step": 722
    },
    {
      "epoch": 1.9941318605453917,
      "grad_norm": 0.2074047029018402,
      "learning_rate": 1.539590541018461e-06,
      "log_odds_chosen": 1.355461597442627,
      "log_odds_ratio": -0.23559004068374634,
      "logits/chosen": -0.7348309755325317,
      "logits/rejected": -1.9341561794281006,
      "logps/chosen": -1.4475115537643433,
      "logps/rejected": -2.612541675567627,
      "loss": 1.5228,
      "nll_loss": 1.4992461204528809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1447511613368988,
      "rewards/margins": 0.1165030226111412,
      "rewards/rejected": -0.2612541913986206,
      "step": 723
    },
    {
      "epoch": 1.9968933379357956,
      "grad_norm": 0.21759307384490967,
      "learning_rate": 1.5321961966571801e-06,
      "log_odds_chosen": 1.2848310470581055,
      "log_odds_ratio": -0.2530062794685364,
      "logits/chosen": -0.7775071859359741,
      "logits/rejected": -1.8053133487701416,
      "logps/chosen": -1.4230960607528687,
      "logps/rejected": -2.5227210521698,
      "loss": 1.4948,
      "nll_loss": 1.469494104385376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14230960607528687,
      "rewards/margins": 0.10996250063180923,
      "rewards/rejected": -0.2522720992565155,
      "step": 724
    },
    {
      "epoch": 1.9996548153261995,
      "grad_norm": 0.20666009187698364,
      "learning_rate": 1.524811797977383e-06,
      "log_odds_chosen": 1.1944769620895386,
      "log_odds_ratio": -0.2727535367012024,
      "logits/chosen": -0.7024763822555542,
      "logits/rejected": -1.7682558298110962,
      "logps/chosen": -1.462043285369873,
      "logps/rejected": -2.4807724952697754,
      "loss": 1.5248,
      "nll_loss": 1.4975111484527588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14620435237884521,
      "rewards/margins": 0.10187289118766785,
      "rewards/rejected": -0.24807721376419067,
      "step": 725
    },
    {
      "epoch": 2.0,
      "grad_norm": 0.49442198872566223,
      "learning_rate": 1.5174374208651913e-06,
      "log_odds_chosen": 1.194680094718933,
      "log_odds_ratio": -0.2645164132118225,
      "logits/chosen": -0.6112655997276306,
      "logits/rejected": -2.1095049381256104,
      "logps/chosen": -1.4013067483901978,
      "logps/rejected": -2.4076037406921387,
      "loss": 1.4474,
      "nll_loss": 1.4209479093551636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1401306837797165,
      "rewards/margins": 0.10062968730926514,
      "rewards/rejected": -0.24076037108898163,
      "step": 726
    },
    {
      "epoch": 2.0027614773904037,
      "grad_norm": 0.19985070824623108,
      "learning_rate": 1.5100731411037395e-06,
      "log_odds_chosen": 1.524064064025879,
      "log_odds_ratio": -0.20543928444385529,
      "logits/chosen": -0.7049272656440735,
      "logits/rejected": -1.9933799505233765,
      "logps/chosen": -1.4320381879806519,
      "logps/rejected": -2.751387119293213,
      "loss": 1.5037,
      "nll_loss": 1.4831523895263672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14320380985736847,
      "rewards/margins": 0.13193488121032715,
      "rewards/rejected": -0.2751386761665344,
      "step": 727
    },
    {
      "epoch": 2.0055229547808078,
      "grad_norm": 0.1995333880186081,
      "learning_rate": 1.502719034372396e-06,
      "log_odds_chosen": 1.1505608558654785,
      "log_odds_ratio": -0.28334516286849976,
      "logits/chosen": -0.78338623046875,
      "logits/rejected": -1.6854302883148193,
      "logps/chosen": -1.3915482759475708,
      "logps/rejected": -2.359539270401001,
      "loss": 1.457,
      "nll_loss": 1.4286986589431763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1391548365354538,
      "rewards/margins": 0.0967991054058075,
      "rewards/rejected": -0.2359539419412613,
      "step": 728
    },
    {
      "epoch": 2.0082844321712114,
      "grad_norm": 0.2035675048828125,
      "learning_rate": 1.4953751762459867e-06,
      "log_odds_chosen": 1.4010441303253174,
      "log_odds_ratio": -0.24087637662887573,
      "logits/chosen": -0.675167977809906,
      "logits/rejected": -1.913004994392395,
      "logps/chosen": -1.3706873655319214,
      "logps/rejected": -2.5642473697662354,
      "loss": 1.4511,
      "nll_loss": 1.4269742965698242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1370687186717987,
      "rewards/margins": 0.11935600638389587,
      "rewards/rejected": -0.2564247250556946,
      "step": 729
    },
    {
      "epoch": 2.0110459095616156,
      "grad_norm": 0.2269647717475891,
      "learning_rate": 1.4880416421940155e-06,
      "log_odds_chosen": 1.3267796039581299,
      "log_odds_ratio": -0.24957221746444702,
      "logits/chosen": -0.7462670803070068,
      "logits/rejected": -1.7765371799468994,
      "logps/chosen": -1.4251582622528076,
      "logps/rejected": -2.561957836151123,
      "loss": 1.4963,
      "nll_loss": 1.471299409866333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14251583814620972,
      "rewards/margins": 0.11367997527122498,
      "rewards/rejected": -0.2561958134174347,
      "step": 730
    },
    {
      "epoch": 2.013807386952019,
      "grad_norm": 0.21582657098770142,
      "learning_rate": 1.4807185075798919e-06,
      "log_odds_chosen": 1.455092430114746,
      "log_odds_ratio": -0.2169358879327774,
      "logits/chosen": -0.8274435997009277,
      "logits/rejected": -2.024759292602539,
      "logps/chosen": -1.4748046398162842,
      "logps/rejected": -2.735257148742676,
      "loss": 1.5375,
      "nll_loss": 1.5158442258834839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14748047292232513,
      "rewards/margins": 0.12604522705078125,
      "rewards/rejected": -0.2735256850719452,
      "step": 731
    },
    {
      "epoch": 2.0165688643424233,
      "grad_norm": 0.20380431413650513,
      "learning_rate": 1.4734058476601553e-06,
      "log_odds_chosen": 1.4209482669830322,
      "log_odds_ratio": -0.2209295928478241,
      "logits/chosen": -0.7545549869537354,
      "logits/rejected": -2.0098748207092285,
      "logps/chosen": -1.4165632724761963,
      "logps/rejected": -2.6308608055114746,
      "loss": 1.4862,
      "nll_loss": 1.4640896320343018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14165633916854858,
      "rewards/margins": 0.12142977118492126,
      "rewards/rejected": -0.26308611035346985,
      "step": 732
    },
    {
      "epoch": 2.019330341732827,
      "grad_norm": 0.20928171277046204,
      "learning_rate": 1.466103737583699e-06,
      "log_odds_chosen": 1.3327419757843018,
      "log_odds_ratio": -0.24103975296020508,
      "logits/chosen": -0.7420281767845154,
      "logits/rejected": -1.969616413116455,
      "logps/chosen": -1.4328577518463135,
      "logps/rejected": -2.5726540088653564,
      "loss": 1.5061,
      "nll_loss": 1.481981873512268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14328575134277344,
      "rewards/margins": 0.11397963762283325,
      "rewards/rejected": -0.2572654187679291,
      "step": 733
    },
    {
      "epoch": 2.022091819123231,
      "grad_norm": 0.20876739919185638,
      "learning_rate": 1.4588122523910032e-06,
      "log_odds_chosen": 1.093473196029663,
      "log_odds_ratio": -0.29052767157554626,
      "logits/chosen": -0.7385894656181335,
      "logits/rejected": -1.8053125143051147,
      "logps/chosen": -1.456524133682251,
      "logps/rejected": -2.3806233406066895,
      "loss": 1.5197,
      "nll_loss": 1.4906771183013916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1456524282693863,
      "rewards/margins": 0.09240993857383728,
      "rewards/rejected": -0.23806238174438477,
      "step": 734
    },
    {
      "epoch": 2.0248532965136348,
      "grad_norm": 0.2023276835680008,
      "learning_rate": 1.4515314670133582e-06,
      "log_odds_chosen": 1.535997986793518,
      "log_odds_ratio": -0.21157340705394745,
      "logits/chosen": -0.7386196851730347,
      "logits/rejected": -1.8687727451324463,
      "logps/chosen": -1.419616937637329,
      "logps/rejected": -2.751965284347534,
      "loss": 1.478,
      "nll_loss": 1.456872820854187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419617086648941,
      "rewards/margins": 0.13323484361171722,
      "rewards/rejected": -0.27519655227661133,
      "step": 735
    },
    {
      "epoch": 2.027614773904039,
      "grad_norm": 0.21986712515354156,
      "learning_rate": 1.4442614562721011e-06,
      "log_odds_chosen": 1.2944157123565674,
      "log_odds_ratio": -0.25431200861930847,
      "logits/chosen": -0.6299113035202026,
      "logits/rejected": -1.713247537612915,
      "logps/chosen": -1.3927693367004395,
      "logps/rejected": -2.483180046081543,
      "loss": 1.4661,
      "nll_loss": 1.440664291381836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.139276921749115,
      "rewards/margins": 0.10904109477996826,
      "rewards/rejected": -0.24831801652908325,
      "step": 736
    },
    {
      "epoch": 2.0303762512944425,
      "grad_norm": 0.2185211479663849,
      "learning_rate": 1.4370022948778383e-06,
      "log_odds_chosen": 1.4415481090545654,
      "log_odds_ratio": -0.2162938416004181,
      "logits/chosen": -0.6487110257148743,
      "logits/rejected": -2.1030993461608887,
      "logps/chosen": -1.4767889976501465,
      "logps/rejected": -2.7231407165527344,
      "loss": 1.5402,
      "nll_loss": 1.5185596942901611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476789265871048,
      "rewards/margins": 0.12463513016700745,
      "rewards/rejected": -0.27231404185295105,
      "step": 737
    },
    {
      "epoch": 2.033137728684846,
      "grad_norm": 0.22171227633953094,
      "learning_rate": 1.429754057429687e-06,
      "log_odds_chosen": 1.3265342712402344,
      "log_odds_ratio": -0.2393992841243744,
      "logits/chosen": -0.8432047963142395,
      "logits/rejected": -1.8966618776321411,
      "logps/chosen": -1.3895668983459473,
      "logps/rejected": -2.5131988525390625,
      "loss": 1.4717,
      "nll_loss": 1.4477918148040771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1389566957950592,
      "rewards/margins": 0.11236318945884705,
      "rewards/rejected": -0.25131988525390625,
      "step": 738
    },
    {
      "epoch": 2.0358992060752503,
      "grad_norm": 0.42570745944976807,
      "learning_rate": 1.422516818414499e-06,
      "log_odds_chosen": 1.2274531126022339,
      "log_odds_ratio": -0.26237213611602783,
      "logits/chosen": -0.754129946231842,
      "logits/rejected": -1.742024540901184,
      "logps/chosen": -1.3718271255493164,
      "logps/rejected": -2.3997960090637207,
      "loss": 1.4529,
      "nll_loss": 1.4266401529312134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13718271255493164,
      "rewards/margins": 0.10279688239097595,
      "rewards/rejected": -0.2399796098470688,
      "step": 739
    },
    {
      "epoch": 2.038660683465654,
      "grad_norm": 0.24944612383842468,
      "learning_rate": 1.415290652206105e-06,
      "log_odds_chosen": 1.3752034902572632,
      "log_odds_ratio": -0.2288976013660431,
      "logits/chosen": -0.784325361251831,
      "logits/rejected": -1.8980276584625244,
      "logps/chosen": -1.4716408252716064,
      "logps/rejected": -2.658496379852295,
      "loss": 1.5268,
      "nll_loss": 1.503926157951355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14716407656669617,
      "rewards/margins": 0.11868555843830109,
      "rewards/rejected": -0.26584964990615845,
      "step": 740
    },
    {
      "epoch": 2.041422160856058,
      "grad_norm": 0.19353602826595306,
      "learning_rate": 1.4080756330645424e-06,
      "log_odds_chosen": 1.3971836566925049,
      "log_odds_ratio": -0.23170241713523865,
      "logits/chosen": -0.757472038269043,
      "logits/rejected": -1.8959999084472656,
      "logps/chosen": -1.3243657350540161,
      "logps/rejected": -2.496757745742798,
      "loss": 1.3909,
      "nll_loss": 1.3677642345428467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1324365735054016,
      "rewards/margins": 0.11723920702934265,
      "rewards/rejected": -0.24967579543590546,
      "step": 741
    },
    {
      "epoch": 2.0441836382464618,
      "grad_norm": 0.2381175011396408,
      "learning_rate": 1.400871835135295e-06,
      "log_odds_chosen": 1.488401174545288,
      "log_odds_ratio": -0.2107824981212616,
      "logits/chosen": -0.8121135830879211,
      "logits/rejected": -1.801040768623352,
      "logps/chosen": -1.395693063735962,
      "logps/rejected": -2.6747279167175293,
      "loss": 1.4644,
      "nll_loss": 1.443336844444275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13956931233406067,
      "rewards/margins": 0.12790347635746002,
      "rewards/rejected": -0.2674728035926819,
      "step": 742
    },
    {
      "epoch": 2.046945115636866,
      "grad_norm": 0.21528197824954987,
      "learning_rate": 1.3936793324485343e-06,
      "log_odds_chosen": 1.426561713218689,
      "log_odds_ratio": -0.2208237648010254,
      "logits/chosen": -0.7020534873008728,
      "logits/rejected": -1.9243749380111694,
      "logps/chosen": -1.377315878868103,
      "logps/rejected": -2.5889315605163574,
      "loss": 1.4521,
      "nll_loss": 1.4300365447998047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13773158192634583,
      "rewards/margins": 0.12116158753633499,
      "rewards/rejected": -0.2588931918144226,
      "step": 743
    },
    {
      "epoch": 2.0497065930272695,
      "grad_norm": 0.21400389075279236,
      "learning_rate": 1.386498198918352e-06,
      "log_odds_chosen": 1.3467150926589966,
      "log_odds_ratio": -0.23828084766864777,
      "logits/chosen": -0.7199838161468506,
      "logits/rejected": -2.0278215408325195,
      "logps/chosen": -1.480491042137146,
      "logps/rejected": -2.645315408706665,
      "loss": 1.5512,
      "nll_loss": 1.527329683303833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14804911613464355,
      "rewards/margins": 0.11648242175579071,
      "rewards/rejected": -0.26453155279159546,
      "step": 744
    },
    {
      "epoch": 2.0524680704176737,
      "grad_norm": 0.2323625087738037,
      "learning_rate": 1.3793285083420077e-06,
      "log_odds_chosen": 1.2312310934066772,
      "log_odds_ratio": -0.26046109199523926,
      "logits/chosen": -0.6492730379104614,
      "logits/rejected": -1.7875938415527344,
      "logps/chosen": -1.5298579931259155,
      "logps/rejected": -2.5959219932556152,
      "loss": 1.5921,
      "nll_loss": 1.5660476684570312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15298578143119812,
      "rewards/margins": 0.10660640150308609,
      "rewards/rejected": -0.259592205286026,
      "step": 745
    },
    {
      "epoch": 2.0552295478080773,
      "grad_norm": 0.21785420179367065,
      "learning_rate": 1.3721703343991634e-06,
      "log_odds_chosen": 1.5281133651733398,
      "log_odds_ratio": -0.20278286933898926,
      "logits/chosen": -0.7953973412513733,
      "logits/rejected": -2.098724603652954,
      "logps/chosen": -1.403007984161377,
      "logps/rejected": -2.718156337738037,
      "loss": 1.4625,
      "nll_loss": 1.4422367811203003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14030081033706665,
      "rewards/margins": 0.13151484727859497,
      "rewards/rejected": -0.2718156576156616,
      "step": 746
    },
    {
      "epoch": 2.057991025198481,
      "grad_norm": 0.2271818220615387,
      "learning_rate": 1.3650237506511333e-06,
      "log_odds_chosen": 1.3591433763504028,
      "log_odds_ratio": -0.24462954699993134,
      "logits/chosen": -0.7451946139335632,
      "logits/rejected": -1.7923208475112915,
      "logps/chosen": -1.3873848915100098,
      "logps/rejected": -2.5456383228302,
      "loss": 1.4824,
      "nll_loss": 1.4579014778137207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1387384980916977,
      "rewards/margins": 0.1158253476023674,
      "rewards/rejected": -0.2545638382434845,
      "step": 747
    },
    {
      "epoch": 2.060752502588885,
      "grad_norm": 0.22212862968444824,
      "learning_rate": 1.3578888305401208e-06,
      "log_odds_chosen": 1.3162353038787842,
      "log_odds_ratio": -0.2563769817352295,
      "logits/chosen": -0.7208426594734192,
      "logits/rejected": -1.8948148488998413,
      "logps/chosen": -1.389916181564331,
      "logps/rejected": -2.5131213665008545,
      "loss": 1.4683,
      "nll_loss": 1.4426243305206299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13899163901805878,
      "rewards/margins": 0.11232049018144608,
      "rewards/rejected": -0.25131210684776306,
      "step": 748
    },
    {
      "epoch": 2.0635139799792888,
      "grad_norm": 0.21235612034797668,
      "learning_rate": 1.3507656473884718e-06,
      "log_odds_chosen": 1.3354262113571167,
      "log_odds_ratio": -0.24460864067077637,
      "logits/chosen": -0.6753593683242798,
      "logits/rejected": -1.8687498569488525,
      "logps/chosen": -1.3753629922866821,
      "logps/rejected": -2.499833583831787,
      "loss": 1.4357,
      "nll_loss": 1.4112131595611572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13753631711006165,
      "rewards/margins": 0.11244705319404602,
      "rewards/rejected": -0.24998337030410767,
      "step": 749
    },
    {
      "epoch": 2.066275457369693,
      "grad_norm": 0.2222043126821518,
      "learning_rate": 1.3436542743979125e-06,
      "log_odds_chosen": 1.1222965717315674,
      "log_odds_ratio": -0.29325950145721436,
      "logits/chosen": -0.7638905644416809,
      "logits/rejected": -1.7164831161499023,
      "logps/chosen": -1.3756606578826904,
      "logps/rejected": -2.3173580169677734,
      "loss": 1.4687,
      "nll_loss": 1.4393634796142578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13756605982780457,
      "rewards/margins": 0.0941697359085083,
      "rewards/rejected": -0.23173581063747406,
      "step": 750
    },
    {
      "epoch": 2.0690369347600965,
      "grad_norm": 0.2122756689786911,
      "learning_rate": 1.3365547846488037e-06,
      "log_odds_chosen": 1.5225597620010376,
      "log_odds_ratio": -0.20094801485538483,
      "logits/chosen": -0.7458276152610779,
      "logits/rejected": -1.9331128597259521,
      "logps/chosen": -1.4181265830993652,
      "logps/rejected": -2.7289392948150635,
      "loss": 1.4843,
      "nll_loss": 1.4641839265823364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14181266725063324,
      "rewards/margins": 0.1310812532901764,
      "rewards/rejected": -0.2728939354419708,
      "step": 751
    },
    {
      "epoch": 2.0717984121505006,
      "grad_norm": 0.2157318890094757,
      "learning_rate": 1.3294672510993862e-06,
      "log_odds_chosen": 1.4378488063812256,
      "log_odds_ratio": -0.22265379130840302,
      "logits/chosen": -0.7862150073051453,
      "logits/rejected": -1.94209885597229,
      "logps/chosen": -1.4849308729171753,
      "logps/rejected": -2.7356879711151123,
      "loss": 1.5487,
      "nll_loss": 1.5264191627502441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14849308133125305,
      "rewards/margins": 0.12507571280002594,
      "rewards/rejected": -0.2735688090324402,
      "step": 752
    },
    {
      "epoch": 2.0745598895409043,
      "grad_norm": 0.21287965774536133,
      "learning_rate": 1.3223917465850344e-06,
      "log_odds_chosen": 1.291487216949463,
      "log_odds_ratio": -0.25558802485466003,
      "logits/chosen": -0.6685554385185242,
      "logits/rejected": -1.8840442895889282,
      "logps/chosen": -1.4667881727218628,
      "logps/rejected": -2.579162836074829,
      "loss": 1.543,
      "nll_loss": 1.5174570083618164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14667882025241852,
      "rewards/margins": 0.11123748123645782,
      "rewards/rejected": -0.25791627168655396,
      "step": 753
    },
    {
      "epoch": 2.0773213669313084,
      "grad_norm": 0.23124369978904724,
      "learning_rate": 1.3153283438175036e-06,
      "log_odds_chosen": 1.5643819570541382,
      "log_odds_ratio": -0.2072456032037735,
      "logits/chosen": -0.7434122562408447,
      "logits/rejected": -1.8903828859329224,
      "logps/chosen": -1.498293161392212,
      "logps/rejected": -2.8726089000701904,
      "loss": 1.5474,
      "nll_loss": 1.526721715927124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14982932806015015,
      "rewards/margins": 0.1374315768480301,
      "rewards/rejected": -0.28726089000701904,
      "step": 754
    },
    {
      "epoch": 2.080082844321712,
      "grad_norm": 0.24547821283340454,
      "learning_rate": 1.3082771153841872e-06,
      "log_odds_chosen": 1.359768271446228,
      "log_odds_ratio": -0.2351943403482437,
      "logits/chosen": -0.786487877368927,
      "logits/rejected": -1.7432615756988525,
      "logps/chosen": -1.4537529945373535,
      "logps/rejected": -2.622997760772705,
      "loss": 1.5433,
      "nll_loss": 1.5197895765304565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14537528157234192,
      "rewards/margins": 0.1169244721531868,
      "rewards/rejected": -0.2622997760772705,
      "step": 755
    },
    {
      "epoch": 2.082844321712116,
      "grad_norm": 0.22222405672073364,
      "learning_rate": 1.3012381337473656e-06,
      "log_odds_chosen": 1.737757682800293,
      "log_odds_ratio": -0.18532472848892212,
      "logits/chosen": -0.6934542059898376,
      "logits/rejected": -2.0250802040100098,
      "logps/chosen": -1.2511564493179321,
      "logps/rejected": -2.703065872192383,
      "loss": 1.3362,
      "nll_loss": 1.3176548480987549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12511564791202545,
      "rewards/margins": 0.14519096910953522,
      "rewards/rejected": -0.2703065872192383,
      "step": 756
    },
    {
      "epoch": 2.08560579910252,
      "grad_norm": 0.21016424894332886,
      "learning_rate": 1.294211471243466e-06,
      "log_odds_chosen": 1.409480094909668,
      "log_odds_ratio": -0.22224737703800201,
      "logits/chosen": -0.7898975610733032,
      "logits/rejected": -2.0010173320770264,
      "logps/chosen": -1.36336088180542,
      "logps/rejected": -2.5560050010681152,
      "loss": 1.4278,
      "nll_loss": 1.4055848121643066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.136336088180542,
      "rewards/margins": 0.1192643940448761,
      "rewards/rejected": -0.2556004822254181,
      "step": 757
    },
    {
      "epoch": 2.0883672764929235,
      "grad_norm": 0.20899075269699097,
      "learning_rate": 1.2871972000823197e-06,
      "log_odds_chosen": 1.3560638427734375,
      "log_odds_ratio": -0.23246008157730103,
      "logits/chosen": -0.7394353151321411,
      "logits/rejected": -2.0328736305236816,
      "logps/chosen": -1.3651076555252075,
      "logps/rejected": -2.5123071670532227,
      "loss": 1.4393,
      "nll_loss": 1.4160305261611938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13651075959205627,
      "rewards/margins": 0.11471996456384659,
      "rewards/rejected": -0.25123071670532227,
      "step": 758
    },
    {
      "epoch": 2.0911287538833276,
      "grad_norm": 0.20122100412845612,
      "learning_rate": 1.2801953923464139e-06,
      "log_odds_chosen": 1.322190761566162,
      "log_odds_ratio": -0.2632856070995331,
      "logits/chosen": -0.7781534790992737,
      "logits/rejected": -1.7706246376037598,
      "logps/chosen": -1.3630403280258179,
      "logps/rejected": -2.4846913814544678,
      "loss": 1.4516,
      "nll_loss": 1.4252920150756836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13630403578281403,
      "rewards/margins": 0.11216511577367783,
      "rewards/rejected": -0.24846915900707245,
      "step": 759
    },
    {
      "epoch": 2.0938902312737313,
      "grad_norm": 0.22142185270786285,
      "learning_rate": 1.2732061199901563e-06,
      "log_odds_chosen": 1.2743151187896729,
      "log_odds_ratio": -0.2580106258392334,
      "logits/chosen": -0.7141470313072205,
      "logits/rejected": -1.5875985622406006,
      "logps/chosen": -1.4468305110931396,
      "logps/rejected": -2.5337939262390137,
      "loss": 1.5244,
      "nll_loss": 1.4985558986663818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14468304812908173,
      "rewards/margins": 0.108696348965168,
      "rewards/rejected": -0.2533794045448303,
      "step": 760
    },
    {
      "epoch": 2.0966517086641354,
      "grad_norm": 0.2264997810125351,
      "learning_rate": 1.266229454839133e-06,
      "log_odds_chosen": 1.2582509517669678,
      "log_odds_ratio": -0.2561207711696625,
      "logits/chosen": -0.8011865615844727,
      "logits/rejected": -1.6508357524871826,
      "logps/chosen": -1.3973218202590942,
      "logps/rejected": -2.462202310562134,
      "loss": 1.4649,
      "nll_loss": 1.4393376111984253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13973219692707062,
      "rewards/margins": 0.10648804903030396,
      "rewards/rejected": -0.24622023105621338,
      "step": 761
    },
    {
      "epoch": 2.099413186054539,
      "grad_norm": 0.23098450899124146,
      "learning_rate": 1.2592654685893757e-06,
      "log_odds_chosen": 1.5264986753463745,
      "log_odds_ratio": -0.20155426859855652,
      "logits/chosen": -0.66983962059021,
      "logits/rejected": -1.9511102437973022,
      "logps/chosen": -1.396782398223877,
      "logps/rejected": -2.706751585006714,
      "loss": 1.4673,
      "nll_loss": 1.4471567869186401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1396782398223877,
      "rewards/margins": 0.1309969276189804,
      "rewards/rejected": -0.2706751525402069,
      "step": 762
    },
    {
      "epoch": 2.102174663444943,
      "grad_norm": 0.23991060256958008,
      "learning_rate": 1.252314232806615e-06,
      "log_odds_chosen": 1.432591438293457,
      "log_odds_ratio": -0.23098701238632202,
      "logits/chosen": -0.8026631474494934,
      "logits/rejected": -1.9127060174942017,
      "logps/chosen": -1.4595669507980347,
      "logps/rejected": -2.698004722595215,
      "loss": 1.5126,
      "nll_loss": 1.4895089864730835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14595669507980347,
      "rewards/margins": 0.12384375929832458,
      "rewards/rejected": -0.26980048418045044,
      "step": 763
    },
    {
      "epoch": 2.104936140835347,
      "grad_norm": 0.21347320079803467,
      "learning_rate": 1.2453758189255568e-06,
      "log_odds_chosen": 1.3525452613830566,
      "log_odds_ratio": -0.2315671443939209,
      "logits/chosen": -0.7290445566177368,
      "logits/rejected": -2.068817138671875,
      "logps/chosen": -1.3774614334106445,
      "logps/rejected": -2.521169900894165,
      "loss": 1.4489,
      "nll_loss": 1.4257827997207642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13774614036083221,
      "rewards/margins": 0.11437083780765533,
      "rewards/rejected": -0.25211697816848755,
      "step": 764
    },
    {
      "epoch": 2.107697618225751,
      "grad_norm": 0.22012005746364594,
      "learning_rate": 1.2384502982491359e-06,
      "log_odds_chosen": 1.4865221977233887,
      "log_odds_ratio": -0.21373732388019562,
      "logits/chosen": -0.7536633610725403,
      "logits/rejected": -2.040149688720703,
      "logps/chosen": -1.5073720216751099,
      "logps/rejected": -2.8081774711608887,
      "loss": 1.578,
      "nll_loss": 1.5566452741622925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1507372111082077,
      "rewards/margins": 0.13008056581020355,
      "rewards/rejected": -0.28081777691841125,
      "step": 765
    },
    {
      "epoch": 2.1104590956161546,
      "grad_norm": 0.22354905307292938,
      "learning_rate": 1.231537741947795e-06,
      "log_odds_chosen": 1.2573996782302856,
      "log_odds_ratio": -0.26209187507629395,
      "logits/chosen": -0.7578604817390442,
      "logits/rejected": -1.8252947330474854,
      "logps/chosen": -1.4263148307800293,
      "logps/rejected": -2.4963927268981934,
      "loss": 1.501,
      "nll_loss": 1.4747966527938843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14263150095939636,
      "rewards/margins": 0.10700778663158417,
      "rewards/rejected": -0.24963928759098053,
      "step": 766
    },
    {
      "epoch": 2.1132205730065583,
      "grad_norm": 0.2208777815103531,
      "learning_rate": 1.2246382210587432e-06,
      "log_odds_chosen": 1.5075188875198364,
      "log_odds_ratio": -0.20569130778312683,
      "logits/chosen": -0.7639518976211548,
      "logits/rejected": -1.967806100845337,
      "logps/chosen": -1.4122600555419922,
      "logps/rejected": -2.710855722427368,
      "loss": 1.4812,
      "nll_loss": 1.4606313705444336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14122600853443146,
      "rewards/margins": 0.1298595815896988,
      "rewards/rejected": -0.27108559012413025,
      "step": 767
    },
    {
      "epoch": 2.1159820503969624,
      "grad_norm": 0.20796267688274384,
      "learning_rate": 1.217751806485235e-06,
      "log_odds_chosen": 1.4210944175720215,
      "log_odds_ratio": -0.22676415741443634,
      "logits/chosen": -0.7732008099555969,
      "logits/rejected": -1.9926378726959229,
      "logps/chosen": -1.3832429647445679,
      "logps/rejected": -2.59248423576355,
      "loss": 1.4545,
      "nll_loss": 1.43184494972229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1383242905139923,
      "rewards/margins": 0.12092413753271103,
      "rewards/rejected": -0.25924843549728394,
      "step": 768
    },
    {
      "epoch": 2.118743527787366,
      "grad_norm": 0.21591438353061676,
      "learning_rate": 1.2108785689958337e-06,
      "log_odds_chosen": 1.2549666166305542,
      "log_odds_ratio": -0.2598121464252472,
      "logits/chosen": -0.6248151659965515,
      "logits/rejected": -1.732608437538147,
      "logps/chosen": -1.4587433338165283,
      "logps/rejected": -2.5326249599456787,
      "loss": 1.5275,
      "nll_loss": 1.5015511512756348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14587435126304626,
      "rewards/margins": 0.10738813877105713,
      "rewards/rejected": -0.253262460231781,
      "step": 769
    },
    {
      "epoch": 2.12150500517777,
      "grad_norm": 0.21577268838882446,
      "learning_rate": 1.2040185792236874e-06,
      "log_odds_chosen": 1.386768102645874,
      "log_odds_ratio": -0.22949083149433136,
      "logits/chosen": -0.7459929585456848,
      "logits/rejected": -1.90436589717865,
      "logps/chosen": -1.3482396602630615,
      "logps/rejected": -2.517188310623169,
      "loss": 1.4235,
      "nll_loss": 1.400557279586792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1348239779472351,
      "rewards/margins": 0.11689485609531403,
      "rewards/rejected": -0.2517188489437103,
      "step": 770
    },
    {
      "epoch": 2.124266482568174,
      "grad_norm": 0.23260287940502167,
      "learning_rate": 1.197171907665808e-06,
      "log_odds_chosen": 1.4571518898010254,
      "log_odds_ratio": -0.21860212087631226,
      "logits/chosen": -0.8009171485900879,
      "logits/rejected": -1.845001459121704,
      "logps/chosen": -1.393100380897522,
      "logps/rejected": -2.638636589050293,
      "loss": 1.4644,
      "nll_loss": 1.4425642490386963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1393100470304489,
      "rewards/margins": 0.1245536133646965,
      "rewards/rejected": -0.2638636827468872,
      "step": 771
    },
    {
      "epoch": 2.127027959958578,
      "grad_norm": 0.21608783304691315,
      "learning_rate": 1.1903386246823363e-06,
      "log_odds_chosen": 1.6979553699493408,
      "log_odds_ratio": -0.1744653582572937,
      "logits/chosen": -0.7080166935920715,
      "logits/rejected": -2.05287504196167,
      "logps/chosen": -1.4491580724716187,
      "logps/rejected": -2.9308226108551025,
      "loss": 1.5145,
      "nll_loss": 1.4970835447311401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14491581916809082,
      "rewards/margins": 0.1481664478778839,
      "rewards/rejected": -0.2930822968482971,
      "step": 772
    },
    {
      "epoch": 2.1297894373489816,
      "grad_norm": 0.21926312148571014,
      "learning_rate": 1.1835188004958298e-06,
      "log_odds_chosen": 1.411818265914917,
      "log_odds_ratio": -0.22714708745479584,
      "logits/chosen": -0.8942671418190002,
      "logits/rejected": -2.0131399631500244,
      "logps/chosen": -1.3782057762145996,
      "logps/rejected": -2.5774431228637695,
      "loss": 1.4443,
      "nll_loss": 1.4216082096099854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13782057166099548,
      "rewards/margins": 0.11992374807596207,
      "rewards/rejected": -0.25774431228637695,
      "step": 773
    },
    {
      "epoch": 2.1325509147393857,
      "grad_norm": 0.22599764168262482,
      "learning_rate": 1.1767125051905314e-06,
      "log_odds_chosen": 1.4329699277877808,
      "log_odds_ratio": -0.2236868441104889,
      "logits/chosen": -0.8578775525093079,
      "logits/rejected": -1.9920375347137451,
      "logps/chosen": -1.3404793739318848,
      "logps/rejected": -2.5507805347442627,
      "loss": 1.4275,
      "nll_loss": 1.4050943851470947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13404794037342072,
      "rewards/margins": 0.12103011459112167,
      "rewards/rejected": -0.2550780475139618,
      "step": 774
    },
    {
      "epoch": 2.1353123921297894,
      "grad_norm": 0.20496487617492676,
      "learning_rate": 1.169919808711659e-06,
      "log_odds_chosen": 1.492319941520691,
      "log_odds_ratio": -0.20643500983715057,
      "logits/chosen": -0.7351235747337341,
      "logits/rejected": -2.068713665008545,
      "logps/chosen": -1.4658230543136597,
      "logps/rejected": -2.7593133449554443,
      "loss": 1.5291,
      "nll_loss": 1.5084741115570068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14658233523368835,
      "rewards/margins": 0.1293489933013916,
      "rewards/rejected": -0.27593132853507996,
      "step": 775
    },
    {
      "epoch": 2.138073869520193,
      "grad_norm": 0.227569580078125,
      "learning_rate": 1.1631407808646758e-06,
      "log_odds_chosen": 1.5084763765335083,
      "log_odds_ratio": -0.20716872811317444,
      "logits/chosen": -0.8234575390815735,
      "logits/rejected": -1.9566833972930908,
      "logps/chosen": -1.480001449584961,
      "logps/rejected": -2.793423891067505,
      "loss": 1.5407,
      "nll_loss": 1.5199940204620361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14800015091896057,
      "rewards/margins": 0.13134226202964783,
      "rewards/rejected": -0.2793424129486084,
      "step": 776
    },
    {
      "epoch": 2.140835346910597,
      "grad_norm": 0.22688134014606476,
      "learning_rate": 1.156375491314585e-06,
      "log_odds_chosen": 1.4476473331451416,
      "log_odds_ratio": -0.21755696833133698,
      "logits/chosen": -0.8190490007400513,
      "logits/rejected": -2.084383964538574,
      "logps/chosen": -1.4168970584869385,
      "logps/rejected": -2.659780979156494,
      "loss": 1.4805,
      "nll_loss": 1.4587358236312866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1416897177696228,
      "rewards/margins": 0.12428837269544601,
      "rewards/rejected": -0.265978068113327,
      "step": 777
    },
    {
      "epoch": 2.143596824301001,
      "grad_norm": 0.21955221891403198,
      "learning_rate": 1.1496240095852002e-06,
      "log_odds_chosen": 1.4811240434646606,
      "log_odds_ratio": -0.2159479707479477,
      "logits/chosen": -0.7700929045677185,
      "logits/rejected": -2.0682449340820312,
      "logps/chosen": -1.4933757781982422,
      "logps/rejected": -2.7856602668762207,
      "loss": 1.5621,
      "nll_loss": 1.5405230522155762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14933758974075317,
      "rewards/margins": 0.12922842800617218,
      "rewards/rejected": -0.27856600284576416,
      "step": 778
    },
    {
      "epoch": 2.146358301691405,
      "grad_norm": 0.19441139698028564,
      "learning_rate": 1.142886405058445e-06,
      "log_odds_chosen": 1.5811694860458374,
      "log_odds_ratio": -0.20655685663223267,
      "logits/chosen": -0.7932131290435791,
      "logits/rejected": -2.209730863571167,
      "logps/chosen": -1.414037823677063,
      "logps/rejected": -2.7863986492156982,
      "loss": 1.4757,
      "nll_loss": 1.4550689458847046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14140377938747406,
      "rewards/margins": 0.137236088514328,
      "rewards/rejected": -0.27863985300064087,
      "step": 779
    },
    {
      "epoch": 2.1491197790818086,
      "grad_norm": 0.21649286150932312,
      "learning_rate": 1.1361627469736286e-06,
      "log_odds_chosen": 1.2342243194580078,
      "log_odds_ratio": -0.26484403014183044,
      "logits/chosen": -0.7019641995429993,
      "logits/rejected": -2.029801368713379,
      "logps/chosen": -1.4683226346969604,
      "logps/rejected": -2.5286829471588135,
      "loss": 1.5529,
      "nll_loss": 1.5264643430709839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14683228731155396,
      "rewards/margins": 0.10603601485490799,
      "rewards/rejected": -0.25286829471588135,
      "step": 780
    },
    {
      "epoch": 2.1518812564722127,
      "grad_norm": 0.21707890927791595,
      "learning_rate": 1.1294531044267418e-06,
      "log_odds_chosen": 1.2695517539978027,
      "log_odds_ratio": -0.2593621015548706,
      "logits/chosen": -0.7232626676559448,
      "logits/rejected": -1.7367572784423828,
      "logps/chosen": -1.4411171674728394,
      "logps/rejected": -2.5253968238830566,
      "loss": 1.5015,
      "nll_loss": 1.4755988121032715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1441117227077484,
      "rewards/margins": 0.1084279716014862,
      "rewards/rejected": -0.2525396943092346,
      "step": 781
    },
    {
      "epoch": 2.1546427338626164,
      "grad_norm": 0.21778395771980286,
      "learning_rate": 1.122757546369744e-06,
      "log_odds_chosen": 1.3463826179504395,
      "log_odds_ratio": -0.24496321380138397,
      "logits/chosen": -0.7988300323486328,
      "logits/rejected": -1.561537504196167,
      "logps/chosen": -1.4459059238433838,
      "logps/rejected": -2.5993716716766357,
      "loss": 1.5201,
      "nll_loss": 1.49556565284729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1445905864238739,
      "rewards/margins": 0.11534658074378967,
      "rewards/rejected": -0.2599371671676636,
      "step": 782
    },
    {
      "epoch": 2.1574042112530205,
      "grad_norm": 0.24371539056301117,
      "learning_rate": 1.1160761416098518e-06,
      "log_odds_chosen": 1.4046707153320312,
      "log_odds_ratio": -0.2345598042011261,
      "logits/chosen": -0.7829396724700928,
      "logits/rejected": -1.7336349487304688,
      "logps/chosen": -1.466633915901184,
      "logps/rejected": -2.6856021881103516,
      "loss": 1.5183,
      "nll_loss": 1.494824767112732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14666339755058289,
      "rewards/margins": 0.12189682573080063,
      "rewards/rejected": -0.2685602307319641,
      "step": 783
    },
    {
      "epoch": 2.160165688643424,
      "grad_norm": 0.2371114045381546,
      "learning_rate": 1.1094089588088383e-06,
      "log_odds_chosen": 1.6812859773635864,
      "log_odds_ratio": -0.18432164192199707,
      "logits/chosen": -0.7592156529426575,
      "logits/rejected": -1.956417202949524,
      "logps/chosen": -1.4686094522476196,
      "logps/rejected": -2.9425501823425293,
      "loss": 1.5348,
      "nll_loss": 1.5163860321044922,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14686094224452972,
      "rewards/margins": 0.1473940908908844,
      "rewards/rejected": -0.2942550480365753,
      "step": 784
    },
    {
      "epoch": 2.1629271660338283,
      "grad_norm": 0.20977692306041718,
      "learning_rate": 1.1027560664823208e-06,
      "log_odds_chosen": 1.4073948860168457,
      "log_odds_ratio": -0.2274792492389679,
      "logits/chosen": -0.7849516272544861,
      "logits/rejected": -2.003600597381592,
      "logps/chosen": -1.4130662679672241,
      "logps/rejected": -2.6155569553375244,
      "loss": 1.4807,
      "nll_loss": 1.4579068422317505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14130662381649017,
      "rewards/margins": 0.12024906277656555,
      "rewards/rejected": -0.2615557014942169,
      "step": 785
    },
    {
      "epoch": 2.165688643424232,
      "grad_norm": 0.21764487028121948,
      "learning_rate": 1.0961175329990631e-06,
      "log_odds_chosen": 1.6485837697982788,
      "log_odds_ratio": -0.1814139187335968,
      "logits/chosen": -0.7036556005477905,
      "logits/rejected": -2.169128179550171,
      "logps/chosen": -1.463451862335205,
      "logps/rejected": -2.9016788005828857,
      "loss": 1.5204,
      "nll_loss": 1.5022188425064087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14634516835212708,
      "rewards/margins": 0.14382271468639374,
      "rewards/rejected": -0.290167897939682,
      "step": 786
    },
    {
      "epoch": 2.1684501208146356,
      "grad_norm": 0.20861107110977173,
      "learning_rate": 1.0894934265802675e-06,
      "log_odds_chosen": 1.4223312139511108,
      "log_odds_ratio": -0.2421904057264328,
      "logits/chosen": -0.7688660621643066,
      "logits/rejected": -1.7660201787948608,
      "logps/chosen": -1.4704207181930542,
      "logps/rejected": -2.7102015018463135,
      "loss": 1.5322,
      "nll_loss": 1.507992148399353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14704206585884094,
      "rewards/margins": 0.12397809326648712,
      "rewards/rejected": -0.27102014422416687,
      "step": 787
    },
    {
      "epoch": 2.1712115982050397,
      "grad_norm": 0.20427848398685455,
      "learning_rate": 1.082883815298876e-06,
      "log_odds_chosen": 1.28458571434021,
      "log_odds_ratio": -0.2563609480857849,
      "logits/chosen": -0.8109991550445557,
      "logits/rejected": -1.8216277360916138,
      "logps/chosen": -1.3573540449142456,
      "logps/rejected": -2.4361014366149902,
      "loss": 1.434,
      "nll_loss": 1.4083268642425537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.135735422372818,
      "rewards/margins": 0.10787472128868103,
      "rewards/rejected": -0.24361014366149902,
      "step": 788
    },
    {
      "epoch": 2.1739730755954434,
      "grad_norm": 0.21633489429950714,
      "learning_rate": 1.0762887670788702e-06,
      "log_odds_chosen": 1.3372876644134521,
      "log_odds_ratio": -0.24584585428237915,
      "logits/chosen": -0.7131776809692383,
      "logits/rejected": -1.6811692714691162,
      "logps/chosen": -1.4008711576461792,
      "logps/rejected": -2.5390570163726807,
      "loss": 1.4978,
      "nll_loss": 1.4731900691986084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14008711278438568,
      "rewards/margins": 0.11381859332323074,
      "rewards/rejected": -0.25390568375587463,
      "step": 789
    },
    {
      "epoch": 2.1767345529858475,
      "grad_norm": 0.22346127033233643,
      "learning_rate": 1.0697083496945766e-06,
      "log_odds_chosen": 1.3910284042358398,
      "log_odds_ratio": -0.22719833254814148,
      "logits/chosen": -0.6821406483650208,
      "logits/rejected": -1.8485900163650513,
      "logps/chosen": -1.376333475112915,
      "logps/rejected": -2.557509422302246,
      "loss": 1.4589,
      "nll_loss": 1.4361891746520996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13763336837291718,
      "rewards/margins": 0.1181175708770752,
      "rewards/rejected": -0.25575095415115356,
      "step": 790
    },
    {
      "epoch": 2.179496030376251,
      "grad_norm": 0.22128628194332123,
      "learning_rate": 1.063142630769963e-06,
      "log_odds_chosen": 1.2716386318206787,
      "log_odds_ratio": -0.2559214234352112,
      "logits/chosen": -0.7769445776939392,
      "logits/rejected": -1.9463646411895752,
      "logps/chosen": -1.3476147651672363,
      "logps/rejected": -2.411182403564453,
      "loss": 1.4282,
      "nll_loss": 1.4026559591293335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1347614824771881,
      "rewards/margins": 0.10635677725076675,
      "rewards/rejected": -0.24111826717853546,
      "step": 791
    },
    {
      "epoch": 2.1822575077666553,
      "grad_norm": 0.21540698409080505,
      "learning_rate": 1.0565916777779519e-06,
      "log_odds_chosen": 1.4014261960983276,
      "log_odds_ratio": -0.2265271246433258,
      "logits/chosen": -0.7393918633460999,
      "logits/rejected": -1.8673772811889648,
      "logps/chosen": -1.4762778282165527,
      "logps/rejected": -2.6873135566711426,
      "loss": 1.5394,
      "nll_loss": 1.5167272090911865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476277858018875,
      "rewards/margins": 0.12110358476638794,
      "rewards/rejected": -0.26873135566711426,
      "step": 792
    },
    {
      "epoch": 2.185018985157059,
      "grad_norm": 0.21713268756866455,
      "learning_rate": 1.0500555580397193e-06,
      "log_odds_chosen": 1.5545601844787598,
      "log_odds_ratio": -0.20979206264019012,
      "logits/chosen": -0.7159947752952576,
      "logits/rejected": -1.8904709815979004,
      "logps/chosen": -1.4165940284729004,
      "logps/rejected": -2.763000965118408,
      "loss": 1.483,
      "nll_loss": 1.4620165824890137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14165939390659332,
      "rewards/margins": 0.13464069366455078,
      "rewards/rejected": -0.2763000726699829,
      "step": 793
    },
    {
      "epoch": 2.187780462547463,
      "grad_norm": 0.22529271245002747,
      "learning_rate": 1.04353433872401e-06,
      "log_odds_chosen": 1.3307311534881592,
      "log_odds_ratio": -0.238363116979599,
      "logits/chosen": -0.7927432060241699,
      "logits/rejected": -1.904407262802124,
      "logps/chosen": -1.4591866731643677,
      "logps/rejected": -2.602426290512085,
      "loss": 1.5187,
      "nll_loss": 1.4948735237121582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14591866731643677,
      "rewards/margins": 0.11432396620512009,
      "rewards/rejected": -0.26024264097213745,
      "step": 794
    },
    {
      "epoch": 2.1905419399378667,
      "grad_norm": 0.20850500464439392,
      "learning_rate": 1.0370280868464405e-06,
      "log_odds_chosen": 1.777038335800171,
      "log_odds_ratio": -0.17781738936901093,
      "logits/chosen": -0.7591485977172852,
      "logits/rejected": -2.2028274536132812,
      "logps/chosen": -1.424153208732605,
      "logps/rejected": -2.9798388481140137,
      "loss": 1.4756,
      "nll_loss": 1.4577784538269043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14241531491279602,
      "rewards/margins": 0.15556861460208893,
      "rewards/rejected": -0.29798394441604614,
      "step": 795
    },
    {
      "epoch": 2.193303417328271,
      "grad_norm": 0.22121626138687134,
      "learning_rate": 1.0305368692688175e-06,
      "log_odds_chosen": 1.6756552457809448,
      "log_odds_ratio": -0.180609792470932,
      "logits/chosen": -0.7418386936187744,
      "logits/rejected": -1.983974814414978,
      "logps/chosen": -1.327376365661621,
      "logps/rejected": -2.759260654449463,
      "loss": 1.3908,
      "nll_loss": 1.3727234601974487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1327376365661621,
      "rewards/margins": 0.14318843185901642,
      "rewards/rejected": -0.2759261131286621,
      "step": 796
    },
    {
      "epoch": 2.1960648947186745,
      "grad_norm": 0.2660160958766937,
      "learning_rate": 1.0240607526984437e-06,
      "log_odds_chosen": 1.6831055879592896,
      "log_odds_ratio": -0.18040511012077332,
      "logits/chosen": -0.8147282004356384,
      "logits/rejected": -1.9211832284927368,
      "logps/chosen": -1.4013046026229858,
      "logps/rejected": -2.8633060455322266,
      "loss": 1.4634,
      "nll_loss": 1.4453954696655273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14013046026229858,
      "rewards/margins": 0.14620015025138855,
      "rewards/rejected": -0.28633058071136475,
      "step": 797
    },
    {
      "epoch": 2.198826372109078,
      "grad_norm": 0.2278819978237152,
      "learning_rate": 1.0175998036874357e-06,
      "log_odds_chosen": 1.3868612051010132,
      "log_odds_ratio": -0.22797061502933502,
      "logits/chosen": -0.8481411933898926,
      "logits/rejected": -1.837113380432129,
      "logps/chosen": -1.3668131828308105,
      "logps/rejected": -2.541306257247925,
      "loss": 1.4386,
      "nll_loss": 1.415839433670044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13668133318424225,
      "rewards/margins": 0.11744929850101471,
      "rewards/rejected": -0.25413063168525696,
      "step": 798
    },
    {
      "epoch": 2.2015878494994823,
      "grad_norm": 0.22600935399532318,
      "learning_rate": 1.0111540886320433e-06,
      "log_odds_chosen": 1.3912237882614136,
      "log_odds_ratio": -0.22471024096012115,
      "logits/chosen": -0.7584688663482666,
      "logits/rejected": -1.8486192226409912,
      "logps/chosen": -1.4408618211746216,
      "logps/rejected": -2.633699893951416,
      "loss": 1.5085,
      "nll_loss": 1.4860626459121704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14408619701862335,
      "rewards/margins": 0.11928381770849228,
      "rewards/rejected": -0.2633700370788574,
      "step": 799
    },
    {
      "epoch": 2.204349326889886,
      "grad_norm": 0.21976998448371887,
      "learning_rate": 1.00472367377196e-06,
      "log_odds_chosen": 1.375922441482544,
      "log_odds_ratio": -0.23655349016189575,
      "logits/chosen": -0.8409130573272705,
      "logits/rejected": -1.8534971475601196,
      "logps/chosen": -1.428146481513977,
      "logps/rejected": -2.6036124229431152,
      "loss": 1.4798,
      "nll_loss": 1.4561710357666016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14281465113162994,
      "rewards/margins": 0.11754661053419113,
      "rewards/rejected": -0.2603612542152405,
      "step": 800
    },
    {
      "epoch": 2.20711080428029,
      "grad_norm": 0.2096586525440216,
      "learning_rate": 9.983086251896493e-07,
      "log_odds_chosen": 1.4792360067367554,
      "log_odds_ratio": -0.2164044976234436,
      "logits/chosen": -0.8477017879486084,
      "logits/rejected": -2.007514476776123,
      "logps/chosen": -1.4218276739120483,
      "logps/rejected": -2.697108030319214,
      "loss": 1.4754,
      "nll_loss": 1.453714370727539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14218276739120483,
      "rewards/margins": 0.12752802670001984,
      "rewards/rejected": -0.2697107791900635,
      "step": 801
    },
    {
      "epoch": 2.2098722816706937,
      "grad_norm": 0.31831514835357666,
      "learning_rate": 9.91909008809659e-07,
      "log_odds_chosen": 1.3559105396270752,
      "log_odds_ratio": -0.24869945645332336,
      "logits/chosen": -0.6960320472717285,
      "logits/rejected": -1.9635218381881714,
      "logps/chosen": -1.4762076139450073,
      "logps/rejected": -2.649646043777466,
      "loss": 1.5351,
      "nll_loss": 1.5102181434631348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476207673549652,
      "rewards/margins": 0.11734385043382645,
      "rewards/rejected": -0.26496461033821106,
      "step": 802
    },
    {
      "epoch": 2.212633759061098,
      "grad_norm": 0.23091106116771698,
      "learning_rate": 9.855248903979505e-07,
      "log_odds_chosen": 1.436722993850708,
      "log_odds_ratio": -0.21449066698551178,
      "logits/chosen": -0.8235598802566528,
      "logits/rejected": -2.0638046264648438,
      "logps/chosen": -1.463809847831726,
      "logps/rejected": -2.703613519668579,
      "loss": 1.5394,
      "nll_loss": 1.517941951751709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14638099074363708,
      "rewards/margins": 0.12398036569356918,
      "rewards/rejected": -0.27036136388778687,
      "step": 803
    },
    {
      "epoch": 2.2153952364515015,
      "grad_norm": 0.2459157109260559,
      "learning_rate": 9.791563355612171e-07,
      "log_odds_chosen": 1.2237358093261719,
      "log_odds_ratio": -0.2616325318813324,
      "logits/chosen": -0.7013297080993652,
      "logits/rejected": -1.9213218688964844,
      "logps/chosen": -1.5507513284683228,
      "logps/rejected": -2.6120126247406006,
      "loss": 1.613,
      "nll_loss": 1.586790919303894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15507514774799347,
      "rewards/margins": 0.10612612217664719,
      "rewards/rejected": -0.26120126247406006,
      "step": 804
    },
    {
      "epoch": 2.2181567138419056,
      "grad_norm": 0.2175196260213852,
      "learning_rate": 9.728034097462144e-07,
      "log_odds_chosen": 1.8408254384994507,
      "log_odds_ratio": -0.16706471145153046,
      "logits/chosen": -0.7350028157234192,
      "logits/rejected": -2.162047863006592,
      "logps/chosen": -1.3898506164550781,
      "logps/rejected": -2.9982106685638428,
      "loss": 1.4664,
      "nll_loss": 1.4496983289718628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1389850676059723,
      "rewards/margins": 0.16083598136901855,
      "rewards/rejected": -0.29982104897499084,
      "step": 805
    },
    {
      "epoch": 2.2209181912323093,
      "grad_norm": 0.22365769743919373,
      "learning_rate": 9.66466178239084e-07,
      "log_odds_chosen": 1.4642393589019775,
      "log_odds_ratio": -0.23663140833377838,
      "logits/chosen": -0.7512733340263367,
      "logits/rejected": -2.0906615257263184,
      "logps/chosen": -1.4405597448349,
      "logps/rejected": -2.709528923034668,
      "loss": 1.4955,
      "nll_loss": 1.4718680381774902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14405599236488342,
      "rewards/margins": 0.1268969029188156,
      "rewards/rejected": -0.27095291018486023,
      "step": 806
    },
    {
      "epoch": 2.2236796686227134,
      "grad_norm": 0.2207762897014618,
      "learning_rate": 9.601447061646838e-07,
      "log_odds_chosen": 1.2826145887374878,
      "log_odds_ratio": -0.247898131608963,
      "logits/chosen": -0.9267134666442871,
      "logits/rejected": -1.7989857196807861,
      "logps/chosen": -1.3823951482772827,
      "logps/rejected": -2.466029644012451,
      "loss": 1.4549,
      "nll_loss": 1.4300906658172607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1382395178079605,
      "rewards/margins": 0.10836344957351685,
      "rewards/rejected": -0.24660295248031616,
      "step": 807
    },
    {
      "epoch": 2.226441146013117,
      "grad_norm": 0.22479695081710815,
      "learning_rate": 9.538390584859213e-07,
      "log_odds_chosen": 1.5794352293014526,
      "log_odds_ratio": -0.1960069239139557,
      "logits/chosen": -0.7440246939659119,
      "logits/rejected": -1.8689384460449219,
      "logps/chosen": -1.3518363237380981,
      "logps/rejected": -2.699951171875,
      "loss": 1.4346,
      "nll_loss": 1.4149811267852783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13518363237380981,
      "rewards/margins": 0.13481149077415466,
      "rewards/rejected": -0.2699951231479645,
      "step": 808
    },
    {
      "epoch": 2.2292026234035207,
      "grad_norm": 0.2177855521440506,
      "learning_rate": 9.47549300003082e-07,
      "log_odds_chosen": 1.5709489583969116,
      "log_odds_ratio": -0.19270090758800507,
      "logits/chosen": -0.8938512802124023,
      "logits/rejected": -1.9860432147979736,
      "logps/chosen": -1.3834973573684692,
      "logps/rejected": -2.7306644916534424,
      "loss": 1.4517,
      "nll_loss": 1.432417869567871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1383497416973114,
      "rewards/margins": 0.1347167193889618,
      "rewards/rejected": -0.2730664610862732,
      "step": 809
    },
    {
      "epoch": 2.231964100793925,
      "grad_norm": 0.24030561745166779,
      "learning_rate": 9.412754953531664e-07,
      "log_odds_chosen": 1.81654691696167,
      "log_odds_ratio": -0.1660957783460617,
      "logits/chosen": -0.7898241877555847,
      "logits/rejected": -2.001150369644165,
      "logps/chosen": -1.365299940109253,
      "logps/rejected": -2.937218189239502,
      "loss": 1.4319,
      "nll_loss": 1.4152500629425049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13652999699115753,
      "rewards/margins": 0.15719182789325714,
      "rewards/rejected": -0.2937218248844147,
      "step": 810
    },
    {
      "epoch": 2.2347255781843285,
      "grad_norm": 0.21561409533023834,
      "learning_rate": 9.350177090092233e-07,
      "log_odds_chosen": 1.6017990112304688,
      "log_odds_ratio": -0.19290661811828613,
      "logits/chosen": -0.663612425327301,
      "logits/rejected": -2.0358691215515137,
      "logps/chosen": -1.4089775085449219,
      "logps/rejected": -2.790071487426758,
      "loss": 1.4723,
      "nll_loss": 1.453056812286377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1408977508544922,
      "rewards/margins": 0.13810938596725464,
      "rewards/rejected": -0.2790071368217468,
      "step": 811
    },
    {
      "epoch": 2.2374870555747326,
      "grad_norm": 0.22939537465572357,
      "learning_rate": 9.28776005279691e-07,
      "log_odds_chosen": 1.6415590047836304,
      "log_odds_ratio": -0.18874233961105347,
      "logits/chosen": -0.7984322905540466,
      "logits/rejected": -2.1387336254119873,
      "logps/chosen": -1.4797344207763672,
      "logps/rejected": -2.917440414428711,
      "loss": 1.5297,
      "nll_loss": 1.5108166933059692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479734182357788,
      "rewards/margins": 0.14377063512802124,
      "rewards/rejected": -0.29174405336380005,
      "step": 812
    },
    {
      "epoch": 2.2402485329651363,
      "grad_norm": 0.21815143525600433,
      "learning_rate": 9.225504483077311e-07,
      "log_odds_chosen": 1.6695032119750977,
      "log_odds_ratio": -0.1855509877204895,
      "logits/chosen": -0.8288291692733765,
      "logits/rejected": -2.1964502334594727,
      "logps/chosen": -1.3856902122497559,
      "logps/rejected": -2.8318369388580322,
      "loss": 1.4392,
      "nll_loss": 1.420630931854248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13856902718544006,
      "rewards/margins": 0.14461466670036316,
      "rewards/rejected": -0.28318366408348083,
      "step": 813
    },
    {
      "epoch": 2.2430100103555404,
      "grad_norm": 0.2161535620689392,
      "learning_rate": 9.163411020705762e-07,
      "log_odds_chosen": 1.3703440427780151,
      "log_odds_ratio": -0.24051517248153687,
      "logits/chosen": -0.7353973984718323,
      "logits/rejected": -2.010030746459961,
      "logps/chosen": -1.3800489902496338,
      "logps/rejected": -2.546851873397827,
      "loss": 1.4496,
      "nll_loss": 1.4255536794662476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13800489902496338,
      "rewards/margins": 0.11668029427528381,
      "rewards/rejected": -0.2546851933002472,
      "step": 814
    },
    {
      "epoch": 2.245771487745944,
      "grad_norm": 0.22893516719341278,
      "learning_rate": 9.101480303788623e-07,
      "log_odds_chosen": 1.2863377332687378,
      "log_odds_ratio": -0.24850118160247803,
      "logits/chosen": -0.7366392016410828,
      "logits/rejected": -1.659240484237671,
      "logps/chosen": -1.3816016912460327,
      "logps/rejected": -2.464106798171997,
      "loss": 1.457,
      "nll_loss": 1.4321224689483643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13816016912460327,
      "rewards/margins": 0.10825051367282867,
      "rewards/rejected": -0.24641066789627075,
      "step": 815
    },
    {
      "epoch": 2.248532965136348,
      "grad_norm": 0.21422047913074493,
      "learning_rate": 9.039712968759864e-07,
      "log_odds_chosen": 1.591874361038208,
      "log_odds_ratio": -0.19570983946323395,
      "logits/chosen": -0.7051729559898376,
      "logits/rejected": -2.0805304050445557,
      "logps/chosen": -1.4287737607955933,
      "logps/rejected": -2.8088440895080566,
      "loss": 1.5089,
      "nll_loss": 1.4893687963485718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14287738502025604,
      "rewards/margins": 0.1380070298910141,
      "rewards/rejected": -0.28088441491127014,
      "step": 816
    },
    {
      "epoch": 2.251294442526752,
      "grad_norm": 0.20570261776447296,
      "learning_rate": 8.978109650374398e-07,
      "log_odds_chosen": 1.599745273590088,
      "log_odds_ratio": -0.1869451403617859,
      "logits/chosen": -0.7604836821556091,
      "logits/rejected": -2.0949907302856445,
      "logps/chosen": -1.4347954988479614,
      "logps/rejected": -2.8207342624664307,
      "loss": 1.5008,
      "nll_loss": 1.482066035270691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14347955584526062,
      "rewards/margins": 0.1385938823223114,
      "rewards/rejected": -0.28207340836524963,
      "step": 817
    },
    {
      "epoch": 2.254055919917156,
      "grad_norm": 0.24999411404132843,
      "learning_rate": 8.916670981701656e-07,
      "log_odds_chosen": 1.4131107330322266,
      "log_odds_ratio": -0.22106236219406128,
      "logits/chosen": -0.794218122959137,
      "logits/rejected": -2.1165714263916016,
      "logps/chosen": -1.517124056816101,
      "logps/rejected": -2.7476296424865723,
      "loss": 1.5648,
      "nll_loss": 1.5427284240722656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15171240270137787,
      "rewards/margins": 0.12305056303739548,
      "rewards/rejected": -0.27476298809051514,
      "step": 818
    },
    {
      "epoch": 2.2568173973075596,
      "grad_norm": 0.21286356449127197,
      "learning_rate": 8.855397594118995e-07,
      "log_odds_chosen": 1.3657292127609253,
      "log_odds_ratio": -0.24322526156902313,
      "logits/chosen": -0.7253862619400024,
      "logits/rejected": -1.8961949348449707,
      "logps/chosen": -1.3823702335357666,
      "logps/rejected": -2.545137882232666,
      "loss": 1.4589,
      "nll_loss": 1.4345393180847168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13823701441287994,
      "rewards/margins": 0.11627674847841263,
      "rewards/rejected": -0.25451380014419556,
      "step": 819
    },
    {
      "epoch": 2.2595788746979633,
      "grad_norm": 0.21439580619335175,
      "learning_rate": 8.794290117305296e-07,
      "log_odds_chosen": 1.5430362224578857,
      "log_odds_ratio": -0.19931325316429138,
      "logits/chosen": -0.8306599259376526,
      "logits/rejected": -2.151496171951294,
      "logps/chosen": -1.3482749462127686,
      "logps/rejected": -2.660677433013916,
      "loss": 1.4238,
      "nll_loss": 1.40384042263031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13482749462127686,
      "rewards/margins": 0.13124024868011475,
      "rewards/rejected": -0.2660677433013916,
      "step": 820
    },
    {
      "epoch": 2.2623403520883674,
      "grad_norm": 0.21566608548164368,
      "learning_rate": 8.733349179234412e-07,
      "log_odds_chosen": 1.4808435440063477,
      "log_odds_ratio": -0.22421996295452118,
      "logits/chosen": -0.8796311616897583,
      "logits/rejected": -1.8572511672973633,
      "logps/chosen": -1.3629071712493896,
      "logps/rejected": -2.6273839473724365,
      "loss": 1.4427,
      "nll_loss": 1.4202845096588135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13629071414470673,
      "rewards/margins": 0.12644769251346588,
      "rewards/rejected": -0.2627384066581726,
      "step": 821
    },
    {
      "epoch": 2.265101829478771,
      "grad_norm": 0.2407940924167633,
      "learning_rate": 8.672575406168782e-07,
      "log_odds_chosen": 1.3748390674591064,
      "log_odds_ratio": -0.2400941550731659,
      "logits/chosen": -0.7804903984069824,
      "logits/rejected": -1.6939703226089478,
      "logps/chosen": -1.4525799751281738,
      "logps/rejected": -2.6338768005371094,
      "loss": 1.5231,
      "nll_loss": 1.4990655183792114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14525800943374634,
      "rewards/margins": 0.1181296706199646,
      "rewards/rejected": -0.26338768005371094,
      "step": 822
    },
    {
      "epoch": 2.267863306869175,
      "grad_norm": 0.21523721516132355,
      "learning_rate": 8.611969422652966e-07,
      "log_odds_chosen": 1.5803931951522827,
      "log_odds_ratio": -0.20858606696128845,
      "logits/chosen": -0.7921524047851562,
      "logits/rejected": -1.8999464511871338,
      "logps/chosen": -1.3710713386535645,
      "logps/rejected": -2.7342100143432617,
      "loss": 1.4605,
      "nll_loss": 1.4396417140960693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13710711896419525,
      "rewards/margins": 0.13631387054920197,
      "rewards/rejected": -0.2734209895133972,
      "step": 823
    },
    {
      "epoch": 2.270624784259579,
      "grad_norm": 0.21190094947814941,
      "learning_rate": 8.551531851507186e-07,
      "log_odds_chosen": 1.431606411933899,
      "log_odds_ratio": -0.22368153929710388,
      "logits/chosen": -0.7815225720405579,
      "logits/rejected": -1.8217527866363525,
      "logps/chosen": -1.3981519937515259,
      "logps/rejected": -2.6237053871154785,
      "loss": 1.4763,
      "nll_loss": 1.4538975954055786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13981519639492035,
      "rewards/margins": 0.12255533784627914,
      "rewards/rejected": -0.2623705565929413,
      "step": 824
    },
    {
      "epoch": 2.273386261649983,
      "grad_norm": 0.2221967726945877,
      "learning_rate": 8.491263313821021e-07,
      "log_odds_chosen": 1.438922643661499,
      "log_odds_ratio": -0.22679126262664795,
      "logits/chosen": -0.8472309112548828,
      "logits/rejected": -1.8394192457199097,
      "logps/chosen": -1.4349843263626099,
      "logps/rejected": -2.6651053428649902,
      "loss": 1.507,
      "nll_loss": 1.4843312501907349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14349845051765442,
      "rewards/margins": 0.12301210314035416,
      "rewards/rejected": -0.266510546207428,
      "step": 825
    },
    {
      "epoch": 2.2761477390403866,
      "grad_norm": 0.27319595217704773,
      "learning_rate": 8.431164428946928e-07,
      "log_odds_chosen": 1.5053706169128418,
      "log_odds_ratio": -0.21574383974075317,
      "logits/chosen": -0.9391317367553711,
      "logits/rejected": -1.8469014167785645,
      "logps/chosen": -1.4213306903839111,
      "logps/rejected": -2.715855121612549,
      "loss": 1.4929,
      "nll_loss": 1.4712984561920166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14213307201862335,
      "rewards/margins": 0.12945246696472168,
      "rewards/rejected": -0.2715855538845062,
      "step": 826
    },
    {
      "epoch": 2.2789092164307903,
      "grad_norm": 0.2336154580116272,
      "learning_rate": 8.371235814493947e-07,
      "log_odds_chosen": 1.6580530405044556,
      "log_odds_ratio": -0.1864277720451355,
      "logits/chosen": -0.8675610423088074,
      "logits/rejected": -2.160818338394165,
      "logps/chosen": -1.4152588844299316,
      "logps/rejected": -2.8566691875457764,
      "loss": 1.4831,
      "nll_loss": 1.4644639492034912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14152587950229645,
      "rewards/margins": 0.14414101839065552,
      "rewards/rejected": -0.28566691279411316,
      "step": 827
    },
    {
      "epoch": 2.2816706938211944,
      "grad_norm": 0.21498602628707886,
      "learning_rate": 8.311478086321301e-07,
      "log_odds_chosen": 1.4063800573349,
      "log_odds_ratio": -0.22114284336566925,
      "logits/chosen": -0.7863494753837585,
      "logits/rejected": -1.8558886051177979,
      "logps/chosen": -1.3713574409484863,
      "logps/rejected": -2.564828395843506,
      "loss": 1.4365,
      "nll_loss": 1.4143601655960083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13713574409484863,
      "rewards/margins": 0.11934711039066315,
      "rewards/rejected": -0.2564828395843506,
      "step": 828
    },
    {
      "epoch": 2.284432171211598,
      "grad_norm": 0.20723487436771393,
      "learning_rate": 8.251891858532124e-07,
      "log_odds_chosen": 1.6476480960845947,
      "log_odds_ratio": -0.1784750372171402,
      "logits/chosen": -0.8183894753456116,
      "logits/rejected": -2.0899806022644043,
      "logps/chosen": -1.3663532733917236,
      "logps/rejected": -2.7823915481567383,
      "loss": 1.4319,
      "nll_loss": 1.4140135049819946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13663533329963684,
      "rewards/margins": 0.14160385727882385,
      "rewards/rejected": -0.2782391905784607,
      "step": 829
    },
    {
      "epoch": 2.287193648602002,
      "grad_norm": 0.2511179447174072,
      "learning_rate": 8.192477743467078e-07,
      "log_odds_chosen": 1.433205485343933,
      "log_odds_ratio": -0.2308994084596634,
      "logits/chosen": -0.9073160886764526,
      "logits/rejected": -1.5418671369552612,
      "logps/chosen": -1.4494136571884155,
      "logps/rejected": -2.689509630203247,
      "loss": 1.5321,
      "nll_loss": 1.5090384483337402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14494135975837708,
      "rewards/margins": 0.12400959432125092,
      "rewards/rejected": -0.2689509689807892,
      "step": 830
    },
    {
      "epoch": 2.289955125992406,
      "grad_norm": 0.22408699989318848,
      "learning_rate": 8.133236351698143e-07,
      "log_odds_chosen": 1.4096298217773438,
      "log_odds_ratio": -0.22869807481765747,
      "logits/chosen": -0.7961119413375854,
      "logits/rejected": -1.8842319250106812,
      "logps/chosen": -1.4202234745025635,
      "logps/rejected": -2.6291675567626953,
      "loss": 1.4995,
      "nll_loss": 1.4766323566436768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14202234148979187,
      "rewards/margins": 0.1208944171667099,
      "rewards/rejected": -0.26291677355766296,
      "step": 831
    },
    {
      "epoch": 2.29271660338281,
      "grad_norm": 0.24003839492797852,
      "learning_rate": 8.074168292022269e-07,
      "log_odds_chosen": 1.4157015085220337,
      "log_odds_ratio": -0.2238752245903015,
      "logits/chosen": -0.7954575419425964,
      "logits/rejected": -1.9194411039352417,
      "logps/chosen": -1.4735336303710938,
      "logps/rejected": -2.6962783336639404,
      "loss": 1.541,
      "nll_loss": 1.5186426639556885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1473533660173416,
      "rewards/margins": 0.1222744733095169,
      "rewards/rejected": -0.2696278393268585,
      "step": 832
    },
    {
      "epoch": 2.2954780807732136,
      "grad_norm": 0.22047549486160278,
      "learning_rate": 8.015274171455151e-07,
      "log_odds_chosen": 1.5765596628189087,
      "log_odds_ratio": -0.1994118094444275,
      "logits/chosen": -0.7459832429885864,
      "logits/rejected": -2.163300037384033,
      "logps/chosen": -1.4016599655151367,
      "logps/rejected": -2.763180732727051,
      "loss": 1.4583,
      "nll_loss": 1.4383318424224854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1401660144329071,
      "rewards/margins": 0.13615208864212036,
      "rewards/rejected": -0.27631810307502747,
      "step": 833
    },
    {
      "epoch": 2.2982395581636177,
      "grad_norm": 0.21983671188354492,
      "learning_rate": 7.956554595225017e-07,
      "log_odds_chosen": 1.4907373189926147,
      "log_odds_ratio": -0.211158886551857,
      "logits/chosen": -0.962412416934967,
      "logits/rejected": -1.9069632291793823,
      "logps/chosen": -1.3121854066848755,
      "logps/rejected": -2.571423292160034,
      "loss": 1.3831,
      "nll_loss": 1.3619731664657593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1312185525894165,
      "rewards/margins": 0.1259237825870514,
      "rewards/rejected": -0.2571423351764679,
      "step": 834
    },
    {
      "epoch": 2.3010010355540214,
      "grad_norm": 0.2331991195678711,
      "learning_rate": 7.898010166766348e-07,
      "log_odds_chosen": 1.552021861076355,
      "log_odds_ratio": -0.20787937939167023,
      "logits/chosen": -0.8280553221702576,
      "logits/rejected": -1.8152735233306885,
      "logps/chosen": -1.4129295349121094,
      "logps/rejected": -2.754270315170288,
      "loss": 1.483,
      "nll_loss": 1.4621742963790894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412929743528366,
      "rewards/margins": 0.13413408398628235,
      "rewards/rejected": -0.27542704343795776,
      "step": 835
    },
    {
      "epoch": 2.3037625129444255,
      "grad_norm": 0.219749316573143,
      "learning_rate": 7.839641487713745e-07,
      "log_odds_chosen": 1.4371466636657715,
      "log_odds_ratio": -0.22468623518943787,
      "logits/chosen": -0.820203423500061,
      "logits/rejected": -1.8462026119232178,
      "logps/chosen": -1.366627812385559,
      "logps/rejected": -2.589350461959839,
      "loss": 1.4294,
      "nll_loss": 1.4069098234176636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1366627812385559,
      "rewards/margins": 0.12227226793766022,
      "rewards/rejected": -0.25893503427505493,
      "step": 836
    },
    {
      "epoch": 2.306523990334829,
      "grad_norm": 0.23122566938400269,
      "learning_rate": 7.781449157895677e-07,
      "log_odds_chosen": 1.547440767288208,
      "log_odds_ratio": -0.1968943178653717,
      "logits/chosen": -0.9170833826065063,
      "logits/rejected": -2.0454964637756348,
      "logps/chosen": -1.455304741859436,
      "logps/rejected": -2.797694683074951,
      "loss": 1.5188,
      "nll_loss": 1.4990651607513428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14553046226501465,
      "rewards/margins": 0.13423900306224823,
      "rewards/rejected": -0.2797694504261017,
      "step": 837
    },
    {
      "epoch": 2.309285467725233,
      "grad_norm": 0.2177518755197525,
      "learning_rate": 7.723433775328385e-07,
      "log_odds_chosen": 2.1011574268341064,
      "log_odds_ratio": -0.13231369853019714,
      "logits/chosen": -0.8008699417114258,
      "logits/rejected": -2.206787586212158,
      "logps/chosen": -1.3365592956542969,
      "logps/rejected": -3.181867837905884,
      "loss": 1.4069,
      "nll_loss": 1.3936563730239868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13365595042705536,
      "rewards/margins": 0.1845308095216751,
      "rewards/rejected": -0.31818675994873047,
      "step": 838
    },
    {
      "epoch": 2.312046945115637,
      "grad_norm": 0.24736399948596954,
      "learning_rate": 7.665595936209674e-07,
      "log_odds_chosen": 1.336047887802124,
      "log_odds_ratio": -0.23815786838531494,
      "logits/chosen": -0.7619213461875916,
      "logits/rejected": -1.9846187829971313,
      "logps/chosen": -1.5085265636444092,
      "logps/rejected": -2.6583876609802246,
      "loss": 1.5721,
      "nll_loss": 1.5482698678970337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15085266530513763,
      "rewards/margins": 0.1149861216545105,
      "rewards/rejected": -0.2658388018608093,
      "step": 839
    },
    {
      "epoch": 2.3148084225060406,
      "grad_norm": 0.21641425788402557,
      "learning_rate": 7.607936234912841e-07,
      "log_odds_chosen": 1.5291671752929688,
      "log_odds_ratio": -0.2096787989139557,
      "logits/chosen": -0.8673451542854309,
      "logits/rejected": -1.9320155382156372,
      "logps/chosen": -1.3398340940475464,
      "logps/rejected": -2.6366426944732666,
      "loss": 1.4023,
      "nll_loss": 1.3813666105270386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13398341834545135,
      "rewards/margins": 0.12968085706233978,
      "rewards/rejected": -0.26366427540779114,
      "step": 840
    },
    {
      "epoch": 2.3175698998964447,
      "grad_norm": 0.22011668980121613,
      "learning_rate": 7.550455263980525e-07,
      "log_odds_chosen": 1.4579017162322998,
      "log_odds_ratio": -0.21185196936130524,
      "logits/chosen": -0.8572349548339844,
      "logits/rejected": -2.1599864959716797,
      "logps/chosen": -1.4740046262741089,
      "logps/rejected": -2.7385215759277344,
      "loss": 1.5327,
      "nll_loss": 1.5115079879760742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14740046858787537,
      "rewards/margins": 0.12645168602466583,
      "rewards/rejected": -0.2738521695137024,
      "step": 841
    },
    {
      "epoch": 2.3203313772868484,
      "grad_norm": 0.22545059025287628,
      "learning_rate": 7.493153614118634e-07,
      "log_odds_chosen": 1.755352258682251,
      "log_odds_ratio": -0.16482490301132202,
      "logits/chosen": -0.8975868225097656,
      "logits/rejected": -1.90993332862854,
      "logps/chosen": -1.3448066711425781,
      "logps/rejected": -2.853928565979004,
      "loss": 1.427,
      "nll_loss": 1.4105489253997803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13448068499565125,
      "rewards/margins": 0.15091221034526825,
      "rewards/rejected": -0.2853928804397583,
      "step": 842
    },
    {
      "epoch": 2.3230928546772525,
      "grad_norm": 0.22612684965133667,
      "learning_rate": 7.436031874190272e-07,
      "log_odds_chosen": 1.5865403413772583,
      "log_odds_ratio": -0.19421318173408508,
      "logits/chosen": -0.7267550826072693,
      "logits/rejected": -2.0342113971710205,
      "logps/chosen": -1.4796632528305054,
      "logps/rejected": -2.8678059577941895,
      "loss": 1.5443,
      "nll_loss": 1.5248392820358276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14796634018421173,
      "rewards/margins": 0.13881425559520721,
      "rewards/rejected": -0.28678056597709656,
      "step": 843
    },
    {
      "epoch": 2.325854332067656,
      "grad_norm": 0.2235104888677597,
      "learning_rate": 7.379090631209712e-07,
      "log_odds_chosen": 1.61055326461792,
      "log_odds_ratio": -0.19099710881710052,
      "logits/chosen": -0.8531662821769714,
      "logits/rejected": -2.0457820892333984,
      "logps/chosen": -1.3573286533355713,
      "logps/rejected": -2.737484931945801,
      "loss": 1.4251,
      "nll_loss": 1.4059858322143555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13573288917541504,
      "rewards/margins": 0.13801559805870056,
      "rewards/rejected": -0.2737484872341156,
      "step": 844
    },
    {
      "epoch": 2.3286158094580602,
      "grad_norm": 0.21737630665302277,
      "learning_rate": 7.322330470336314e-07,
      "log_odds_chosen": 1.5560356378555298,
      "log_odds_ratio": -0.19873520731925964,
      "logits/chosen": -0.8432495594024658,
      "logits/rejected": -1.9900035858154297,
      "logps/chosen": -1.3147186040878296,
      "logps/rejected": -2.630089282989502,
      "loss": 1.391,
      "nll_loss": 1.371086835861206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13147185742855072,
      "rewards/margins": 0.1315370798110962,
      "rewards/rejected": -0.2630089521408081,
      "step": 845
    },
    {
      "epoch": 2.331377286848464,
      "grad_norm": 0.22583864629268646,
      "learning_rate": 7.265751974868554e-07,
      "log_odds_chosen": 1.4394731521606445,
      "log_odds_ratio": -0.22187888622283936,
      "logits/chosen": -0.8381003141403198,
      "logits/rejected": -1.7541844844818115,
      "logps/chosen": -1.3545819520950317,
      "logps/rejected": -2.5769851207733154,
      "loss": 1.4338,
      "nll_loss": 1.411636471748352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13545820116996765,
      "rewards/margins": 0.12224029004573822,
      "rewards/rejected": -0.25769850611686707,
      "step": 846
    },
    {
      "epoch": 2.334138764238868,
      "grad_norm": 0.23510843515396118,
      "learning_rate": 7.209355726238024e-07,
      "log_odds_chosen": 1.6167476177215576,
      "log_odds_ratio": -0.20064175128936768,
      "logits/chosen": -0.8486358523368835,
      "logits/rejected": -2.045172929763794,
      "logps/chosen": -1.4668694734573364,
      "logps/rejected": -2.884349822998047,
      "loss": 1.5193,
      "nll_loss": 1.4991921186447144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14668695628643036,
      "rewards/margins": 0.14174804091453552,
      "rewards/rejected": -0.2884350121021271,
      "step": 847
    },
    {
      "epoch": 2.3369002416292717,
      "grad_norm": 0.2219594568014145,
      "learning_rate": 7.153142304003419e-07,
      "log_odds_chosen": 1.559194564819336,
      "log_odds_ratio": -0.20855563879013062,
      "logits/chosen": -0.820976734161377,
      "logits/rejected": -1.947663426399231,
      "logps/chosen": -1.4534995555877686,
      "logps/rejected": -2.8155107498168945,
      "loss": 1.5203,
      "nll_loss": 1.4994096755981445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14534996449947357,
      "rewards/margins": 0.1362011432647705,
      "rewards/rejected": -0.2815510928630829,
      "step": 848
    },
    {
      "epoch": 2.3396617190196753,
      "grad_norm": 0.22509777545928955,
      "learning_rate": 7.097112285844643e-07,
      "log_odds_chosen": 1.5197633504867554,
      "log_odds_ratio": -0.2113376408815384,
      "logits/chosen": -0.8353469967842102,
      "logits/rejected": -1.7747441530227661,
      "logps/chosen": -1.3918851613998413,
      "logps/rejected": -2.6959621906280518,
      "loss": 1.4579,
      "nll_loss": 1.4367426633834839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1391885131597519,
      "rewards/margins": 0.13040770590305328,
      "rewards/rejected": -0.2695962190628052,
      "step": 849
    },
    {
      "epoch": 2.3424231964100795,
      "grad_norm": 0.22396017611026764,
      "learning_rate": 7.041266247556814e-07,
      "log_odds_chosen": 1.5306124687194824,
      "log_odds_ratio": -0.2112870216369629,
      "logits/chosen": -0.731094479560852,
      "logits/rejected": -1.919350266456604,
      "logps/chosen": -1.4003535509109497,
      "logps/rejected": -2.7186193466186523,
      "loss": 1.4648,
      "nll_loss": 1.4436579942703247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14003536105155945,
      "rewards/margins": 0.13182657957077026,
      "rewards/rejected": -0.2718619406223297,
      "step": 850
    },
    {
      "epoch": 2.345184673800483,
      "grad_norm": 0.22262442111968994,
      "learning_rate": 6.985604763044382e-07,
      "log_odds_chosen": 1.577681303024292,
      "log_odds_ratio": -0.20103678107261658,
      "logits/chosen": -0.8643767237663269,
      "logits/rejected": -1.923162579536438,
      "logps/chosen": -1.3931130170822144,
      "logps/rejected": -2.7541937828063965,
      "loss": 1.4549,
      "nll_loss": 1.4347765445709229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1393112987279892,
      "rewards/margins": 0.13610810041427612,
      "rewards/rejected": -0.2754194140434265,
      "step": 851
    },
    {
      "epoch": 2.3479461511908872,
      "grad_norm": 0.22737272083759308,
      "learning_rate": 6.930128404315214e-07,
      "log_odds_chosen": 1.5616035461425781,
      "log_odds_ratio": -0.20092810690402985,
      "logits/chosen": -0.875281035900116,
      "logits/rejected": -1.9816867113113403,
      "logps/chosen": -1.4429211616516113,
      "logps/rejected": -2.799321174621582,
      "loss": 1.5083,
      "nll_loss": 1.488168478012085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14429211616516113,
      "rewards/margins": 0.1356399655342102,
      "rewards/rejected": -0.27993208169937134,
      "step": 852
    },
    {
      "epoch": 2.350707628581291,
      "grad_norm": 0.21603858470916748,
      "learning_rate": 6.874837741474744e-07,
      "log_odds_chosen": 1.561700701713562,
      "log_odds_ratio": -0.2068319022655487,
      "logits/chosen": -0.8764750957489014,
      "logits/rejected": -1.9299256801605225,
      "logps/chosen": -1.362099289894104,
      "logps/rejected": -2.70504093170166,
      "loss": 1.4298,
      "nll_loss": 1.4091013669967651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13620993494987488,
      "rewards/margins": 0.13429416716098785,
      "rewards/rejected": -0.2705041170120239,
      "step": 853
    },
    {
      "epoch": 2.353469105971695,
      "grad_norm": 0.2454364150762558,
      "learning_rate": 6.819733342720067e-07,
      "log_odds_chosen": 1.6186573505401611,
      "log_odds_ratio": -0.19199815392494202,
      "logits/chosen": -0.9424867630004883,
      "logits/rejected": -1.8045995235443115,
      "logps/chosen": -1.4089401960372925,
      "logps/rejected": -2.8112776279449463,
      "loss": 1.4579,
      "nll_loss": 1.4386602640151978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14089402556419373,
      "rewards/margins": 0.14023372530937195,
      "rewards/rejected": -0.2811277508735657,
      "step": 854
    },
    {
      "epoch": 2.3562305833620987,
      "grad_norm": 0.24490110576152802,
      "learning_rate": 6.764815774334149e-07,
      "log_odds_chosen": 1.357130527496338,
      "log_odds_ratio": -0.23701991140842438,
      "logits/chosen": -0.857049822807312,
      "logits/rejected": -2.015214443206787,
      "logps/chosen": -1.4582642316818237,
      "logps/rejected": -2.625171422958374,
      "loss": 1.5179,
      "nll_loss": 1.4941763877868652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14582642912864685,
      "rewards/margins": 0.11669071018695831,
      "rewards/rejected": -0.26251715421676636,
      "step": 855
    },
    {
      "epoch": 2.3589920607525023,
      "grad_norm": 0.22016265988349915,
      "learning_rate": 6.710085600679967e-07,
      "log_odds_chosen": 1.4929471015930176,
      "log_odds_ratio": -0.21420983970165253,
      "logits/chosen": -0.7236154675483704,
      "logits/rejected": -1.8628549575805664,
      "logps/chosen": -1.3636245727539062,
      "logps/rejected": -2.6389825344085693,
      "loss": 1.4373,
      "nll_loss": 1.4158825874328613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1363624632358551,
      "rewards/margins": 0.12753577530384064,
      "rewards/rejected": -0.26389825344085693,
      "step": 856
    },
    {
      "epoch": 2.3617535381429065,
      "grad_norm": 0.24124066531658173,
      "learning_rate": 6.65554338419474e-07,
      "log_odds_chosen": 1.5906827449798584,
      "log_odds_ratio": -0.2055184245109558,
      "logits/chosen": -0.8457351922988892,
      "logits/rejected": -1.848963737487793,
      "logps/chosen": -1.486085057258606,
      "logps/rejected": -2.886665105819702,
      "loss": 1.5418,
      "nll_loss": 1.5212182998657227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486085057258606,
      "rewards/margins": 0.1400580108165741,
      "rewards/rejected": -0.2886665165424347,
      "step": 857
    },
    {
      "epoch": 2.3645150155333106,
      "grad_norm": 0.22789664566516876,
      "learning_rate": 6.601189685384127e-07,
      "log_odds_chosen": 1.4584548473358154,
      "log_odds_ratio": -0.21383509039878845,
      "logits/chosen": -0.8199442625045776,
      "logits/rejected": -1.9394491910934448,
      "logps/chosen": -1.4257709980010986,
      "logps/rejected": -2.6794774532318115,
      "loss": 1.4848,
      "nll_loss": 1.4634119272232056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1425771266222,
      "rewards/margins": 0.12537063658237457,
      "rewards/rejected": -0.2679477632045746,
      "step": 858
    },
    {
      "epoch": 2.3672764929237142,
      "grad_norm": 0.2334202527999878,
      "learning_rate": 6.547025062816487e-07,
      "log_odds_chosen": 1.7176035642623901,
      "log_odds_ratio": -0.17412160336971283,
      "logits/chosen": -0.8760895133018494,
      "logits/rejected": -1.910693645477295,
      "logps/chosen": -1.4110983610153198,
      "logps/rejected": -2.9046947956085205,
      "loss": 1.4698,
      "nll_loss": 1.4523557424545288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14110985398292542,
      "rewards/margins": 0.14935962855815887,
      "rewards/rejected": -0.2904694974422455,
      "step": 859
    },
    {
      "epoch": 2.370037970314118,
      "grad_norm": 0.2590780556201935,
      "learning_rate": 6.493050073117115e-07,
      "log_odds_chosen": 1.4820945262908936,
      "log_odds_ratio": -0.21282009780406952,
      "logits/chosen": -0.8368885517120361,
      "logits/rejected": -1.9160068035125732,
      "logps/chosen": -1.379476547241211,
      "logps/rejected": -2.642612934112549,
      "loss": 1.4554,
      "nll_loss": 1.4341613054275513,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13794764876365662,
      "rewards/margins": 0.12631365656852722,
      "rewards/rejected": -0.26426130533218384,
      "step": 860
    },
    {
      "epoch": 2.372799447704522,
      "grad_norm": 0.2302803099155426,
      "learning_rate": 6.439265270962538e-07,
      "log_odds_chosen": 1.5066372156143188,
      "log_odds_ratio": -0.21063902974128723,
      "logits/chosen": -0.7722992300987244,
      "logits/rejected": -1.9620102643966675,
      "logps/chosen": -1.336499810218811,
      "logps/rejected": -2.615638494491577,
      "loss": 1.4171,
      "nll_loss": 1.3959881067276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13364997506141663,
      "rewards/margins": 0.12791387736797333,
      "rewards/rejected": -0.26156383752822876,
      "step": 861
    },
    {
      "epoch": 2.3755609250949257,
      "grad_norm": 0.20692874491214752,
      "learning_rate": 6.385671209074829e-07,
      "log_odds_chosen": 1.7060338258743286,
      "log_odds_ratio": -0.17033779621124268,
      "logits/chosen": -0.7261908650398254,
      "logits/rejected": -2.0713276863098145,
      "logps/chosen": -1.3343982696533203,
      "logps/rejected": -2.7912662029266357,
      "loss": 1.3985,
      "nll_loss": 1.3814215660095215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1334398239850998,
      "rewards/margins": 0.1456868201494217,
      "rewards/rejected": -0.2791266441345215,
      "step": 862
    },
    {
      "epoch": 2.37832240248533,
      "grad_norm": 0.2275240123271942,
      "learning_rate": 6.332268438215878e-07,
      "log_odds_chosen": 1.7242450714111328,
      "log_odds_ratio": -0.1701982617378235,
      "logits/chosen": -0.8238985538482666,
      "logits/rejected": -2.1816961765289307,
      "logps/chosen": -1.4347082376480103,
      "logps/rejected": -2.9419825077056885,
      "loss": 1.4911,
      "nll_loss": 1.4741240739822388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14347082376480103,
      "rewards/margins": 0.15072746574878693,
      "rewards/rejected": -0.29419827461242676,
      "step": 863
    },
    {
      "epoch": 2.3810838798757334,
      "grad_norm": 0.25345364212989807,
      "learning_rate": 6.279057507181796e-07,
      "log_odds_chosen": 1.7354642152786255,
      "log_odds_ratio": -0.16710469126701355,
      "logits/chosen": -0.8893179893493652,
      "logits/rejected": -1.9863471984863281,
      "logps/chosen": -1.4171266555786133,
      "logps/rejected": -2.9303369522094727,
      "loss": 1.4731,
      "nll_loss": 1.4564212560653687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14171266555786133,
      "rewards/margins": 0.15132102370262146,
      "rewards/rejected": -0.2930336892604828,
      "step": 864
    },
    {
      "epoch": 2.3838453572661376,
      "grad_norm": 0.2283063381910324,
      "learning_rate": 6.226038962797218e-07,
      "log_odds_chosen": 1.4934532642364502,
      "log_odds_ratio": -0.2183188498020172,
      "logits/chosen": -0.8219193816184998,
      "logits/rejected": -1.9478013515472412,
      "logps/chosen": -1.3244479894638062,
      "logps/rejected": -2.5922112464904785,
      "loss": 1.4077,
      "nll_loss": 1.3858716487884521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13244479894638062,
      "rewards/margins": 0.1267762929201126,
      "rewards/rejected": -0.2592211067676544,
      "step": 865
    },
    {
      "epoch": 2.386606834656541,
      "grad_norm": 0.2298060655593872,
      "learning_rate": 6.17321334990973e-07,
      "log_odds_chosen": 1.481666088104248,
      "log_odds_ratio": -0.20736965537071228,
      "logits/chosen": -0.748926043510437,
      "logits/rejected": -1.9716696739196777,
      "logps/chosen": -1.4211902618408203,
      "logps/rejected": -2.6946358680725098,
      "loss": 1.481,
      "nll_loss": 1.4602220058441162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14211902022361755,
      "rewards/margins": 0.12734454870224,
      "rewards/rejected": -0.26946359872817993,
      "step": 866
    },
    {
      "epoch": 2.389368312046945,
      "grad_norm": 0.21714286506175995,
      "learning_rate": 6.120581211384222e-07,
      "log_odds_chosen": 1.6732999086380005,
      "log_odds_ratio": -0.2173757553100586,
      "logits/chosen": -0.7669368386268616,
      "logits/rejected": -1.9981590509414673,
      "logps/chosen": -1.3830281496047974,
      "logps/rejected": -2.8426358699798584,
      "loss": 1.4629,
      "nll_loss": 1.4411296844482422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13830281794071198,
      "rewards/margins": 0.14596077799797058,
      "rewards/rejected": -0.28426361083984375,
      "step": 867
    },
    {
      "epoch": 2.392129789437349,
      "grad_norm": 0.23840640485286713,
      "learning_rate": 6.068143088097372e-07,
      "log_odds_chosen": 1.6300663948059082,
      "log_odds_ratio": -0.1832505315542221,
      "logits/chosen": -0.846603512763977,
      "logits/rejected": -1.958760142326355,
      "logps/chosen": -1.360883116722107,
      "logps/rejected": -2.7600831985473633,
      "loss": 1.4466,
      "nll_loss": 1.4282522201538086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1360883116722107,
      "rewards/margins": 0.13992001116275787,
      "rewards/rejected": -0.27600833773612976,
      "step": 868
    },
    {
      "epoch": 2.3948912668277527,
      "grad_norm": 0.21919068694114685,
      "learning_rate": 6.015899518932005e-07,
      "log_odds_chosen": 1.7166774272918701,
      "log_odds_ratio": -0.17428842186927795,
      "logits/chosen": -0.8554909825325012,
      "logits/rejected": -1.9679923057556152,
      "logps/chosen": -1.366170048713684,
      "logps/rejected": -2.845726490020752,
      "loss": 1.4217,
      "nll_loss": 1.404255986213684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1366170048713684,
      "rewards/margins": 0.14795567095279694,
      "rewards/rejected": -0.28457266092300415,
      "step": 869
    },
    {
      "epoch": 2.3976527442181568,
      "grad_norm": 0.21695028245449066,
      "learning_rate": 5.963851040771639e-07,
      "log_odds_chosen": 1.4922925233840942,
      "log_odds_ratio": -0.22315824031829834,
      "logits/chosen": -0.7975160479545593,
      "logits/rejected": -1.9359625577926636,
      "logps/chosen": -1.3698251247406006,
      "logps/rejected": -2.6484439373016357,
      "loss": 1.4442,
      "nll_loss": 1.4218785762786865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1369825154542923,
      "rewards/margins": 0.1278618574142456,
      "rewards/rejected": -0.2648443579673767,
      "step": 870
    },
    {
      "epoch": 2.4004142216085604,
      "grad_norm": 0.2348206788301468,
      "learning_rate": 5.911998188494924e-07,
      "log_odds_chosen": 1.6946475505828857,
      "log_odds_ratio": -0.17974430322647095,
      "logits/chosen": -0.8474396467208862,
      "logits/rejected": -2.162605047225952,
      "logps/chosen": -1.439106822013855,
      "logps/rejected": -2.918508291244507,
      "loss": 1.4884,
      "nll_loss": 1.4704686403274536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14391069114208221,
      "rewards/margins": 0.14794014394283295,
      "rewards/rejected": -0.29185086488723755,
      "step": 871
    },
    {
      "epoch": 2.4031756989989645,
      "grad_norm": 0.21678882837295532,
      "learning_rate": 5.860341494970132e-07,
      "log_odds_chosen": 1.686343789100647,
      "log_odds_ratio": -0.176815465092659,
      "logits/chosen": -0.8357268571853638,
      "logits/rejected": -2.049056053161621,
      "logps/chosen": -1.3776237964630127,
      "logps/rejected": -2.8327560424804688,
      "loss": 1.4463,
      "nll_loss": 1.4285705089569092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1377623826265335,
      "rewards/margins": 0.14551322162151337,
      "rewards/rejected": -0.28327563405036926,
      "step": 872
    },
    {
      "epoch": 2.405937176389368,
      "grad_norm": 0.23483207821846008,
      "learning_rate": 5.808881491049723e-07,
      "log_odds_chosen": 1.5488799810409546,
      "log_odds_ratio": -0.21071894466876984,
      "logits/chosen": -0.8064358234405518,
      "logits/rejected": -1.6850775480270386,
      "logps/chosen": -1.4290885925292969,
      "logps/rejected": -2.7724051475524902,
      "loss": 1.4928,
      "nll_loss": 1.4716988801956177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14290885627269745,
      "rewards/margins": 0.13433167338371277,
      "rewards/rejected": -0.277240514755249,
      "step": 873
    },
    {
      "epoch": 2.4086986537797723,
      "grad_norm": 0.21404524147510529,
      "learning_rate": 5.757618705564849e-07,
      "log_odds_chosen": 1.6298061609268188,
      "log_odds_ratio": -0.18933196365833282,
      "logits/chosen": -0.7765361070632935,
      "logits/rejected": -2.0819590091705322,
      "logps/chosen": -1.3593862056732178,
      "logps/rejected": -2.7558720111846924,
      "loss": 1.4306,
      "nll_loss": 1.4117118120193481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1359386146068573,
      "rewards/margins": 0.13964857161045074,
      "rewards/rejected": -0.27558720111846924,
      "step": 874
    },
    {
      "epoch": 2.411460131170176,
      "grad_norm": 0.2233470380306244,
      "learning_rate": 5.706553665319955e-07,
      "log_odds_chosen": 1.6726114749908447,
      "log_odds_ratio": -0.18509700894355774,
      "logits/chosen": -0.7863438129425049,
      "logits/rejected": -2.077401876449585,
      "logps/chosen": -1.4051029682159424,
      "logps/rejected": -2.8573782444000244,
      "loss": 1.4697,
      "nll_loss": 1.4511629343032837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14051030576229095,
      "rewards/margins": 0.14522749185562134,
      "rewards/rejected": -0.2857378125190735,
      "step": 875
    },
    {
      "epoch": 2.41422160856058,
      "grad_norm": 0.24152301251888275,
      "learning_rate": 5.655686895087328e-07,
      "log_odds_chosen": 1.6769300699234009,
      "log_odds_ratio": -0.18689285218715668,
      "logits/chosen": -0.8174842596054077,
      "logits/rejected": -2.004807949066162,
      "logps/chosen": -1.435114860534668,
      "logps/rejected": -2.8996763229370117,
      "loss": 1.502,
      "nll_loss": 1.4832693338394165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14351148903369904,
      "rewards/margins": 0.14645616710186005,
      "rewards/rejected": -0.2899676561355591,
      "step": 876
    },
    {
      "epoch": 2.4169830859509838,
      "grad_norm": 0.2544821798801422,
      "learning_rate": 5.605018917601754e-07,
      "log_odds_chosen": 1.5323283672332764,
      "log_odds_ratio": -0.20912876725196838,
      "logits/chosen": -0.7400486469268799,
      "logits/rejected": -1.9783880710601807,
      "logps/chosen": -1.423980712890625,
      "logps/rejected": -2.744169235229492,
      "loss": 1.4978,
      "nll_loss": 1.476898431777954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14239807426929474,
      "rewards/margins": 0.13201886415481567,
      "rewards/rejected": -0.2744169235229492,
      "step": 877
    },
    {
      "epoch": 2.4197445633413874,
      "grad_norm": 0.25507211685180664,
      "learning_rate": 5.554550253555066e-07,
      "log_odds_chosen": 1.4117803573608398,
      "log_odds_ratio": -0.22325754165649414,
      "logits/chosen": -0.8820750713348389,
      "logits/rejected": -1.6174063682556152,
      "logps/chosen": -1.352549433708191,
      "logps/rejected": -2.5473551750183105,
      "loss": 1.4452,
      "nll_loss": 1.4228417873382568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13525494933128357,
      "rewards/margins": 0.11948058754205704,
      "rewards/rejected": -0.2547355592250824,
      "step": 878
    },
    {
      "epoch": 2.4225060407317915,
      "grad_norm": 0.22250917553901672,
      "learning_rate": 5.504281421590898e-07,
      "log_odds_chosen": 1.6617779731750488,
      "log_odds_ratio": -0.1937955915927887,
      "logits/chosen": -0.8170138001441956,
      "logits/rejected": -2.026573657989502,
      "logps/chosen": -1.3464716672897339,
      "logps/rejected": -2.777165651321411,
      "loss": 1.4149,
      "nll_loss": 1.395482063293457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1346471756696701,
      "rewards/margins": 0.14306940138339996,
      "rewards/rejected": -0.27771657705307007,
      "step": 879
    },
    {
      "epoch": 2.425267518122195,
      "grad_norm": 0.23034892976284027,
      "learning_rate": 5.454212938299256e-07,
      "log_odds_chosen": 1.646827220916748,
      "log_odds_ratio": -0.1863548904657364,
      "logits/chosen": -0.9122263789176941,
      "logits/rejected": -1.9986140727996826,
      "logps/chosen": -1.412491798400879,
      "logps/rejected": -2.8388447761535645,
      "loss": 1.4726,
      "nll_loss": 1.4539694786071777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412491649389267,
      "rewards/margins": 0.14263533055782318,
      "rewards/rejected": -0.2838844954967499,
      "step": 880
    },
    {
      "epoch": 2.4280289955125993,
      "grad_norm": 0.20888766646385193,
      "learning_rate": 5.404345318211294e-07,
      "log_odds_chosen": 1.5793465375900269,
      "log_odds_ratio": -0.20006848871707916,
      "logits/chosen": -0.8890371918678284,
      "logits/rejected": -1.9650756120681763,
      "logps/chosen": -1.4828230142593384,
      "logps/rejected": -2.86384654045105,
      "loss": 1.5303,
      "nll_loss": 1.5103161334991455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14828228950500488,
      "rewards/margins": 0.13810236752033234,
      "rewards/rejected": -0.2863846719264984,
      "step": 881
    },
    {
      "epoch": 2.430790472903003,
      "grad_norm": 0.24945060908794403,
      "learning_rate": 5.354679073793942e-07,
      "log_odds_chosen": 1.5567692518234253,
      "log_odds_ratio": -0.2121957540512085,
      "logits/chosen": -0.8492003679275513,
      "logits/rejected": -1.7714898586273193,
      "logps/chosen": -1.370238184928894,
      "logps/rejected": -2.7132062911987305,
      "loss": 1.4423,
      "nll_loss": 1.4210622310638428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13702382147312164,
      "rewards/margins": 0.13429680466651917,
      "rewards/rejected": -0.271320641040802,
      "step": 882
    },
    {
      "epoch": 2.433551950293407,
      "grad_norm": 0.22785811126232147,
      "learning_rate": 5.305214715444726e-07,
      "log_odds_chosen": 1.3213268518447876,
      "log_odds_ratio": -0.2632153034210205,
      "logits/chosen": -0.7535655498504639,
      "logits/rejected": -1.8558615446090698,
      "logps/chosen": -1.468719482421875,
      "logps/rejected": -2.605940580368042,
      "loss": 1.5309,
      "nll_loss": 1.5045620203018188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14687195420265198,
      "rewards/margins": 0.11372211575508118,
      "rewards/rejected": -0.26059406995773315,
      "step": 883
    },
    {
      "epoch": 2.4363134276838108,
      "grad_norm": 0.2245754897594452,
      "learning_rate": 5.255952751486442e-07,
      "log_odds_chosen": 1.3915148973464966,
      "log_odds_ratio": -0.2355232983827591,
      "logits/chosen": -0.7501202821731567,
      "logits/rejected": -1.6952217817306519,
      "logps/chosen": -1.428135871887207,
      "logps/rejected": -2.6236112117767334,
      "loss": 1.4991,
      "nll_loss": 1.475578784942627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14281359314918518,
      "rewards/margins": 0.1195475235581398,
      "rewards/rejected": -0.2623611390590668,
      "step": 884
    },
    {
      "epoch": 2.439074905074215,
      "grad_norm": 0.2622843384742737,
      "learning_rate": 5.20689368816201e-07,
      "log_odds_chosen": 1.627671241760254,
      "log_odds_ratio": -0.18791747093200684,
      "logits/chosen": -0.7995626926422119,
      "logits/rejected": -2.078826665878296,
      "logps/chosen": -1.5019242763519287,
      "logps/rejected": -2.931610584259033,
      "loss": 1.5565,
      "nll_loss": 1.537746548652649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15019242465496063,
      "rewards/margins": 0.14296863973140717,
      "rewards/rejected": -0.2931610643863678,
      "step": 885
    },
    {
      "epoch": 2.4418363824646185,
      "grad_norm": 0.22714628279209137,
      "learning_rate": 5.158038029629195e-07,
      "log_odds_chosen": 1.4414920806884766,
      "log_odds_ratio": -0.21628612279891968,
      "logits/chosen": -0.8436942100524902,
      "logits/rejected": -1.9419432878494263,
      "logps/chosen": -1.461559772491455,
      "logps/rejected": -2.708940267562866,
      "loss": 1.5133,
      "nll_loss": 1.4916455745697021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14615598320960999,
      "rewards/margins": 0.12473806738853455,
      "rewards/rejected": -0.27089405059814453,
      "step": 886
    },
    {
      "epoch": 2.4445978598550226,
      "grad_norm": 0.23047834634780884,
      "learning_rate": 5.109386277955477e-07,
      "log_odds_chosen": 1.4839013814926147,
      "log_odds_ratio": -0.21651501953601837,
      "logits/chosen": -0.7708752155303955,
      "logits/rejected": -1.970124363899231,
      "logps/chosen": -1.391357421875,
      "logps/rejected": -2.6658291816711426,
      "loss": 1.4515,
      "nll_loss": 1.4298431873321533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13913573324680328,
      "rewards/margins": 0.12744717299938202,
      "rewards/rejected": -0.2665829360485077,
      "step": 887
    },
    {
      "epoch": 2.4473593372454263,
      "grad_norm": 0.22831033170223236,
      "learning_rate": 5.060938933112891e-07,
      "log_odds_chosen": 1.4900858402252197,
      "log_odds_ratio": -0.22767001390457153,
      "logits/chosen": -0.9148192405700684,
      "logits/rejected": -1.889332890510559,
      "logps/chosen": -1.411495566368103,
      "logps/rejected": -2.7004637718200684,
      "loss": 1.4818,
      "nll_loss": 1.4590768814086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14114956557750702,
      "rewards/margins": 0.1288968026638031,
      "rewards/rejected": -0.2700463831424713,
      "step": 888
    },
    {
      "epoch": 2.45012081463583,
      "grad_norm": 0.22466985881328583,
      "learning_rate": 5.012696492972852e-07,
      "log_odds_chosen": 1.5186207294464111,
      "log_odds_ratio": -0.21536147594451904,
      "logits/chosen": -0.8782755136489868,
      "logits/rejected": -1.8780262470245361,
      "logps/chosen": -1.3794608116149902,
      "logps/rejected": -2.681459903717041,
      "loss": 1.4452,
      "nll_loss": 1.4237024784088135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13794609904289246,
      "rewards/margins": 0.13019990921020508,
      "rewards/rejected": -0.26814597845077515,
      "step": 889
    },
    {
      "epoch": 2.452882292026234,
      "grad_norm": 0.19952109456062317,
      "learning_rate": 4.964659453301088e-07,
      "log_odds_chosen": 1.964066505432129,
      "log_odds_ratio": -0.1502349078655243,
      "logits/chosen": -0.7120921611785889,
      "logits/rejected": -2.4251503944396973,
      "logps/chosen": -1.3529983758926392,
      "logps/rejected": -3.066349506378174,
      "loss": 1.4145,
      "nll_loss": 1.3994845151901245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13529984652996063,
      "rewards/margins": 0.17133508622646332,
      "rewards/rejected": -0.30663493275642395,
      "step": 890
    },
    {
      "epoch": 2.4556437694166378,
      "grad_norm": 0.21941575407981873,
      "learning_rate": 4.916828307752489e-07,
      "log_odds_chosen": 1.5352859497070312,
      "log_odds_ratio": -0.20665743947029114,
      "logits/chosen": -0.854226291179657,
      "logits/rejected": -1.9289751052856445,
      "logps/chosen": -1.417055606842041,
      "logps/rejected": -2.7416605949401855,
      "loss": 1.474,
      "nll_loss": 1.453291893005371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14170557260513306,
      "rewards/margins": 0.13246050477027893,
      "rewards/rejected": -0.274166077375412,
      "step": 891
    },
    {
      "epoch": 2.458405246807042,
      "grad_norm": 0.22385253012180328,
      "learning_rate": 4.869203547866097e-07,
      "log_odds_chosen": 1.5946441888809204,
      "log_odds_ratio": -0.20085304975509644,
      "logits/chosen": -0.7629926800727844,
      "logits/rejected": -2.0393998622894287,
      "logps/chosen": -1.4476027488708496,
      "logps/rejected": -2.840999126434326,
      "loss": 1.4987,
      "nll_loss": 1.4786458015441895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14476028084754944,
      "rewards/margins": 0.13933967053890228,
      "rewards/rejected": -0.2840999364852905,
      "step": 892
    },
    {
      "epoch": 2.4611667241974455,
      "grad_norm": 0.23724229633808136,
      "learning_rate": 4.821785663059991e-07,
      "log_odds_chosen": 1.5859582424163818,
      "log_odds_ratio": -0.19816404581069946,
      "logits/chosen": -0.7932180762290955,
      "logits/rejected": -1.9734810590744019,
      "logps/chosen": -1.4295369386672974,
      "logps/rejected": -2.806306838989258,
      "loss": 1.4843,
      "nll_loss": 1.4644794464111328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14295370876789093,
      "rewards/margins": 0.13767698407173157,
      "rewards/rejected": -0.2806306779384613,
      "step": 893
    },
    {
      "epoch": 2.4639282015878496,
      "grad_norm": 0.24634841084480286,
      "learning_rate": 4.774575140626317e-07,
      "log_odds_chosen": 1.6122064590454102,
      "log_odds_ratio": -0.2016284167766571,
      "logits/chosen": -0.79060959815979,
      "logits/rejected": -1.965233564376831,
      "logps/chosen": -1.497755527496338,
      "logps/rejected": -2.916637420654297,
      "loss": 1.5529,
      "nll_loss": 1.532710313796997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14977557957172394,
      "rewards/margins": 0.14188817143440247,
      "rewards/rejected": -0.2916637361049652,
      "step": 894
    },
    {
      "epoch": 2.4666896789782533,
      "grad_norm": 0.22494585812091827,
      "learning_rate": 4.7275724657262293e-07,
      "log_odds_chosen": 1.7693073749542236,
      "log_odds_ratio": -0.17247578501701355,
      "logits/chosen": -0.7484267354011536,
      "logits/rejected": -2.083472728729248,
      "logps/chosen": -1.3836661577224731,
      "logps/rejected": -2.9199507236480713,
      "loss": 1.4367,
      "nll_loss": 1.4195003509521484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13836662471294403,
      "rewards/margins": 0.15362845361232758,
      "rewards/rejected": -0.2919950485229492,
      "step": 895
    },
    {
      "epoch": 2.4694511563686574,
      "grad_norm": 0.22843420505523682,
      "learning_rate": 4.6807781213849354e-07,
      "log_odds_chosen": 1.6209605932235718,
      "log_odds_ratio": -0.20323897898197174,
      "logits/chosen": -0.8005288243293762,
      "logits/rejected": -1.9480664730072021,
      "logps/chosen": -1.4012664556503296,
      "logps/rejected": -2.8044662475585938,
      "loss": 1.4799,
      "nll_loss": 1.459563136100769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14012664556503296,
      "rewards/margins": 0.14031997323036194,
      "rewards/rejected": -0.2804466485977173,
      "step": 896
    },
    {
      "epoch": 2.472212633759061,
      "grad_norm": 0.2613917291164398,
      "learning_rate": 4.63419258848673e-07,
      "log_odds_chosen": 1.5579595565795898,
      "log_odds_ratio": -0.21004337072372437,
      "logits/chosen": -0.8195448517799377,
      "logits/rejected": -1.721724033355713,
      "logps/chosen": -1.4049590826034546,
      "logps/rejected": -2.7522690296173096,
      "loss": 1.474,
      "nll_loss": 1.4529823064804077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1404959112405777,
      "rewards/margins": 0.1347309947013855,
      "rewards/rejected": -0.275226891040802,
      "step": 897
    },
    {
      "epoch": 2.474974111149465,
      "grad_norm": 0.23283885419368744,
      "learning_rate": 4.5878163457700327e-07,
      "log_odds_chosen": 1.6135423183441162,
      "log_odds_ratio": -0.20367959141731262,
      "logits/chosen": -0.7768855094909668,
      "logits/rejected": -2.0446574687957764,
      "logps/chosen": -1.45505952835083,
      "logps/rejected": -2.865198850631714,
      "loss": 1.4948,
      "nll_loss": 1.4743952751159668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14550594985485077,
      "rewards/margins": 0.14101392030715942,
      "rewards/rejected": -0.2865198850631714,
      "step": 898
    },
    {
      "epoch": 2.477735588539869,
      "grad_norm": 0.2130129635334015,
      "learning_rate": 4.5416498698225034e-07,
      "log_odds_chosen": 1.580468773841858,
      "log_odds_ratio": -0.20243686437606812,
      "logits/chosen": -0.7336734533309937,
      "logits/rejected": -2.138326644897461,
      "logps/chosen": -1.4078233242034912,
      "logps/rejected": -2.7767281532287598,
      "loss": 1.4735,
      "nll_loss": 1.4533002376556396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14078234136104584,
      "rewards/margins": 0.1368904709815979,
      "rewards/rejected": -0.27767282724380493,
      "step": 899
    },
    {
      "epoch": 2.4804970659302725,
      "grad_norm": 0.2216830551624298,
      "learning_rate": 4.495693635076101e-07,
      "log_odds_chosen": 1.535346508026123,
      "log_odds_ratio": -0.20014424622058868,
      "logits/chosen": -0.8925894498825073,
      "logits/rejected": -2.127046585083008,
      "logps/chosen": -1.376361608505249,
      "logps/rejected": -2.692883014678955,
      "loss": 1.4484,
      "nll_loss": 1.4284191131591797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13763616979122162,
      "rewards/margins": 0.13165216147899628,
      "rewards/rejected": -0.2692883312702179,
      "step": 900
    },
    {
      "epoch": 2.4832585433206766,
      "grad_norm": 0.21299172937870026,
      "learning_rate": 4.4499481138022546e-07,
      "log_odds_chosen": 1.933703064918518,
      "log_odds_ratio": -0.14841099083423615,
      "logits/chosen": -0.8260481953620911,
      "logits/rejected": -2.3376615047454834,
      "logps/chosen": -1.372591495513916,
      "logps/rejected": -3.0575618743896484,
      "loss": 1.4185,
      "nll_loss": 1.4036223888397217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13725915551185608,
      "rewards/margins": 0.16849704086780548,
      "rewards/rejected": -0.30575618147850037,
      "step": 901
    },
    {
      "epoch": 2.4860200207110803,
      "grad_norm": 0.23825101554393768,
      "learning_rate": 4.4044137761069584e-07,
      "log_odds_chosen": 1.4366477727890015,
      "log_odds_ratio": -0.22512398660182953,
      "logits/chosen": -0.7757810950279236,
      "logits/rejected": -1.893244981765747,
      "logps/chosen": -1.4974112510681152,
      "logps/rejected": -2.7518978118896484,
      "loss": 1.564,
      "nll_loss": 1.5414974689483643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14974112808704376,
      "rewards/margins": 0.12544865906238556,
      "rewards/rejected": -0.2751897871494293,
      "step": 902
    },
    {
      "epoch": 2.4887814981014844,
      "grad_norm": 0.23197093605995178,
      "learning_rate": 4.359091089925999e-07,
      "log_odds_chosen": 1.7243913412094116,
      "log_odds_ratio": -0.1668962985277176,
      "logits/chosen": -0.7759650349617004,
      "logits/rejected": -2.1746482849121094,
      "logps/chosen": -1.4509389400482178,
      "logps/rejected": -2.961793899536133,
      "loss": 1.4962,
      "nll_loss": 1.479463815689087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1450938880443573,
      "rewards/margins": 0.1510854959487915,
      "rewards/rejected": -0.2961794137954712,
      "step": 903
    },
    {
      "epoch": 2.491542975491888,
      "grad_norm": 0.23694312572479248,
      "learning_rate": 4.3139805210200917e-07,
      "log_odds_chosen": 1.805801510810852,
      "log_odds_ratio": -0.1572706699371338,
      "logits/chosen": -0.8221292495727539,
      "logits/rejected": -2.0871081352233887,
      "logps/chosen": -1.4446747303009033,
      "logps/rejected": -3.031667470932007,
      "loss": 1.5126,
      "nll_loss": 1.496914029121399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14446748793125153,
      "rewards/margins": 0.15869927406311035,
      "rewards/rejected": -0.3031667470932007,
      "step": 904
    },
    {
      "epoch": 2.494304452882292,
      "grad_norm": 0.22664044797420502,
      "learning_rate": 4.2690825329701313e-07,
      "log_odds_chosen": 1.5435681343078613,
      "log_odds_ratio": -0.1995273232460022,
      "logits/chosen": -0.7720770835876465,
      "logits/rejected": -2.0005624294281006,
      "logps/chosen": -1.3651032447814941,
      "logps/rejected": -2.6808128356933594,
      "loss": 1.4317,
      "nll_loss": 1.4117234945297241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13651034235954285,
      "rewards/margins": 0.1315709352493286,
      "rewards/rejected": -0.26808127760887146,
      "step": 905
    },
    {
      "epoch": 2.497065930272696,
      "grad_norm": 0.22075149416923523,
      "learning_rate": 4.2243975871724017e-07,
      "log_odds_chosen": 1.3728384971618652,
      "log_odds_ratio": -0.2398672103881836,
      "logits/chosen": -0.8564939498901367,
      "logits/rejected": -1.6772489547729492,
      "logps/chosen": -1.420592188835144,
      "logps/rejected": -2.5997753143310547,
      "loss": 1.499,
      "nll_loss": 1.474989652633667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14205922186374664,
      "rewards/margins": 0.11791832000017166,
      "rewards/rejected": -0.2599775493144989,
      "step": 906
    },
    {
      "epoch": 2.4998274076630995,
      "grad_norm": 0.2160337120294571,
      "learning_rate": 4.179926142833876e-07,
      "log_odds_chosen": 1.5068174600601196,
      "log_odds_ratio": -0.20641684532165527,
      "logits/chosen": -0.7754029631614685,
      "logits/rejected": -2.1742076873779297,
      "logps/chosen": -1.46108078956604,
      "logps/rejected": -2.767190456390381,
      "loss": 1.5144,
      "nll_loss": 1.4937331676483154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14610810577869415,
      "rewards/margins": 0.13061094284057617,
      "rewards/rejected": -0.27671903371810913,
      "step": 907
    },
    {
      "epoch": 2.5025888850535036,
      "grad_norm": 0.23876003921031952,
      "learning_rate": 4.1356686569674344e-07,
      "log_odds_chosen": 1.5174545049667358,
      "log_odds_ratio": -0.21640154719352722,
      "logits/chosen": -0.8613941669464111,
      "logits/rejected": -1.8654732704162598,
      "logps/chosen": -1.4302722215652466,
      "logps/rejected": -2.7472269535064697,
      "loss": 1.4941,
      "nll_loss": 1.4724962711334229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14302721619606018,
      "rewards/margins": 0.1316954791545868,
      "rewards/rejected": -0.27472272515296936,
      "step": 908
    },
    {
      "epoch": 2.5053503624439077,
      "grad_norm": 0.2481316179037094,
      "learning_rate": 4.091625584387238e-07,
      "log_odds_chosen": 1.6250629425048828,
      "log_odds_ratio": -0.18948356807231903,
      "logits/chosen": -0.830609917640686,
      "logits/rejected": -2.119431495666504,
      "logps/chosen": -1.4281871318817139,
      "logps/rejected": -2.837716817855835,
      "loss": 1.4933,
      "nll_loss": 1.4743633270263672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14281873404979706,
      "rewards/margins": 0.1409529745578766,
      "rewards/rejected": -0.28377169370651245,
      "step": 909
    },
    {
      "epoch": 2.5081118398343114,
      "grad_norm": 0.22198787331581116,
      "learning_rate": 4.0477973777039854e-07,
      "log_odds_chosen": 1.4991892576217651,
      "log_odds_ratio": -0.21167874336242676,
      "logits/chosen": -0.884207010269165,
      "logits/rejected": -1.8678700923919678,
      "logps/chosen": -1.407118797302246,
      "logps/rejected": -2.6911559104919434,
      "loss": 1.4627,
      "nll_loss": 1.4415607452392578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14071188867092133,
      "rewards/margins": 0.1284037083387375,
      "rewards/rejected": -0.2691155970096588,
      "step": 910
    },
    {
      "epoch": 2.510873317224715,
      "grad_norm": 0.2337605506181717,
      "learning_rate": 4.004184487320317e-07,
      "log_odds_chosen": 1.7035045623779297,
      "log_odds_ratio": -0.1746985763311386,
      "logits/chosen": -0.7929513454437256,
      "logits/rejected": -2.064314842224121,
      "logps/chosen": -1.472114086151123,
      "logps/rejected": -2.966843843460083,
      "loss": 1.5242,
      "nll_loss": 1.5067758560180664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14721143245697021,
      "rewards/margins": 0.14947296679019928,
      "rewards/rejected": -0.2966843843460083,
      "step": 911
    },
    {
      "epoch": 2.513634794615119,
      "grad_norm": 0.22415797412395477,
      "learning_rate": 3.9607873614261717e-07,
      "log_odds_chosen": 1.6890547275543213,
      "log_odds_ratio": -0.17474831640720367,
      "logits/chosen": -0.7648621201515198,
      "logits/rejected": -2.0227315425872803,
      "logps/chosen": -1.430749773979187,
      "logps/rejected": -2.9043784141540527,
      "loss": 1.4881,
      "nll_loss": 1.4706588983535767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14307497441768646,
      "rewards/margins": 0.1473628729581833,
      "rewards/rejected": -0.29043784737586975,
      "step": 912
    },
    {
      "epoch": 2.516396272005523,
      "grad_norm": 0.24374602735042572,
      "learning_rate": 3.917606445994146e-07,
      "log_odds_chosen": 1.6098326444625854,
      "log_odds_ratio": -0.193284809589386,
      "logits/chosen": -0.8505316972732544,
      "logits/rejected": -1.93906569480896,
      "logps/chosen": -1.4172815084457397,
      "logps/rejected": -2.814074754714966,
      "loss": 1.4815,
      "nll_loss": 1.4621838331222534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14172816276550293,
      "rewards/margins": 0.13967935740947723,
      "rewards/rejected": -0.28140750527381897,
      "step": 913
    },
    {
      "epoch": 2.519157749395927,
      "grad_norm": 0.21846385300159454,
      "learning_rate": 3.874642184774977e-07,
      "log_odds_chosen": 1.5628008842468262,
      "log_odds_ratio": -0.1962602436542511,
      "logits/chosen": -0.8740373253822327,
      "logits/rejected": -2.151853084564209,
      "logps/chosen": -1.3966370820999146,
      "logps/rejected": -2.7418158054351807,
      "loss": 1.458,
      "nll_loss": 1.4383662939071655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1396637111902237,
      "rewards/margins": 0.13451789319515228,
      "rewards/rejected": -0.274181604385376,
      "step": 914
    },
    {
      "epoch": 2.5219192267863306,
      "grad_norm": 0.24229982495307922,
      "learning_rate": 3.831895019292897e-07,
      "log_odds_chosen": 1.8062140941619873,
      "log_odds_ratio": -0.18693974614143372,
      "logits/chosen": -0.8387290239334106,
      "logits/rejected": -1.937464714050293,
      "logps/chosen": -1.4534918069839478,
      "logps/rejected": -3.0489726066589355,
      "loss": 1.5246,
      "nll_loss": 1.5059365034103394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1453491747379303,
      "rewards/margins": 0.1595480740070343,
      "rewards/rejected": -0.304897278547287,
      "step": 915
    },
    {
      "epoch": 2.5246807041767347,
      "grad_norm": 0.23088249564170837,
      "learning_rate": 3.789365388841193e-07,
      "log_odds_chosen": 1.73601233959198,
      "log_odds_ratio": -0.1753402054309845,
      "logits/chosen": -0.8377318382263184,
      "logits/rejected": -2.0821731090545654,
      "logps/chosen": -1.432181477546692,
      "logps/rejected": -2.9494948387145996,
      "loss": 1.5013,
      "nll_loss": 1.4837443828582764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14321815967559814,
      "rewards/margins": 0.15173134207725525,
      "rewards/rejected": -0.2949495315551758,
      "step": 916
    },
    {
      "epoch": 2.5274421815671384,
      "grad_norm": 0.24658960103988647,
      "learning_rate": 3.7470537304776037e-07,
      "log_odds_chosen": 1.4152191877365112,
      "log_odds_ratio": -0.22389471530914307,
      "logits/chosen": -0.7509374618530273,
      "logits/rejected": -1.9573094844818115,
      "logps/chosen": -1.4993343353271484,
      "logps/rejected": -2.7293834686279297,
      "loss": 1.5611,
      "nll_loss": 1.5387518405914307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14993345737457275,
      "rewards/margins": 0.12300491333007812,
      "rewards/rejected": -0.2729383409023285,
      "step": 917
    },
    {
      "epoch": 2.530203658957542,
      "grad_norm": 0.22455866634845734,
      "learning_rate": 3.7049604790198976e-07,
      "log_odds_chosen": 1.5770710706710815,
      "log_odds_ratio": -0.19922304153442383,
      "logits/chosen": -0.8366534113883972,
      "logits/rejected": -2.1329345703125,
      "logps/chosen": -1.4017503261566162,
      "logps/rejected": -2.7608864307403564,
      "loss": 1.4762,
      "nll_loss": 1.4563055038452148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14017504453659058,
      "rewards/margins": 0.13591359555721283,
      "rewards/rejected": -0.2760886251926422,
      "step": 918
    },
    {
      "epoch": 2.532965136347946,
      "grad_norm": 0.22560246288776398,
      "learning_rate": 3.663086067041352e-07,
      "log_odds_chosen": 1.6462829113006592,
      "log_odds_ratio": -0.18665999174118042,
      "logits/chosen": -0.7782268524169922,
      "logits/rejected": -1.9328405857086182,
      "logps/chosen": -1.4376909732818604,
      "logps/rejected": -2.872429609298706,
      "loss": 1.4934,
      "nll_loss": 1.4747681617736816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14376911520957947,
      "rewards/margins": 0.14347384870052338,
      "rewards/rejected": -0.28724294900894165,
      "step": 919
    },
    {
      "epoch": 2.5357266137383503,
      "grad_norm": 0.23410910367965698,
      "learning_rate": 3.621430924866348e-07,
      "log_odds_chosen": 1.5727438926696777,
      "log_odds_ratio": -0.19569893181324005,
      "logits/chosen": -0.8823702931404114,
      "logits/rejected": -1.9492605924606323,
      "logps/chosen": -1.3187779188156128,
      "logps/rejected": -2.647488594055176,
      "loss": 1.4058,
      "nll_loss": 1.3862695693969727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13187779486179352,
      "rewards/margins": 0.13287106156349182,
      "rewards/rejected": -0.26474887132644653,
      "step": 920
    },
    {
      "epoch": 2.538488091128754,
      "grad_norm": 0.22586840391159058,
      "learning_rate": 3.579995480565909e-07,
      "log_odds_chosen": 1.5409903526306152,
      "log_odds_ratio": -0.2025071531534195,
      "logits/chosen": -0.6942576766014099,
      "logits/rejected": -1.9230279922485352,
      "logps/chosen": -1.4208831787109375,
      "logps/rejected": -2.7438626289367676,
      "loss": 1.4877,
      "nll_loss": 1.4674333333969116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14208830893039703,
      "rewards/margins": 0.13229797780513763,
      "rewards/rejected": -0.27438628673553467,
      "step": 921
    },
    {
      "epoch": 2.5412495685191576,
      "grad_norm": 0.21825292706489563,
      "learning_rate": 3.538780159953348e-07,
      "log_odds_chosen": 1.6649723052978516,
      "log_odds_ratio": -0.198430597782135,
      "logits/chosen": -0.7998772263526917,
      "logits/rejected": -1.8460190296173096,
      "logps/chosen": -1.3014402389526367,
      "logps/rejected": -2.702242851257324,
      "loss": 1.3639,
      "nll_loss": 1.344069480895996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13014401495456696,
      "rewards/margins": 0.1400802731513977,
      "rewards/rejected": -0.27022430300712585,
      "step": 922
    },
    {
      "epoch": 2.5440110459095617,
      "grad_norm": 0.2391572743654251,
      "learning_rate": 3.4977853865798466e-07,
      "log_odds_chosen": 1.5206778049468994,
      "log_odds_ratio": -0.21618333458900452,
      "logits/chosen": -0.8164076209068298,
      "logits/rejected": -1.6934431791305542,
      "logps/chosen": -1.3476591110229492,
      "logps/rejected": -2.639338970184326,
      "loss": 1.4295,
      "nll_loss": 1.4078460931777954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13476590812206268,
      "rewards/margins": 0.12916800379753113,
      "rewards/rejected": -0.2639338970184326,
      "step": 923
    },
    {
      "epoch": 2.5467725232999654,
      "grad_norm": 0.258484423160553,
      "learning_rate": 3.457011581730124e-07,
      "log_odds_chosen": 1.6479763984680176,
      "log_odds_ratio": -0.20399057865142822,
      "logits/chosen": -0.8047228455543518,
      "logits/rejected": -1.9174230098724365,
      "logps/chosen": -1.397106647491455,
      "logps/rejected": -2.8355350494384766,
      "loss": 1.4614,
      "nll_loss": 1.4410136938095093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1397106796503067,
      "rewards/margins": 0.14384284615516663,
      "rewards/rejected": -0.28355351090431213,
      "step": 924
    },
    {
      "epoch": 2.5495340006903695,
      "grad_norm": 0.23048213124275208,
      "learning_rate": 3.4164591644181233e-07,
      "log_odds_chosen": 1.558042287826538,
      "log_odds_ratio": -0.19656692445278168,
      "logits/chosen": -0.803486704826355,
      "logits/rejected": -1.7794066667556763,
      "logps/chosen": -1.3663177490234375,
      "logps/rejected": -2.700619697570801,
      "loss": 1.4391,
      "nll_loss": 1.4194140434265137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1366317868232727,
      "rewards/margins": 0.13343019783496857,
      "rewards/rejected": -0.27006199955940247,
      "step": 925
    },
    {
      "epoch": 2.552295478080773,
      "grad_norm": 0.23470544815063477,
      "learning_rate": 3.3761285513826627e-07,
      "log_odds_chosen": 1.6755598783493042,
      "log_odds_ratio": -0.1804642528295517,
      "logits/chosen": -0.7886132001876831,
      "logits/rejected": -1.880044937133789,
      "logps/chosen": -1.3564350605010986,
      "logps/rejected": -2.797255516052246,
      "loss": 1.4131,
      "nll_loss": 1.3950680494308472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13564349710941315,
      "rewards/margins": 0.14408203959465027,
      "rewards/rejected": -0.2797255516052246,
      "step": 926
    },
    {
      "epoch": 2.5550569554711773,
      "grad_norm": 0.2104436457157135,
      "learning_rate": 3.3360201570832e-07,
      "log_odds_chosen": 1.78098464012146,
      "log_odds_ratio": -0.18341396749019623,
      "logits/chosen": -0.8001008629798889,
      "logits/rejected": -1.9677042961120605,
      "logps/chosen": -1.3394931554794312,
      "logps/rejected": -2.8801090717315674,
      "loss": 1.402,
      "nll_loss": 1.383684754371643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13394932448863983,
      "rewards/margins": 0.15406160056591034,
      "rewards/rejected": -0.28801092505455017,
      "step": 927
    },
    {
      "epoch": 2.557818432861581,
      "grad_norm": 0.2330377697944641,
      "learning_rate": 3.2961343936955384e-07,
      "log_odds_chosen": 1.5845892429351807,
      "log_odds_ratio": -0.19865839183330536,
      "logits/chosen": -0.7797976732254028,
      "logits/rejected": -1.9081692695617676,
      "logps/chosen": -1.4326475858688354,
      "logps/rejected": -2.804527759552002,
      "loss": 1.5074,
      "nll_loss": 1.487499713897705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1432647556066513,
      "rewards/margins": 0.13718801736831665,
      "rewards/rejected": -0.28045278787612915,
      "step": 928
    },
    {
      "epoch": 2.5605799102519846,
      "grad_norm": 0.23377783596515656,
      "learning_rate": 3.256471671107617e-07,
      "log_odds_chosen": 1.3970494270324707,
      "log_odds_ratio": -0.23868386447429657,
      "logits/chosen": -0.775575578212738,
      "logits/rejected": -1.8493518829345703,
      "logps/chosen": -1.3924803733825684,
      "logps/rejected": -2.5844945907592773,
      "loss": 1.4607,
      "nll_loss": 1.436859369277954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1392480432987213,
      "rewards/margins": 0.1192014068365097,
      "rewards/rejected": -0.2584494650363922,
      "step": 929
    },
    {
      "epoch": 2.5633413876423887,
      "grad_norm": 0.23156331479549408,
      "learning_rate": 3.217032396915265e-07,
      "log_odds_chosen": 1.3594629764556885,
      "log_odds_ratio": -0.2378542125225067,
      "logits/chosen": -0.7673474550247192,
      "logits/rejected": -1.6643857955932617,
      "logps/chosen": -1.3724521398544312,
      "logps/rejected": -2.525003433227539,
      "loss": 1.4354,
      "nll_loss": 1.4115841388702393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13724520802497864,
      "rewards/margins": 0.11525516957044601,
      "rewards/rejected": -0.25250038504600525,
      "step": 930
    },
    {
      "epoch": 2.5661028650327924,
      "grad_norm": 0.20743201673030853,
      "learning_rate": 3.1778169764180575e-07,
      "log_odds_chosen": 1.3011500835418701,
      "log_odds_ratio": -0.2518286108970642,
      "logits/chosen": -0.762809157371521,
      "logits/rejected": -1.9848495721817017,
      "logps/chosen": -1.408487319946289,
      "logps/rejected": -2.5149049758911133,
      "loss": 1.469,
      "nll_loss": 1.4438223838806152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14084872603416443,
      "rewards/margins": 0.11064177751541138,
      "rewards/rejected": -0.2514905035495758,
      "step": 931
    },
    {
      "epoch": 2.5688643424231965,
      "grad_norm": 0.23613394796848297,
      "learning_rate": 3.1388258126151093e-07,
      "log_odds_chosen": 1.639764666557312,
      "log_odds_ratio": -0.18439045548439026,
      "logits/chosen": -0.8464503288269043,
      "logits/rejected": -2.162656784057617,
      "logps/chosen": -1.3933584690093994,
      "logps/rejected": -2.8041787147521973,
      "loss": 1.4705,
      "nll_loss": 1.4521013498306274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13933585584163666,
      "rewards/margins": 0.14108198881149292,
      "rewards/rejected": -0.28041785955429077,
      "step": 932
    },
    {
      "epoch": 2.5716258198136,
      "grad_norm": 0.22031117975711823,
      "learning_rate": 3.100059306200959e-07,
      "log_odds_chosen": 1.5673953294754028,
      "log_odds_ratio": -0.19293257594108582,
      "logits/chosen": -0.8196423649787903,
      "logits/rejected": -2.1118574142456055,
      "logps/chosen": -1.4162676334381104,
      "logps/rejected": -2.7700836658477783,
      "loss": 1.4824,
      "nll_loss": 1.4630780220031738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1416267603635788,
      "rewards/margins": 0.1353815793991089,
      "rewards/rejected": -0.2770083546638489,
      "step": 933
    },
    {
      "epoch": 2.5743872972040043,
      "grad_norm": 0.22910936176776886,
      "learning_rate": 3.06151785556143e-07,
      "log_odds_chosen": 1.5494434833526611,
      "log_odds_ratio": -0.20234021544456482,
      "logits/chosen": -0.8608691692352295,
      "logits/rejected": -2.1640162467956543,
      "logps/chosen": -1.4172899723052979,
      "logps/rejected": -2.7569541931152344,
      "loss": 1.468,
      "nll_loss": 1.447774887084961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1417289823293686,
      "rewards/margins": 0.13396643102169037,
      "rewards/rejected": -0.27569544315338135,
      "step": 934
    },
    {
      "epoch": 2.577148774594408,
      "grad_norm": 0.22023598849773407,
      "learning_rate": 3.02320185676957e-07,
      "log_odds_chosen": 1.4850515127182007,
      "log_odds_ratio": -0.21363158524036407,
      "logits/chosen": -0.745186984539032,
      "logits/rejected": -1.937159538269043,
      "logps/chosen": -1.4178829193115234,
      "logps/rejected": -2.6953628063201904,
      "loss": 1.4788,
      "nll_loss": 1.457419514656067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1417883038520813,
      "rewards/margins": 0.12774798274040222,
      "rewards/rejected": -0.2695362865924835,
      "step": 935
    },
    {
      "epoch": 2.5799102519848116,
      "grad_norm": 0.24953798949718475,
      "learning_rate": 2.98511170358155e-07,
      "log_odds_chosen": 1.6144005060195923,
      "log_odds_ratio": -0.19470098614692688,
      "logits/chosen": -0.884423553943634,
      "logits/rejected": -2.062915325164795,
      "logps/chosen": -1.519521713256836,
      "logps/rejected": -2.9385733604431152,
      "loss": 1.5727,
      "nll_loss": 1.5532505512237549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15195219218730927,
      "rewards/margins": 0.14190514385700226,
      "rewards/rejected": -0.29385730624198914,
      "step": 936
    },
    {
      "epoch": 2.5826717293752157,
      "grad_norm": 0.223532572388649,
      "learning_rate": 2.947247787432625e-07,
      "log_odds_chosen": 1.4764959812164307,
      "log_odds_ratio": -0.21015074849128723,
      "logits/chosen": -0.6920619606971741,
      "logits/rejected": -1.8913313150405884,
      "logps/chosen": -1.4597268104553223,
      "logps/rejected": -2.7401700019836426,
      "loss": 1.5324,
      "nll_loss": 1.5114260911941528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14597268402576447,
      "rewards/margins": 0.12804432213306427,
      "rewards/rejected": -0.27401700615882874,
      "step": 937
    },
    {
      "epoch": 2.58543320676562,
      "grad_norm": 0.24072711169719696,
      "learning_rate": 2.9096104974331186e-07,
      "log_odds_chosen": 1.680213212966919,
      "log_odds_ratio": -0.1735863834619522,
      "logits/chosen": -0.8284453749656677,
      "logits/rejected": -1.9820377826690674,
      "logps/chosen": -1.5137585401535034,
      "logps/rejected": -2.988992214202881,
      "loss": 1.5605,
      "nll_loss": 1.5431687831878662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.151375874876976,
      "rewards/margins": 0.14752335846424103,
      "rewards/rejected": -0.29889923334121704,
      "step": 938
    },
    {
      "epoch": 2.5881946841560235,
      "grad_norm": 0.2170211672782898,
      "learning_rate": 2.872200220364413e-07,
      "log_odds_chosen": 1.6567010879516602,
      "log_odds_ratio": -0.17847812175750732,
      "logits/chosen": -0.7924590110778809,
      "logits/rejected": -2.16463041305542,
      "logps/chosen": -1.3362153768539429,
      "logps/rejected": -2.7536497116088867,
      "loss": 1.3989,
      "nll_loss": 1.3810375928878784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13362154364585876,
      "rewards/margins": 0.14174343645572662,
      "rewards/rejected": -0.2753649652004242,
      "step": 939
    },
    {
      "epoch": 2.590956161546427,
      "grad_norm": 0.23444724082946777,
      "learning_rate": 2.8350173406749975e-07,
      "log_odds_chosen": 1.5718295574188232,
      "log_odds_ratio": -0.19533279538154602,
      "logits/chosen": -0.8168334364891052,
      "logits/rejected": -1.8591034412384033,
      "logps/chosen": -1.378641963005066,
      "logps/rejected": -2.7296016216278076,
      "loss": 1.4542,
      "nll_loss": 1.4346563816070557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13786420226097107,
      "rewards/margins": 0.13509595394134521,
      "rewards/rejected": -0.2729601562023163,
      "step": 940
    },
    {
      "epoch": 2.5937176389368313,
      "grad_norm": 0.2225639373064041,
      "learning_rate": 2.7980622404764876e-07,
      "log_odds_chosen": 1.6045359373092651,
      "log_odds_ratio": -0.18750955164432526,
      "logits/chosen": -0.8011846542358398,
      "logits/rejected": -2.075422763824463,
      "logps/chosen": -1.4409395456314087,
      "logps/rejected": -2.8338732719421387,
      "loss": 1.4968,
      "nll_loss": 1.478006362915039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14409396052360535,
      "rewards/margins": 0.1392933875322342,
      "rewards/rejected": -0.28338736295700073,
      "step": 941
    },
    {
      "epoch": 2.596479116327235,
      "grad_norm": 0.22642040252685547,
      "learning_rate": 2.761335299539708e-07,
      "log_odds_chosen": 1.523880958557129,
      "log_odds_ratio": -0.2097545862197876,
      "logits/chosen": -0.7880173921585083,
      "logits/rejected": -2.0264089107513428,
      "logps/chosen": -1.3784044981002808,
      "logps/rejected": -2.6851532459259033,
      "loss": 1.4377,
      "nll_loss": 1.4167677164077759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13784044981002808,
      "rewards/margins": 0.13067489862442017,
      "rewards/rejected": -0.26851534843444824,
      "step": 942
    },
    {
      "epoch": 2.599240593717639,
      "grad_norm": 0.3195907473564148,
      "learning_rate": 2.7248368952908055e-07,
      "log_odds_chosen": 1.461771845817566,
      "log_odds_ratio": -0.21980953216552734,
      "logits/chosen": -0.848595380783081,
      "logits/rejected": -1.7577111721038818,
      "logps/chosen": -1.4408600330352783,
      "logps/rejected": -2.7052931785583496,
      "loss": 1.5089,
      "nll_loss": 1.4868979454040527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14408600330352783,
      "rewards/margins": 0.12644335627555847,
      "rewards/rejected": -0.2705293893814087,
      "step": 943
    },
    {
      "epoch": 2.6020020711080427,
      "grad_norm": 0.26655301451683044,
      "learning_rate": 2.688567402807357e-07,
      "log_odds_chosen": 1.8628268241882324,
      "log_odds_ratio": -0.15494072437286377,
      "logits/chosen": -0.8694819211959839,
      "logits/rejected": -1.999489188194275,
      "logps/chosen": -1.3600715398788452,
      "logps/rejected": -2.98134708404541,
      "loss": 1.4275,
      "nll_loss": 1.4120118618011475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1360071450471878,
      "rewards/margins": 0.16212758421897888,
      "rewards/rejected": -0.2981347441673279,
      "step": 944
    },
    {
      "epoch": 2.604763548498447,
      "grad_norm": 0.23980240523815155,
      "learning_rate": 2.652527194814511e-07,
      "log_odds_chosen": 1.4213838577270508,
      "log_odds_ratio": -0.22679410874843597,
      "logits/chosen": -0.8118551969528198,
      "logits/rejected": -1.8827842473983765,
      "logps/chosen": -1.3833218812942505,
      "logps/rejected": -2.594524383544922,
      "loss": 1.4616,
      "nll_loss": 1.4388917684555054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13833221793174744,
      "rewards/margins": 0.12112022936344147,
      "rewards/rejected": -0.2594524323940277,
      "step": 945
    },
    {
      "epoch": 2.6075250258888505,
      "grad_norm": 0.2134982943534851,
      "learning_rate": 2.6167166416811745e-07,
      "log_odds_chosen": 1.740950345993042,
      "log_odds_ratio": -0.16992521286010742,
      "logits/chosen": -0.8220755457878113,
      "logits/rejected": -1.9810476303100586,
      "logps/chosen": -1.3848917484283447,
      "logps/rejected": -2.8960180282592773,
      "loss": 1.46,
      "nll_loss": 1.4429820775985718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13848918676376343,
      "rewards/margins": 0.1511126607656479,
      "rewards/rejected": -0.2896018326282501,
      "step": 946
    },
    {
      "epoch": 2.610286503279254,
      "grad_norm": 0.26111698150634766,
      "learning_rate": 2.5811361114161745e-07,
      "log_odds_chosen": 1.6446951627731323,
      "log_odds_ratio": -0.18224820494651794,
      "logits/chosen": -0.8129628896713257,
      "logits/rejected": -1.9805124998092651,
      "logps/chosen": -1.4249690771102905,
      "logps/rejected": -2.854400396347046,
      "loss": 1.4867,
      "nll_loss": 1.4684371948242188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14249692857265472,
      "rewards/margins": 0.1429431289434433,
      "rewards/rejected": -0.285440057516098,
      "step": 947
    },
    {
      "epoch": 2.6130479806696583,
      "grad_norm": 0.2299281805753708,
      "learning_rate": 2.5457859696645245e-07,
      "log_odds_chosen": 1.480647325515747,
      "log_odds_ratio": -0.21617284417152405,
      "logits/chosen": -0.8652956485748291,
      "logits/rejected": -1.9248936176300049,
      "logps/chosen": -1.398078203201294,
      "logps/rejected": -2.6709463596343994,
      "loss": 1.461,
      "nll_loss": 1.4394274950027466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1398078352212906,
      "rewards/margins": 0.12728680670261383,
      "rewards/rejected": -0.2670946419239044,
      "step": 948
    },
    {
      "epoch": 2.6158094580600624,
      "grad_norm": 0.2682130038738251,
      "learning_rate": 2.510666579703616e-07,
      "log_odds_chosen": 1.8603851795196533,
      "log_odds_ratio": -0.15828196704387665,
      "logits/chosen": -0.9050633907318115,
      "logits/rejected": -1.748826026916504,
      "logps/chosen": -1.3446389436721802,
      "logps/rejected": -2.956476926803589,
      "loss": 1.4083,
      "nll_loss": 1.3925139904022217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13446390628814697,
      "rewards/margins": 0.16118377447128296,
      "rewards/rejected": -0.29564768075942993,
      "step": 949
    },
    {
      "epoch": 2.618570935450466,
      "grad_norm": 0.22530537843704224,
      "learning_rate": 2.4757783024395244e-07,
      "log_odds_chosen": 1.7264577150344849,
      "log_odds_ratio": -0.1791447103023529,
      "logits/chosen": -0.7733549475669861,
      "logits/rejected": -2.1992223262786865,
      "logps/chosen": -1.4929616451263428,
      "logps/rejected": -3.01509428024292,
      "loss": 1.5547,
      "nll_loss": 1.5368317365646362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14929616451263428,
      "rewards/margins": 0.15221327543258667,
      "rewards/rejected": -0.30150943994522095,
      "step": 950
    },
    {
      "epoch": 2.6213324128408697,
      "grad_norm": 0.23810777068138123,
      "learning_rate": 2.441121496403273e-07,
      "log_odds_chosen": 1.6358726024627686,
      "log_odds_ratio": -0.18412204086780548,
      "logits/chosen": -0.8609445095062256,
      "logits/rejected": -2.067732810974121,
      "logps/chosen": -1.4377186298370361,
      "logps/rejected": -2.861238479614258,
      "loss": 1.4963,
      "nll_loss": 1.4778951406478882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14377185702323914,
      "rewards/margins": 0.14235195517539978,
      "rewards/rejected": -0.2861238121986389,
      "step": 951
    },
    {
      "epoch": 2.624093890231274,
      "grad_norm": 0.22364532947540283,
      "learning_rate": 2.4066965177471645e-07,
      "log_odds_chosen": 1.5541980266571045,
      "log_odds_ratio": -0.2067326307296753,
      "logits/chosen": -0.7771640419960022,
      "logits/rejected": -1.7838490009307861,
      "logps/chosen": -1.354691505432129,
      "logps/rejected": -2.687770366668701,
      "loss": 1.4278,
      "nll_loss": 1.407082200050354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13546916842460632,
      "rewards/margins": 0.13330788910388947,
      "rewards/rejected": -0.268777072429657,
      "step": 952
    },
    {
      "epoch": 2.6268553676216775,
      "grad_norm": 0.2233099341392517,
      "learning_rate": 2.372503720241126e-07,
      "log_odds_chosen": 1.640329122543335,
      "log_odds_ratio": -0.1796395182609558,
      "logits/chosen": -0.7143995761871338,
      "logits/rejected": -1.8810615539550781,
      "logps/chosen": -1.4569883346557617,
      "logps/rejected": -2.888185501098633,
      "loss": 1.5234,
      "nll_loss": 1.5054751634597778,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14569886028766632,
      "rewards/margins": 0.1431196928024292,
      "rewards/rejected": -0.2888185381889343,
      "step": 953
    },
    {
      "epoch": 2.6296168450120816,
      "grad_norm": 0.2334192395210266,
      "learning_rate": 2.338543455269046e-07,
      "log_odds_chosen": 1.6295439004898071,
      "log_odds_ratio": -0.18369105458259583,
      "logits/chosen": -0.7941421866416931,
      "logits/rejected": -1.828993797302246,
      "logps/chosen": -1.334347128868103,
      "logps/rejected": -2.7231056690216064,
      "loss": 1.4216,
      "nll_loss": 1.4031999111175537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1334347277879715,
      "rewards/margins": 0.13887585699558258,
      "rewards/rejected": -0.2723105847835541,
      "step": 954
    },
    {
      "epoch": 2.6323783224024853,
      "grad_norm": 0.21528248488903046,
      "learning_rate": 2.3048160718252e-07,
      "log_odds_chosen": 1.8298442363739014,
      "log_odds_ratio": -0.15041130781173706,
      "logits/chosen": -0.9097654223442078,
      "logits/rejected": -2.181267499923706,
      "logps/chosen": -1.3622467517852783,
      "logps/rejected": -2.9486942291259766,
      "loss": 1.4346,
      "nll_loss": 1.419554591178894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1362246870994568,
      "rewards/margins": 0.15864476561546326,
      "rewards/rejected": -0.29486945271492004,
      "step": 955
    },
    {
      "epoch": 2.6351397997928894,
      "grad_norm": 0.24860750138759613,
      "learning_rate": 2.271321916510627e-07,
      "log_odds_chosen": 2.00567889213562,
      "log_odds_ratio": -0.13462091982364655,
      "logits/chosen": -0.8964054584503174,
      "logits/rejected": -2.0301802158355713,
      "logps/chosen": -1.3431572914123535,
      "logps/rejected": -3.094275712966919,
      "loss": 1.4133,
      "nll_loss": 1.3998205661773682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13431572914123535,
      "rewards/margins": 0.17511184513568878,
      "rewards/rejected": -0.3094276189804077,
      "step": 956
    },
    {
      "epoch": 2.637901277183293,
      "grad_norm": 0.2248452603816986,
      "learning_rate": 2.2380613335296037e-07,
      "log_odds_chosen": 1.632684588432312,
      "log_odds_ratio": -0.19020292162895203,
      "logits/chosen": -0.7716628909111023,
      "logits/rejected": -2.124098300933838,
      "logps/chosen": -1.4696786403656006,
      "logps/rejected": -2.9018936157226562,
      "loss": 1.5189,
      "nll_loss": 1.499929428100586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14696787297725677,
      "rewards/margins": 0.14322149753570557,
      "rewards/rejected": -0.29018938541412354,
      "step": 957
    },
    {
      "epoch": 2.6406627545736967,
      "grad_norm": 0.2505524158477783,
      "learning_rate": 2.205034664686076e-07,
      "log_odds_chosen": 1.6690312623977661,
      "log_odds_ratio": -0.19188562035560608,
      "logits/chosen": -0.8181016445159912,
      "logits/rejected": -1.8956444263458252,
      "logps/chosen": -1.399005651473999,
      "logps/rejected": -2.8518404960632324,
      "loss": 1.4847,
      "nll_loss": 1.4655529260635376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1399005800485611,
      "rewards/margins": 0.14528347551822662,
      "rewards/rejected": -0.2851840555667877,
      "step": 958
    },
    {
      "epoch": 2.643424231964101,
      "grad_norm": 0.21002128720283508,
      "learning_rate": 2.1722422493801716e-07,
      "log_odds_chosen": 1.861242413520813,
      "log_odds_ratio": -0.16027867794036865,
      "logits/chosen": -0.788373589515686,
      "logits/rejected": -2.0933547019958496,
      "logps/chosen": -1.401833415031433,
      "logps/rejected": -3.0320067405700684,
      "loss": 1.4803,
      "nll_loss": 1.4643118381500244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14018335938453674,
      "rewards/margins": 0.16301734745502472,
      "rewards/rejected": -0.3032006621360779,
      "step": 959
    },
    {
      "epoch": 2.646185709354505,
      "grad_norm": 0.24437056481838226,
      "learning_rate": 2.1396844246046904e-07,
      "log_odds_chosen": 1.6724685430526733,
      "log_odds_ratio": -0.1835877001285553,
      "logits/chosen": -0.8264733552932739,
      "logits/rejected": -1.9276909828186035,
      "logps/chosen": -1.4727625846862793,
      "logps/rejected": -2.939807415008545,
      "loss": 1.5417,
      "nll_loss": 1.52329683303833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14727625250816345,
      "rewards/margins": 0.14670446515083313,
      "rewards/rejected": -0.29398074746131897,
      "step": 960
    },
    {
      "epoch": 2.6489471867449086,
      "grad_norm": 0.2100415825843811,
      "learning_rate": 2.1073615249416536e-07,
      "log_odds_chosen": 1.6273092031478882,
      "log_odds_ratio": -0.18217778205871582,
      "logits/chosen": -0.7561967372894287,
      "logits/rejected": -2.013071298599243,
      "logps/chosen": -1.3234224319458008,
      "logps/rejected": -2.7003798484802246,
      "loss": 1.3868,
      "nll_loss": 1.3686209917068481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13234223425388336,
      "rewards/margins": 0.13769574463367462,
      "rewards/rejected": -0.270037978887558,
      "step": 961
    },
    {
      "epoch": 2.6517086641353123,
      "grad_norm": 0.22587327659130096,
      "learning_rate": 2.075273882558873e-07,
      "log_odds_chosen": 1.6430224180221558,
      "log_odds_ratio": -0.1941165179014206,
      "logits/chosen": -0.8549885153770447,
      "logits/rejected": -1.9616295099258423,
      "logps/chosen": -1.4437212944030762,
      "logps/rejected": -2.8785641193389893,
      "loss": 1.5107,
      "nll_loss": 1.491284728050232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1443721204996109,
      "rewards/margins": 0.14348429441452026,
      "rewards/rejected": -0.28785640001296997,
      "step": 962
    },
    {
      "epoch": 2.6544701415257164,
      "grad_norm": 0.23256927728652954,
      "learning_rate": 2.0434218272065154e-07,
      "log_odds_chosen": 1.7010345458984375,
      "log_odds_ratio": -0.17828573286533356,
      "logits/chosen": -0.7128124833106995,
      "logits/rejected": -1.9806416034698486,
      "logps/chosen": -1.4034103155136108,
      "logps/rejected": -2.88523530960083,
      "loss": 1.4768,
      "nll_loss": 1.4589712619781494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14034104347229004,
      "rewards/margins": 0.1481824815273285,
      "rewards/rejected": -0.28852352499961853,
      "step": 963
    },
    {
      "epoch": 2.65723161891612,
      "grad_norm": 0.2344788759946823,
      "learning_rate": 2.0118056862137358e-07,
      "log_odds_chosen": 1.6828176975250244,
      "log_odds_ratio": -0.1736169308423996,
      "logits/chosen": -0.8599337339401245,
      "logits/rejected": -2.0625905990600586,
      "logps/chosen": -1.3553197383880615,
      "logps/rejected": -2.8010125160217285,
      "loss": 1.4039,
      "nll_loss": 1.3865183591842651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13553199172019958,
      "rewards/margins": 0.1445692479610443,
      "rewards/rejected": -0.2801012396812439,
      "step": 964
    },
    {
      "epoch": 2.659993096306524,
      "grad_norm": 0.236652672290802,
      "learning_rate": 1.980425784485293e-07,
      "log_odds_chosen": 1.5190538167953491,
      "log_odds_ratio": -0.21639059484004974,
      "logits/chosen": -0.7456527948379517,
      "logits/rejected": -1.8640908002853394,
      "logps/chosen": -1.3491963148117065,
      "logps/rejected": -2.6379213333129883,
      "loss": 1.4086,
      "nll_loss": 1.3869160413742065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1349196434020996,
      "rewards/margins": 0.12887249886989594,
      "rewards/rejected": -0.26379215717315674,
      "step": 965
    },
    {
      "epoch": 2.662754573696928,
      "grad_norm": 0.21286852657794952,
      "learning_rate": 1.9492824444982378e-07,
      "log_odds_chosen": 1.6951912641525269,
      "log_odds_ratio": -0.17544129490852356,
      "logits/chosen": -0.792149007320404,
      "logits/rejected": -2.137972593307495,
      "logps/chosen": -1.3928117752075195,
      "logps/rejected": -2.8630385398864746,
      "loss": 1.4502,
      "nll_loss": 1.4327032566070557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13928118348121643,
      "rewards/margins": 0.14702269434928894,
      "rewards/rejected": -0.286303848028183,
      "step": 966
    },
    {
      "epoch": 2.665516051087332,
      "grad_norm": 0.23913699388504028,
      "learning_rate": 1.918375986298565e-07,
      "log_odds_chosen": 1.722267508506775,
      "log_odds_ratio": -0.17880691587924957,
      "logits/chosen": -0.7047528028488159,
      "logits/rejected": -1.9316701889038086,
      "logps/chosen": -1.433093786239624,
      "logps/rejected": -2.9400246143341064,
      "loss": 1.488,
      "nll_loss": 1.4701223373413086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14330938458442688,
      "rewards/margins": 0.15069308876991272,
      "rewards/rejected": -0.2940024733543396,
      "step": 967
    },
    {
      "epoch": 2.6682775284777356,
      "grad_norm": 0.24249720573425293,
      "learning_rate": 1.887706727497965e-07,
      "log_odds_chosen": 1.5532490015029907,
      "log_odds_ratio": -0.21471494436264038,
      "logits/chosen": -0.787434995174408,
      "logits/rejected": -1.668123722076416,
      "logps/chosen": -1.3064297437667847,
      "logps/rejected": -2.625889539718628,
      "loss": 1.3785,
      "nll_loss": 1.3570327758789062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13064298033714294,
      "rewards/margins": 0.13194599747657776,
      "rewards/rejected": -0.2625889778137207,
      "step": 968
    },
    {
      "epoch": 2.6710390058681392,
      "grad_norm": 0.2098485380411148,
      "learning_rate": 1.8572749832705074e-07,
      "log_odds_chosen": 1.7862035036087036,
      "log_odds_ratio": -0.16700312495231628,
      "logits/chosen": -0.8437852263450623,
      "logits/rejected": -2.3016438484191895,
      "logps/chosen": -1.377842903137207,
      "logps/rejected": -2.9322235584259033,
      "loss": 1.4347,
      "nll_loss": 1.4179821014404297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13778428733348846,
      "rewards/margins": 0.15543806552886963,
      "rewards/rejected": -0.2932223677635193,
      "step": 969
    },
    {
      "epoch": 2.6738004832585434,
      "grad_norm": 0.22286038100719452,
      "learning_rate": 1.8270810663494591e-07,
      "log_odds_chosen": 1.5529109239578247,
      "log_odds_ratio": -0.1945139765739441,
      "logits/chosen": -0.765722393989563,
      "logits/rejected": -2.180504322052002,
      "logps/chosen": -1.436352252960205,
      "logps/rejected": -2.7805469036102295,
      "loss": 1.5017,
      "nll_loss": 1.482216715812683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14363522827625275,
      "rewards/margins": 0.1344194859266281,
      "rewards/rejected": -0.27805471420288086,
      "step": 970
    },
    {
      "epoch": 2.676561960648947,
      "grad_norm": 0.21117275953292847,
      "learning_rate": 1.7971252870240292e-07,
      "log_odds_chosen": 1.7019497156143188,
      "log_odds_ratio": -0.17576996982097626,
      "logits/chosen": -0.7316712737083435,
      "logits/rejected": -2.2303671836853027,
      "logps/chosen": -1.419854998588562,
      "logps/rejected": -2.902592182159424,
      "loss": 1.4805,
      "nll_loss": 1.4629535675048828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14198550581932068,
      "rewards/margins": 0.14827370643615723,
      "rewards/rejected": -0.2902592122554779,
      "step": 971
    },
    {
      "epoch": 2.679323438039351,
      "grad_norm": 0.23764653503894806,
      "learning_rate": 1.767407953136202e-07,
      "log_odds_chosen": 1.4426761865615845,
      "log_odds_ratio": -0.21827517449855804,
      "logits/chosen": -0.8238155245780945,
      "logits/rejected": -1.960955262184143,
      "logps/chosen": -1.4752743244171143,
      "logps/rejected": -2.727480888366699,
      "loss": 1.5432,
      "nll_loss": 1.5213298797607422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14752742648124695,
      "rewards/margins": 0.1252206712961197,
      "rewards/rejected": -0.27274811267852783,
      "step": 972
    },
    {
      "epoch": 2.682084915429755,
      "grad_norm": 0.23228906095027924,
      "learning_rate": 1.737929370077554e-07,
      "log_odds_chosen": 1.5190385580062866,
      "log_odds_ratio": -0.21730369329452515,
      "logits/chosen": -0.7989239692687988,
      "logits/rejected": -1.6477134227752686,
      "logps/chosen": -1.3697959184646606,
      "logps/rejected": -2.6735963821411133,
      "loss": 1.442,
      "nll_loss": 1.4202287197113037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1369796097278595,
      "rewards/margins": 0.1303800344467163,
      "rewards/rejected": -0.2673596441745758,
      "step": 973
    },
    {
      "epoch": 2.684846392820159,
      "grad_norm": 0.22859090566635132,
      "learning_rate": 1.7086898407861486e-07,
      "log_odds_chosen": 1.8996504545211792,
      "log_odds_ratio": -0.1469726860523224,
      "logits/chosen": -0.8028102517127991,
      "logits/rejected": -2.2114791870117188,
      "logps/chosen": -1.437674880027771,
      "logps/rejected": -3.1114306449890137,
      "loss": 1.5027,
      "nll_loss": 1.488043189048767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14376749098300934,
      "rewards/margins": 0.1673755794763565,
      "rewards/rejected": -0.31114307045936584,
      "step": 974
    },
    {
      "epoch": 2.6876078702105626,
      "grad_norm": 0.21925011277198792,
      "learning_rate": 1.679689665743381e-07,
      "log_odds_chosen": 1.5367701053619385,
      "log_odds_ratio": -0.20496302843093872,
      "logits/chosen": -0.7729565501213074,
      "logits/rejected": -2.092475652694702,
      "logps/chosen": -1.4283016920089722,
      "logps/rejected": -2.751614809036255,
      "loss": 1.4789,
      "nll_loss": 1.458438515663147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14283017814159393,
      "rewards/margins": 0.13233129680156708,
      "rewards/rejected": -0.275161474943161,
      "step": 975
    },
    {
      "epoch": 2.6903693476009662,
      "grad_norm": 0.2413259893655777,
      "learning_rate": 1.6509291429709224e-07,
      "log_odds_chosen": 1.5076382160186768,
      "log_odds_ratio": -0.20157039165496826,
      "logits/chosen": -0.8660017848014832,
      "logits/rejected": -1.917562484741211,
      "logps/chosen": -1.4516851902008057,
      "logps/rejected": -2.7548704147338867,
      "loss": 1.5102,
      "nll_loss": 1.49000084400177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1451684981584549,
      "rewards/margins": 0.1303185373544693,
      "rewards/rejected": -0.2754870653152466,
      "step": 976
    },
    {
      "epoch": 2.6931308249913704,
      "grad_norm": 0.21110667288303375,
      "learning_rate": 1.6224085680276418e-07,
      "log_odds_chosen": 1.7144944667816162,
      "log_odds_ratio": -0.17031097412109375,
      "logits/chosen": -0.911084771156311,
      "logits/rejected": -2.202202558517456,
      "logps/chosen": -1.3859715461730957,
      "logps/rejected": -2.872420310974121,
      "loss": 1.4479,
      "nll_loss": 1.4308537244796753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13859716057777405,
      "rewards/margins": 0.14864489436149597,
      "rewards/rejected": -0.28724205493927,
      "step": 977
    },
    {
      "epoch": 2.6958923023817745,
      "grad_norm": 0.2264646589756012,
      "learning_rate": 1.59412823400657e-07,
      "log_odds_chosen": 1.5225772857666016,
      "log_odds_ratio": -0.20823891460895538,
      "logits/chosen": -0.8968706727027893,
      "logits/rejected": -1.7435719966888428,
      "logps/chosen": -1.3663400411605835,
      "logps/rejected": -2.6695518493652344,
      "loss": 1.4288,
      "nll_loss": 1.4079402685165405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1366340070962906,
      "rewards/margins": 0.1303211748600006,
      "rewards/rejected": -0.2669551968574524,
      "step": 978
    },
    {
      "epoch": 2.698653779772178,
      "grad_norm": 0.2247675657272339,
      "learning_rate": 1.5660884315319036e-07,
      "log_odds_chosen": 1.6573270559310913,
      "log_odds_ratio": -0.18946120142936707,
      "logits/chosen": -0.743579626083374,
      "logits/rejected": -2.06406569480896,
      "logps/chosen": -1.497208833694458,
      "logps/rejected": -2.953242301940918,
      "loss": 1.5269,
      "nll_loss": 1.5079742670059204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14972089231014252,
      "rewards/margins": 0.14560338854789734,
      "rewards/rejected": -0.29532429575920105,
      "step": 979
    },
    {
      "epoch": 2.701415257162582,
      "grad_norm": 0.24291600286960602,
      "learning_rate": 1.538289448755989e-07,
      "log_odds_chosen": 1.976696491241455,
      "log_odds_ratio": -0.13205505907535553,
      "logits/chosen": -0.8543053865432739,
      "logits/rejected": -2.206322193145752,
      "logps/chosen": -1.392202615737915,
      "logps/rejected": -3.125058174133301,
      "loss": 1.4566,
      "nll_loss": 1.4434106349945068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13922026753425598,
      "rewards/margins": 0.17328554391860962,
      "rewards/rejected": -0.3125058114528656,
      "step": 980
    },
    {
      "epoch": 2.704176734552986,
      "grad_norm": 0.22567534446716309,
      "learning_rate": 1.5107315713563942e-07,
      "log_odds_chosen": 1.5911579132080078,
      "log_odds_ratio": -0.2004013955593109,
      "logits/chosen": -0.8179699182510376,
      "logits/rejected": -1.968597650527954,
      "logps/chosen": -1.3965879678726196,
      "logps/rejected": -2.766774892807007,
      "loss": 1.4621,
      "nll_loss": 1.4420568943023682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13965877890586853,
      "rewards/margins": 0.13701872527599335,
      "rewards/rejected": -0.27667751908302307,
      "step": 981
    },
    {
      "epoch": 2.7069382119433896,
      "grad_norm": 0.22604532539844513,
      "learning_rate": 1.483415082532938e-07,
      "log_odds_chosen": 1.6166930198669434,
      "log_odds_ratio": -0.18223077058792114,
      "logits/chosen": -0.8054912090301514,
      "logits/rejected": -1.9989535808563232,
      "logps/chosen": -1.4191519021987915,
      "logps/rejected": -2.8198201656341553,
      "loss": 1.4748,
      "nll_loss": 1.456567645072937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419151872396469,
      "rewards/margins": 0.14006684720516205,
      "rewards/rejected": -0.28198203444480896,
      "step": 982
    },
    {
      "epoch": 2.7096996893337937,
      "grad_norm": 0.26856815814971924,
      "learning_rate": 1.456340263004813e-07,
      "log_odds_chosen": 1.6243596076965332,
      "log_odds_ratio": -0.18607592582702637,
      "logits/chosen": -0.7150140404701233,
      "logits/rejected": -1.617521047592163,
      "logps/chosen": -1.441032886505127,
      "logps/rejected": -2.855644702911377,
      "loss": 1.5154,
      "nll_loss": 1.496822476387024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1441032886505127,
      "rewards/margins": 0.14146117866039276,
      "rewards/rejected": -0.28556448221206665,
      "step": 983
    },
    {
      "epoch": 2.7124611667241973,
      "grad_norm": 0.23320171236991882,
      "learning_rate": 1.4295073910076757e-07,
      "log_odds_chosen": 1.7140666246414185,
      "log_odds_ratio": -0.17526394128799438,
      "logits/chosen": -0.7767703533172607,
      "logits/rejected": -1.9408453702926636,
      "logps/chosen": -1.2976926565170288,
      "logps/rejected": -2.762031078338623,
      "loss": 1.3844,
      "nll_loss": 1.366840124130249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12976926565170288,
      "rewards/margins": 0.14643386006355286,
      "rewards/rejected": -0.2762031555175781,
      "step": 984
    },
    {
      "epoch": 2.7152226441146015,
      "grad_norm": 0.2552090883255005,
      "learning_rate": 1.4029167422908107e-07,
      "log_odds_chosen": 1.5235956907272339,
      "log_odds_ratio": -0.20630496740341187,
      "logits/chosen": -0.8022536635398865,
      "logits/rejected": -1.7930173873901367,
      "logps/chosen": -1.3867225646972656,
      "logps/rejected": -2.6964144706726074,
      "loss": 1.4484,
      "nll_loss": 1.4278028011322021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13867226243019104,
      "rewards/margins": 0.13096915185451508,
      "rewards/rejected": -0.2696414589881897,
      "step": 985
    },
    {
      "epoch": 2.717984121505005,
      "grad_norm": 0.22836096584796906,
      "learning_rate": 1.3765685901142718e-07,
      "log_odds_chosen": 1.4520777463912964,
      "log_odds_ratio": -0.22023352980613708,
      "logits/chosen": -0.8523675203323364,
      "logits/rejected": -1.841475009918213,
      "logps/chosen": -1.402612328529358,
      "logps/rejected": -2.647597074508667,
      "loss": 1.4789,
      "nll_loss": 1.456899881362915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1402612328529358,
      "rewards/margins": 0.12449847161769867,
      "rewards/rejected": -0.26475971937179565,
      "step": 986
    },
    {
      "epoch": 2.720745598895409,
      "grad_norm": 0.2556321620941162,
      "learning_rate": 1.3504632052460803e-07,
      "log_odds_chosen": 1.4889233112335205,
      "log_odds_ratio": -0.21026012301445007,
      "logits/chosen": -0.790485143661499,
      "logits/rejected": -1.8951472043991089,
      "logps/chosen": -1.3497055768966675,
      "logps/rejected": -2.614793539047241,
      "loss": 1.4307,
      "nll_loss": 1.4096403121948242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.134970560669899,
      "rewards/margins": 0.12650878727436066,
      "rewards/rejected": -0.26147934794425964,
      "step": 987
    },
    {
      "epoch": 2.723507076285813,
      "grad_norm": 0.22848525643348694,
      "learning_rate": 1.3246008559594708e-07,
      "log_odds_chosen": 1.435408353805542,
      "log_odds_ratio": -0.2240704894065857,
      "logits/chosen": -0.8080549240112305,
      "logits/rejected": -1.9176137447357178,
      "logps/chosen": -1.3366992473602295,
      "logps/rejected": -2.5517029762268066,
      "loss": 1.4005,
      "nll_loss": 1.378090262413025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1336699277162552,
      "rewards/margins": 0.12150037288665771,
      "rewards/rejected": -0.2551702857017517,
      "step": 988
    },
    {
      "epoch": 2.726268553676217,
      "grad_norm": 0.22359248995780945,
      "learning_rate": 1.2989818080300836e-07,
      "log_odds_chosen": 1.6755691766738892,
      "log_odds_ratio": -0.17601075768470764,
      "logits/chosen": -0.7978137135505676,
      "logits/rejected": -2.044973134994507,
      "logps/chosen": -1.3630049228668213,
      "logps/rejected": -2.8028440475463867,
      "loss": 1.4252,
      "nll_loss": 1.4075582027435303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13630050420761108,
      "rewards/margins": 0.1439839005470276,
      "rewards/rejected": -0.28028440475463867,
      "step": 989
    },
    {
      "epoch": 2.7290300310666207,
      "grad_norm": 0.2378959059715271,
      "learning_rate": 1.273606324733284e-07,
      "log_odds_chosen": 1.7684226036071777,
      "log_odds_ratio": -0.1636146754026413,
      "logits/chosen": -0.8993475437164307,
      "logits/rejected": -1.91559636592865,
      "logps/chosen": -1.445130705833435,
      "logps/rejected": -2.9962029457092285,
      "loss": 1.5039,
      "nll_loss": 1.4875314235687256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1445130705833435,
      "rewards/margins": 0.15510722994804382,
      "rewards/rejected": -0.2996203303337097,
      "step": 990
    },
    {
      "epoch": 2.7317915084570243,
      "grad_norm": 0.2171250879764557,
      "learning_rate": 1.2484746668414176e-07,
      "log_odds_chosen": 1.6286214590072632,
      "log_odds_ratio": -0.1897205263376236,
      "logits/chosen": -0.7582758069038391,
      "logits/rejected": -2.000474691390991,
      "logps/chosen": -1.4032773971557617,
      "logps/rejected": -2.8157689571380615,
      "loss": 1.4781,
      "nll_loss": 1.459139347076416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14032772183418274,
      "rewards/margins": 0.1412491649389267,
      "rewards/rejected": -0.28157690167427063,
      "step": 991
    },
    {
      "epoch": 2.7345529858474285,
      "grad_norm": 0.21213258802890778,
      "learning_rate": 1.223587092621162e-07,
      "log_odds_chosen": 1.5115113258361816,
      "log_odds_ratio": -0.20901797711849213,
      "logits/chosen": -0.8601601123809814,
      "logits/rejected": -1.9279264211654663,
      "logps/chosen": -1.4223215579986572,
      "logps/rejected": -2.725409746170044,
      "loss": 1.4812,
      "nll_loss": 1.460340142250061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14223216474056244,
      "rewards/margins": 0.1303088366985321,
      "rewards/rejected": -0.27254101634025574,
      "step": 992
    },
    {
      "epoch": 2.737314463237832,
      "grad_norm": 0.2581441402435303,
      "learning_rate": 1.1989438578308394e-07,
      "log_odds_chosen": 1.6569344997406006,
      "log_odds_ratio": -0.181952103972435,
      "logits/chosen": -0.8941519856452942,
      "logits/rejected": -1.814483880996704,
      "logps/chosen": -1.4135879278182983,
      "logps/rejected": -2.8543639183044434,
      "loss": 1.4987,
      "nll_loss": 1.4805060625076294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14135879278182983,
      "rewards/margins": 0.1440775990486145,
      "rewards/rejected": -0.28543636202812195,
      "step": 993
    },
    {
      "epoch": 2.7400759406282362,
      "grad_norm": 0.2210199534893036,
      "learning_rate": 1.1745452157178206e-07,
      "log_odds_chosen": 1.7576181888580322,
      "log_odds_ratio": -0.18280641734600067,
      "logits/chosen": -0.8181760311126709,
      "logits/rejected": -2.02272891998291,
      "logps/chosen": -1.3050153255462646,
      "logps/rejected": -2.8192529678344727,
      "loss": 1.3727,
      "nll_loss": 1.3544033765792847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13050153851509094,
      "rewards/margins": 0.15142375230789185,
      "rewards/rejected": -0.2819252610206604,
      "step": 994
    },
    {
      "epoch": 2.74283741801864,
      "grad_norm": 0.21427030861377716,
      "learning_rate": 1.1503914170159058e-07,
      "log_odds_chosen": 1.7605931758880615,
      "log_odds_ratio": -0.16457465291023254,
      "logits/chosen": -0.7366156578063965,
      "logits/rejected": -2.1901280879974365,
      "logps/chosen": -1.4154034852981567,
      "logps/rejected": -2.9496779441833496,
      "loss": 1.4716,
      "nll_loss": 1.455174446105957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14154034852981567,
      "rewards/margins": 0.15342746675014496,
      "rewards/rejected": -0.29496777057647705,
      "step": 995
    },
    {
      "epoch": 2.745598895409044,
      "grad_norm": 0.22262705862522125,
      "learning_rate": 1.1264827099427417e-07,
      "log_odds_chosen": 1.4252431392669678,
      "log_odds_ratio": -0.23039494454860687,
      "logits/chosen": -0.813992977142334,
      "logits/rejected": -1.825137734413147,
      "logps/chosen": -1.3184692859649658,
      "logps/rejected": -2.5151712894439697,
      "loss": 1.3915,
      "nll_loss": 1.3685065507888794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13184693455696106,
      "rewards/margins": 0.11967021971940994,
      "rewards/rejected": -0.2515171468257904,
      "step": 996
    },
    {
      "epoch": 2.7483603727994477,
      "grad_norm": 0.3166691064834595,
      "learning_rate": 1.1028193401972865e-07,
      "log_odds_chosen": 1.5943597555160522,
      "log_odds_ratio": -0.19616588950157166,
      "logits/chosen": -0.7697460055351257,
      "logits/rejected": -1.8766896724700928,
      "logps/chosen": -1.3608465194702148,
      "logps/rejected": -2.720303773880005,
      "loss": 1.4281,
      "nll_loss": 1.4085121154785156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.136084645986557,
      "rewards/margins": 0.13594573736190796,
      "rewards/rejected": -0.27203038334846497,
      "step": 997
    },
    {
      "epoch": 2.7511218501898513,
      "grad_norm": 0.25932177901268005,
      "learning_rate": 1.0794015509572819e-07,
      "log_odds_chosen": 1.7572764158248901,
      "log_odds_ratio": -0.17751987278461456,
      "logits/chosen": -0.7274531126022339,
      "logits/rejected": -2.109034776687622,
      "logps/chosen": -1.4595894813537598,
      "logps/rejected": -3.003801107406616,
      "loss": 1.5366,
      "nll_loss": 1.518887996673584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14595896005630493,
      "rewards/margins": 0.1544211506843567,
      "rewards/rejected": -0.300380140542984,
      "step": 998
    },
    {
      "epoch": 2.7538833275802554,
      "grad_norm": 0.21890608966350555,
      "learning_rate": 1.0562295828767388e-07,
      "log_odds_chosen": 1.4044764041900635,
      "log_odds_ratio": -0.223425954580307,
      "logits/chosen": -0.8820509910583496,
      "logits/rejected": -1.9093981981277466,
      "logps/chosen": -1.4362064599990845,
      "logps/rejected": -2.643540859222412,
      "loss": 1.5029,
      "nll_loss": 1.4805753231048584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14362065494060516,
      "rewards/margins": 0.12073343247175217,
      "rewards/rejected": -0.26435407996177673,
      "step": 999
    },
    {
      "epoch": 2.7566448049706596,
      "grad_norm": 0.24640429019927979,
      "learning_rate": 1.0333036740834857e-07,
      "log_odds_chosen": 1.58961820602417,
      "log_odds_ratio": -0.19259408116340637,
      "logits/chosen": -0.7218985557556152,
      "logits/rejected": -1.7569760084152222,
      "logps/chosen": -1.4714789390563965,
      "logps/rejected": -2.855508804321289,
      "loss": 1.5109,
      "nll_loss": 1.4916527271270752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14714789390563965,
      "rewards/margins": 0.13840298354625702,
      "rewards/rejected": -0.28555089235305786,
      "step": 1000
    },
    {
      "epoch": 2.7594062823610632,
      "grad_norm": 0.21864457428455353,
      "learning_rate": 1.0106240601767042e-07,
      "log_odds_chosen": 2.0963706970214844,
      "log_odds_ratio": -0.11976057291030884,
      "logits/chosen": -0.8929874300956726,
      "logits/rejected": -2.175584077835083,
      "logps/chosen": -1.3230259418487549,
      "logps/rejected": -3.150878667831421,
      "loss": 1.3855,
      "nll_loss": 1.373477578163147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13230259716510773,
      "rewards/margins": 0.1827852725982666,
      "rewards/rejected": -0.3150879144668579,
      "step": 1001
    },
    {
      "epoch": 2.762167759751467,
      "grad_norm": 0.22030071914196014,
      "learning_rate": 9.881909742245177e-08,
      "log_odds_chosen": 1.7982732057571411,
      "log_odds_ratio": -0.17107604444026947,
      "logits/chosen": -0.8717531561851501,
      "logits/rejected": -1.9789412021636963,
      "logps/chosen": -1.355002522468567,
      "logps/rejected": -2.9149961471557617,
      "loss": 1.4187,
      "nll_loss": 1.4016332626342773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1355002522468567,
      "rewards/margins": 0.1559993475675583,
      "rewards/rejected": -0.2914995849132538,
      "step": 1002
    },
    {
      "epoch": 2.764929237141871,
      "grad_norm": 0.22139884531497955,
      "learning_rate": 9.66004646761598e-08,
      "log_odds_chosen": 1.4478939771652222,
      "log_odds_ratio": -0.23130184412002563,
      "logits/chosen": -0.765604555606842,
      "logits/rejected": -1.7363425493240356,
      "logps/chosen": -1.3936283588409424,
      "logps/rejected": -2.6382076740264893,
      "loss": 1.4489,
      "nll_loss": 1.4257304668426514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13936282694339752,
      "rewards/margins": 0.1244579553604126,
      "rewards/rejected": -0.2638207674026489,
      "step": 1003
    },
    {
      "epoch": 2.7676907145322747,
      "grad_norm": 0.2260187566280365,
      "learning_rate": 9.440653057867816e-08,
      "log_odds_chosen": 1.5612225532531738,
      "log_odds_ratio": -0.19809266924858093,
      "logits/chosen": -0.8372408151626587,
      "logits/rejected": -1.9978140592575073,
      "logps/chosen": -1.4296715259552002,
      "logps/rejected": -2.781604290008545,
      "loss": 1.4996,
      "nll_loss": 1.4798098802566528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14296714961528778,
      "rewards/margins": 0.13519325852394104,
      "rewards/rejected": -0.27816042304039,
      "step": 1004
    },
    {
      "epoch": 2.7704521919226788,
      "grad_norm": 0.23682570457458496,
      "learning_rate": 9.223731767607436e-08,
      "log_odds_chosen": 1.5558514595031738,
      "log_odds_ratio": -0.20280753076076508,
      "logits/chosen": -0.8458856344223022,
      "logits/rejected": -1.9428819417953491,
      "logps/chosen": -1.3925399780273438,
      "logps/rejected": -2.7330634593963623,
      "loss": 1.4611,
      "nll_loss": 1.440795660018921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13925398886203766,
      "rewards/margins": 0.1340523660182953,
      "rewards/rejected": -0.27330636978149414,
      "step": 1005
    },
    {
      "epoch": 2.7732136693130824,
      "grad_norm": 0.29044103622436523,
      "learning_rate": 9.00928482603669e-08,
      "log_odds_chosen": 1.513688325881958,
      "log_odds_ratio": -0.20887477695941925,
      "logits/chosen": -0.6599283814430237,
      "logits/rejected": -1.8435096740722656,
      "logps/chosen": -1.4128738641738892,
      "logps/rejected": -2.7181389331817627,
      "loss": 1.4785,
      "nll_loss": 1.4576369524002075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412874013185501,
      "rewards/margins": 0.13052652776241302,
      "rewards/rejected": -0.27181389927864075,
      "step": 1006
    },
    {
      "epoch": 2.7759751467034866,
      "grad_norm": 0.21997316181659698,
      "learning_rate": 8.797314436929744e-08,
      "log_odds_chosen": 1.2918083667755127,
      "log_odds_ratio": -0.25957056879997253,
      "logits/chosen": -0.7696832418441772,
      "logits/rejected": -1.4652957916259766,
      "logps/chosen": -1.4523776769638062,
      "logps/rejected": -2.558363437652588,
      "loss": 1.5169,
      "nll_loss": 1.4908944368362427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452377736568451,
      "rewards/margins": 0.11059856414794922,
      "rewards/rejected": -0.2558363378047943,
      "step": 1007
    },
    {
      "epoch": 2.77873662409389,
      "grad_norm": 0.219607412815094,
      "learning_rate": 8.587822778610284e-08,
      "log_odds_chosen": 1.7893471717834473,
      "log_odds_ratio": -0.1684584766626358,
      "logits/chosen": -0.7286239266395569,
      "logits/rejected": -1.910126805305481,
      "logps/chosen": -1.3401432037353516,
      "logps/rejected": -2.8861756324768066,
      "loss": 1.4018,
      "nll_loss": 1.3849496841430664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1340143233537674,
      "rewards/margins": 0.15460322797298431,
      "rewards/rejected": -0.2886175513267517,
      "step": 1008
    },
    {
      "epoch": 2.781498101484294,
      "grad_norm": 0.22155028581619263,
      "learning_rate": 8.380812003929262e-08,
      "log_odds_chosen": 1.4924676418304443,
      "log_odds_ratio": -0.214213564991951,
      "logits/chosen": -0.7699148058891296,
      "logits/rejected": -2.101424217224121,
      "logps/chosen": -1.4633634090423584,
      "logps/rejected": -2.7596728801727295,
      "loss": 1.4937,
      "nll_loss": 1.4722706079483032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1463363617658615,
      "rewards/margins": 0.1296309381723404,
      "rewards/rejected": -0.2759672999382019,
      "step": 1009
    },
    {
      "epoch": 2.784259578874698,
      "grad_norm": 0.21873348951339722,
      "learning_rate": 8.176284240242638e-08,
      "log_odds_chosen": 1.6635732650756836,
      "log_odds_ratio": -0.19265908002853394,
      "logits/chosen": -0.7586681842803955,
      "logits/rejected": -2.0201668739318848,
      "logps/chosen": -1.3821016550064087,
      "logps/rejected": -2.8210911750793457,
      "loss": 1.4441,
      "nll_loss": 1.4248504638671875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13821017742156982,
      "rewards/margins": 0.14389893412590027,
      "rewards/rejected": -0.2821091115474701,
      "step": 1010
    },
    {
      "epoch": 2.787021056265102,
      "grad_norm": 0.24069830775260925,
      "learning_rate": 7.97424158938967e-08,
      "log_odds_chosen": 1.7478724718093872,
      "log_odds_ratio": -0.1709989607334137,
      "logits/chosen": -0.7697115540504456,
      "logits/rejected": -2.1029648780822754,
      "logps/chosen": -1.4904512166976929,
      "logps/rejected": -3.030327320098877,
      "loss": 1.5413,
      "nll_loss": 1.5241514444351196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14904512465000153,
      "rewards/margins": 0.15398761630058289,
      "rewards/rejected": -0.3030327558517456,
      "step": 1011
    },
    {
      "epoch": 2.7897825336555058,
      "grad_norm": 0.25440898537635803,
      "learning_rate": 7.774686127671183e-08,
      "log_odds_chosen": 1.7158838510513306,
      "log_odds_ratio": -0.17278915643692017,
      "logits/chosen": -0.9012861251831055,
      "logits/rejected": -1.9430004358291626,
      "logps/chosen": -1.3608380556106567,
      "logps/rejected": -2.839327096939087,
      "loss": 1.4156,
      "nll_loss": 1.398327350616455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13608382642269135,
      "rewards/margins": 0.1478489190340042,
      "rewards/rejected": -0.28393274545669556,
      "step": 1012
    },
    {
      "epoch": 2.7925440110459094,
      "grad_norm": 0.21382860839366913,
      "learning_rate": 7.577619905828281e-08,
      "log_odds_chosen": 1.4606471061706543,
      "log_odds_ratio": -0.21766388416290283,
      "logits/chosen": -0.7786560654640198,
      "logits/rejected": -2.091545820236206,
      "logps/chosen": -1.405806064605713,
      "logps/rejected": -2.663357973098755,
      "loss": 1.4767,
      "nll_loss": 1.4549262523651123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14058060944080353,
      "rewards/margins": 0.1257551908493042,
      "rewards/rejected": -0.26633578538894653,
      "step": 1013
    },
    {
      "epoch": 2.7953054884363135,
      "grad_norm": 0.22583609819412231,
      "learning_rate": 7.383044949021339e-08,
      "log_odds_chosen": 1.6658389568328857,
      "log_odds_ratio": -0.1828538179397583,
      "logits/chosen": -0.7913579344749451,
      "logits/rejected": -2.100635528564453,
      "logps/chosen": -1.4507535696029663,
      "logps/rejected": -2.9083504676818848,
      "loss": 1.5045,
      "nll_loss": 1.486259937286377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14507536590099335,
      "rewards/margins": 0.1457597017288208,
      "rewards/rejected": -0.29083505272865295,
      "step": 1014
    },
    {
      "epoch": 2.798066965826717,
      "grad_norm": 0.2718450129032135,
      "learning_rate": 7.190963256809069e-08,
      "log_odds_chosen": 1.8116415739059448,
      "log_odds_ratio": -0.1589178889989853,
      "logits/chosen": -0.8496946692466736,
      "logits/rejected": -2.0155324935913086,
      "logps/chosen": -1.2840626239776611,
      "logps/rejected": -2.827091693878174,
      "loss": 1.3719,
      "nll_loss": 1.3559659719467163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12840627133846283,
      "rewards/margins": 0.15430289506912231,
      "rewards/rejected": -0.28270918130874634,
      "step": 1015
    },
    {
      "epoch": 2.8008284432171213,
      "grad_norm": 0.21330896019935608,
      "learning_rate": 7.001376803128041e-08,
      "log_odds_chosen": 1.8585755825042725,
      "log_odds_ratio": -0.15236105024814606,
      "logits/chosen": -0.8107438683509827,
      "logits/rejected": -2.0344109535217285,
      "logps/chosen": -1.3621731996536255,
      "logps/rejected": -2.976593494415283,
      "loss": 1.448,
      "nll_loss": 1.432778000831604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13621731102466583,
      "rewards/margins": 0.16144204139709473,
      "rewards/rejected": -0.29765933752059937,
      "step": 1016
    },
    {
      "epoch": 2.803589920607525,
      "grad_norm": 0.20405304431915283,
      "learning_rate": 6.81428753627239e-08,
      "log_odds_chosen": 1.5870081186294556,
      "log_odds_ratio": -0.20238448679447174,
      "logits/chosen": -0.7854856848716736,
      "logits/rejected": -2.0655314922332764,
      "logps/chosen": -1.4296401739120483,
      "logps/rejected": -2.8039541244506836,
      "loss": 1.4973,
      "nll_loss": 1.4770238399505615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14296402037143707,
      "rewards/margins": 0.13743142783641815,
      "rewards/rejected": -0.2803954482078552,
      "step": 1017
    },
    {
      "epoch": 2.806351397997929,
      "grad_norm": 0.2566664516925812,
      "learning_rate": 6.629697378873839e-08,
      "log_odds_chosen": 1.4977682828903198,
      "log_odds_ratio": -0.21054419875144958,
      "logits/chosen": -0.9586695432662964,
      "logits/rejected": -1.8258790969848633,
      "logps/chosen": -1.3797677755355835,
      "logps/rejected": -2.663789987564087,
      "loss": 1.4449,
      "nll_loss": 1.4238519668579102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1379767656326294,
      "rewards/margins": 0.1284022480249405,
      "rewards/rejected": -0.2663789987564087,
      "step": 1018
    },
    {
      "epoch": 2.8091128753883328,
      "grad_norm": 0.22852636873722076,
      "learning_rate": 6.447608227881791e-08,
      "log_odds_chosen": 1.5595167875289917,
      "log_odds_ratio": -0.2022312581539154,
      "logits/chosen": -0.8290364146232605,
      "logits/rejected": -1.985574722290039,
      "logps/chosen": -1.3868674039840698,
      "logps/rejected": -2.7301459312438965,
      "loss": 1.4508,
      "nll_loss": 1.4305788278579712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13868674635887146,
      "rewards/margins": 0.13432787358760834,
      "rewards/rejected": -0.2730146050453186,
      "step": 1019
    },
    {
      "epoch": 2.8118743527787364,
      "grad_norm": 0.21699006855487823,
      "learning_rate": 6.268021954544095e-08,
      "log_odds_chosen": 1.9818371534347534,
      "log_odds_ratio": -0.15955258905887604,
      "logits/chosen": -0.8705717325210571,
      "logits/rejected": -2.0158958435058594,
      "logps/chosen": -1.3820674419403076,
      "logps/rejected": -3.1282825469970703,
      "loss": 1.4534,
      "nll_loss": 1.4374067783355713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13820675015449524,
      "rewards/margins": 0.17462150752544403,
      "rewards/rejected": -0.31282827258110046,
      "step": 1020
    },
    {
      "epoch": 2.8146358301691405,
      "grad_norm": 0.2339145392179489,
      "learning_rate": 6.090940404387513e-08,
      "log_odds_chosen": 1.4671416282653809,
      "log_odds_ratio": -0.2181987166404724,
      "logits/chosen": -0.7629361748695374,
      "logits/rejected": -1.8477253913879395,
      "logps/chosen": -1.452636480331421,
      "logps/rejected": -2.7193386554718018,
      "loss": 1.51,
      "nll_loss": 1.488135814666748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452636420726776,
      "rewards/margins": 0.12667018175125122,
      "rewards/rejected": -0.2719338536262512,
      "step": 1021
    },
    {
      "epoch": 2.817397307559544,
      "grad_norm": 0.2369954138994217,
      "learning_rate": 5.9163653971989754e-08,
      "log_odds_chosen": 1.9682278633117676,
      "log_odds_ratio": -0.1359677016735077,
      "logits/chosen": -0.8030321598052979,
      "logits/rejected": -2.2195727825164795,
      "logps/chosen": -1.3785918951034546,
      "logps/rejected": -3.1034982204437256,
      "loss": 1.4406,
      "nll_loss": 1.4270315170288086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13785919547080994,
      "rewards/margins": 0.17249061167240143,
      "rewards/rejected": -0.31034982204437256,
      "step": 1022
    },
    {
      "epoch": 2.8201587849499483,
      "grad_norm": 0.2538975179195404,
      "learning_rate": 5.744298727006769e-08,
      "log_odds_chosen": 1.4653068780899048,
      "log_odds_ratio": -0.22388669848442078,
      "logits/chosen": -0.8812580108642578,
      "logits/rejected": -1.8244831562042236,
      "logps/chosen": -1.4292573928833008,
      "logps/rejected": -2.692967414855957,
      "loss": 1.4846,
      "nll_loss": 1.4622304439544678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14292573928833008,
      "rewards/margins": 0.12637099623680115,
      "rewards/rejected": -0.2692967355251312,
      "step": 1023
    },
    {
      "epoch": 2.822920262340352,
      "grad_norm": 0.22296157479286194,
      "learning_rate": 5.574742162062163e-08,
      "log_odds_chosen": 1.5828129053115845,
      "log_odds_ratio": -0.20364131033420563,
      "logits/chosen": -0.822142481803894,
      "logits/rejected": -2.045734167098999,
      "logps/chosen": -1.4172347784042358,
      "logps/rejected": -2.793931722640991,
      "loss": 1.4825,
      "nll_loss": 1.4621087312698364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14172348380088806,
      "rewards/margins": 0.13766971230506897,
      "rewards/rejected": -0.27939319610595703,
      "step": 1024
    },
    {
      "epoch": 2.825681739730756,
      "grad_norm": 0.27316057682037354,
      "learning_rate": 5.407697444821169e-08,
      "log_odds_chosen": 1.5825798511505127,
      "log_odds_ratio": -0.1950148344039917,
      "logits/chosen": -0.8047593832015991,
      "logits/rejected": -1.9445643424987793,
      "logps/chosen": -1.3558223247528076,
      "logps/rejected": -2.7116684913635254,
      "loss": 1.439,
      "nll_loss": 1.419493556022644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13558222353458405,
      "rewards/margins": 0.13558462262153625,
      "rewards/rejected": -0.2711668610572815,
      "step": 1025
    },
    {
      "epoch": 2.8284432171211598,
      "grad_norm": 0.22010549902915955,
      "learning_rate": 5.2431662919267825e-08,
      "log_odds_chosen": 1.6912816762924194,
      "log_odds_ratio": -0.17304657399654388,
      "logits/chosen": -0.8214058876037598,
      "logits/rejected": -2.1822879314422607,
      "logps/chosen": -1.488768219947815,
      "logps/rejected": -2.977398157119751,
      "loss": 1.5355,
      "nll_loss": 1.5182044506072998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1488768458366394,
      "rewards/margins": 0.14886298775672913,
      "rewards/rejected": -0.29773983359336853,
      "step": 1026
    },
    {
      "epoch": 2.8312046945115634,
      "grad_norm": 0.23960784077644348,
      "learning_rate": 5.0811503941911314e-08,
      "log_odds_chosen": 1.5059239864349365,
      "log_odds_ratio": -0.21391917765140533,
      "logits/chosen": -0.7945892810821533,
      "logits/rejected": -1.8174819946289062,
      "logps/chosen": -1.4129031896591187,
      "logps/rejected": -2.7106990814208984,
      "loss": 1.4695,
      "nll_loss": 1.448089599609375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412903219461441,
      "rewards/margins": 0.12977956235408783,
      "rewards/rejected": -0.27106988430023193,
      "step": 1027
    },
    {
      "epoch": 2.8339661719019675,
      "grad_norm": 0.23000836372375488,
      "learning_rate": 4.9216514165781885e-08,
      "log_odds_chosen": 1.7422019243240356,
      "log_odds_ratio": -0.16638413071632385,
      "logits/chosen": -0.7677799463272095,
      "logits/rejected": -1.9751092195510864,
      "logps/chosen": -1.39187753200531,
      "logps/rejected": -2.9060842990875244,
      "loss": 1.465,
      "nll_loss": 1.4483141899108887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.139187753200531,
      "rewards/margins": 0.15142066776752472,
      "rewards/rejected": -0.2906084358692169,
      "step": 1028
    },
    {
      "epoch": 2.8367276492923716,
      "grad_norm": 0.23794633150100708,
      "learning_rate": 4.7646709981868376e-08,
      "log_odds_chosen": 1.6807467937469482,
      "log_odds_ratio": -0.18301939964294434,
      "logits/chosen": -0.8502731323242188,
      "logits/rejected": -1.98602294921875,
      "logps/chosen": -1.3824533224105835,
      "logps/rejected": -2.8385558128356934,
      "loss": 1.4542,
      "nll_loss": 1.4359116554260254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1382453441619873,
      "rewards/margins": 0.14561022818088531,
      "rewards/rejected": -0.2838555872440338,
      "step": 1029
    },
    {
      "epoch": 2.8394891266827753,
      "grad_norm": 0.231614887714386,
      "learning_rate": 4.61021075223364e-08,
      "log_odds_chosen": 1.7197368144989014,
      "log_odds_ratio": -0.16804471611976624,
      "logits/chosen": -0.9392529129981995,
      "logits/rejected": -1.9837356805801392,
      "logps/chosen": -1.3155676126480103,
      "logps/rejected": -2.783620595932007,
      "loss": 1.4019,
      "nll_loss": 1.3851432800292969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13155676424503326,
      "rewards/margins": 0.1468052715063095,
      "rewards/rejected": -0.2783620357513428,
      "step": 1030
    },
    {
      "epoch": 2.842250604073179,
      "grad_norm": 0.24176815152168274,
      "learning_rate": 4.458272266036706e-08,
      "log_odds_chosen": 1.55967378616333,
      "log_odds_ratio": -0.19546456634998322,
      "logits/chosen": -0.762442946434021,
      "logits/rejected": -1.9133505821228027,
      "logps/chosen": -1.335386872291565,
      "logps/rejected": -2.65338134765625,
      "loss": 1.4057,
      "nll_loss": 1.3861135244369507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13353869318962097,
      "rewards/margins": 0.13179941475391388,
      "rewards/rejected": -0.26533812284469604,
      "step": 1031
    },
    {
      "epoch": 2.845012081463583,
      "grad_norm": 0.23981979489326477,
      "learning_rate": 4.308857100999042e-08,
      "log_odds_chosen": 1.8310432434082031,
      "log_odds_ratio": -0.16425098478794098,
      "logits/chosen": -0.7952133417129517,
      "logits/rejected": -1.92050302028656,
      "logps/chosen": -1.368022084236145,
      "logps/rejected": -2.9524054527282715,
      "loss": 1.4436,
      "nll_loss": 1.427202820777893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13680219650268555,
      "rewards/margins": 0.15843833982944489,
      "rewards/rejected": -0.2952405512332916,
      "step": 1032
    },
    {
      "epoch": 2.8477735588539868,
      "grad_norm": 0.21197573840618134,
      "learning_rate": 4.161966792592592e-08,
      "log_odds_chosen": 1.3388291597366333,
      "log_odds_ratio": -0.24759265780448914,
      "logits/chosen": -0.7649065256118774,
      "logits/rejected": -1.754675030708313,
      "logps/chosen": -1.348063349723816,
      "logps/rejected": -2.4776344299316406,
      "loss": 1.4147,
      "nll_loss": 1.3899341821670532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13480634987354279,
      "rewards/margins": 0.11295709013938904,
      "rewards/rejected": -0.24776342511177063,
      "step": 1033
    },
    {
      "epoch": 2.850535036244391,
      "grad_norm": 0.22503337264060974,
      "learning_rate": 4.017602850342584e-08,
      "log_odds_chosen": 1.665643334388733,
      "log_odds_ratio": -0.17374378442764282,
      "logits/chosen": -0.7818294167518616,
      "logits/rejected": -2.1388704776763916,
      "logps/chosen": -1.4129152297973633,
      "logps/rejected": -2.8565893173217773,
      "loss": 1.4763,
      "nll_loss": 1.458892583847046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1412915140390396,
      "rewards/margins": 0.14436742663383484,
      "rewards/rejected": -0.28565895557403564,
      "step": 1034
    },
    {
      "epoch": 2.8532965136347945,
      "grad_norm": 0.23232252895832062,
      "learning_rate": 3.8757667578119e-08,
      "log_odds_chosen": 1.9293830394744873,
      "log_odds_ratio": -0.1441594660282135,
      "logits/chosen": -0.7760025262832642,
      "logits/rejected": -2.0618064403533936,
      "logps/chosen": -1.413285493850708,
      "logps/rejected": -3.106600761413574,
      "loss": 1.477,
      "nll_loss": 1.4625605344772339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14132854342460632,
      "rewards/margins": 0.16933153569698334,
      "rewards/rejected": -0.31066006422042847,
      "step": 1035
    },
    {
      "epoch": 2.8560579910251986,
      "grad_norm": 0.22999964654445648,
      "learning_rate": 3.736459972585815e-08,
      "log_odds_chosen": 1.4999421834945679,
      "log_odds_ratio": -0.20854999125003815,
      "logits/chosen": -0.7216120958328247,
      "logits/rejected": -2.1589982509613037,
      "logps/chosen": -1.503130555152893,
      "logps/rejected": -2.8132858276367188,
      "loss": 1.5602,
      "nll_loss": 1.5393545627593994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15031304955482483,
      "rewards/margins": 0.13101550936698914,
      "rewards/rejected": -0.28132855892181396,
      "step": 1036
    },
    {
      "epoch": 2.8588194684156023,
      "grad_norm": 0.2181134819984436,
      "learning_rate": 3.5996839262571194e-08,
      "log_odds_chosen": 1.4268596172332764,
      "log_odds_ratio": -0.2180023491382599,
      "logits/chosen": -0.8290450572967529,
      "logits/rejected": -1.8734112977981567,
      "logps/chosen": -1.4272325038909912,
      "logps/rejected": -2.6529884338378906,
      "loss": 1.4948,
      "nll_loss": 1.4729880094528198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14272327721118927,
      "rewards/margins": 0.12257558107376099,
      "rewards/rejected": -0.26529884338378906,
      "step": 1037
    },
    {
      "epoch": 2.861580945806006,
      "grad_norm": 0.21630226075649261,
      "learning_rate": 3.4654400244112654e-08,
      "log_odds_chosen": 1.6311062574386597,
      "log_odds_ratio": -0.18956345319747925,
      "logits/chosen": -0.7698126435279846,
      "logits/rejected": -2.0631442070007324,
      "logps/chosen": -1.4231253862380981,
      "logps/rejected": -2.838050127029419,
      "loss": 1.4917,
      "nll_loss": 1.4727219343185425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14231254160404205,
      "rewards/margins": 0.14149242639541626,
      "rewards/rejected": -0.2838050127029419,
      "step": 1038
    },
    {
      "epoch": 2.86434242319641,
      "grad_norm": 0.2282114028930664,
      "learning_rate": 3.333729646612077e-08,
      "log_odds_chosen": 1.1922599077224731,
      "log_odds_ratio": -0.28115126490592957,
      "logits/chosen": -0.8158324956893921,
      "logits/rejected": -1.5724602937698364,
      "logps/chosen": -1.439221978187561,
      "logps/rejected": -2.454829454421997,
      "loss": 1.5126,
      "nll_loss": 1.4844740629196167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14392220973968506,
      "rewards/margins": 0.10156073421239853,
      "rewards/rejected": -0.245482936501503,
      "step": 1039
    },
    {
      "epoch": 2.867103900586814,
      "grad_norm": 0.21230220794677734,
      "learning_rate": 3.204554146387456e-08,
      "log_odds_chosen": 1.720137357711792,
      "log_odds_ratio": -0.173086479306221,
      "logits/chosen": -0.8133577108383179,
      "logits/rejected": -2.1605277061462402,
      "logps/chosen": -1.3774471282958984,
      "logps/rejected": -2.8622329235076904,
      "loss": 1.4418,
      "nll_loss": 1.4244989156723022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1377447098493576,
      "rewards/margins": 0.14847859740257263,
      "rewards/rejected": -0.2862233519554138,
      "step": 1040
    },
    {
      "epoch": 2.869865377977218,
      "grad_norm": 0.2199348360300064,
      "learning_rate": 3.077914851215585e-08,
      "log_odds_chosen": 1.7481780052185059,
      "log_odds_ratio": -0.16739507019519806,
      "logits/chosen": -0.8020718693733215,
      "logits/rejected": -2.197960138320923,
      "logps/chosen": -1.3972680568695068,
      "logps/rejected": -2.9158308506011963,
      "loss": 1.4411,
      "nll_loss": 1.4243154525756836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13972681760787964,
      "rewards/margins": 0.15185627341270447,
      "rewards/rejected": -0.2915830910205841,
      "step": 1041
    },
    {
      "epoch": 2.8726268553676215,
      "grad_norm": 0.22360913455486298,
      "learning_rate": 2.9538130625110796e-08,
      "log_odds_chosen": 1.7841436862945557,
      "log_odds_ratio": -0.16358324885368347,
      "logits/chosen": -0.7960190773010254,
      "logits/rejected": -2.2209391593933105,
      "logps/chosen": -1.4135005474090576,
      "logps/rejected": -2.9724953174591064,
      "loss": 1.4611,
      "nll_loss": 1.4447667598724365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14135007560253143,
      "rewards/margins": 0.15589945018291473,
      "rewards/rejected": -0.29724955558776855,
      "step": 1042
    },
    {
      "epoch": 2.8753883327580256,
      "grad_norm": 0.2290942221879959,
      "learning_rate": 2.8322500556118847e-08,
      "log_odds_chosen": 1.751172661781311,
      "log_odds_ratio": -0.16167476773262024,
      "logits/chosen": -0.7484520673751831,
      "logits/rejected": -1.920337200164795,
      "logps/chosen": -1.4236546754837036,
      "logps/rejected": -2.9533438682556152,
      "loss": 1.4913,
      "nll_loss": 1.475111961364746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1423654705286026,
      "rewards/margins": 0.15296894311904907,
      "rewards/rejected": -0.2953343987464905,
      "step": 1043
    },
    {
      "epoch": 2.8781498101484293,
      "grad_norm": 0.22796989977359772,
      "learning_rate": 2.7132270797659567e-08,
      "log_odds_chosen": 1.6315937042236328,
      "log_odds_ratio": -0.18258602917194366,
      "logits/chosen": -0.6966241002082825,
      "logits/rejected": -2.115699052810669,
      "logps/chosen": -1.3849658966064453,
      "logps/rejected": -2.7913544178009033,
      "loss": 1.4382,
      "nll_loss": 1.4199258089065552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13849659264087677,
      "rewards/margins": 0.14063885807991028,
      "rewards/rejected": -0.27913543581962585,
      "step": 1044
    },
    {
      "epoch": 2.8809112875388334,
      "grad_norm": 0.23840682208538055,
      "learning_rate": 2.5967453581185187e-08,
      "log_odds_chosen": 1.6786651611328125,
      "log_odds_ratio": -0.1893036663532257,
      "logits/chosen": -0.6516211032867432,
      "logits/rejected": -1.883230209350586,
      "logps/chosen": -1.4420583248138428,
      "logps/rejected": -2.913923501968384,
      "loss": 1.524,
      "nll_loss": 1.5050742626190186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14420585334300995,
      "rewards/margins": 0.1471865177154541,
      "rewards/rejected": -0.29139238595962524,
      "step": 1045
    },
    {
      "epoch": 2.883672764929237,
      "grad_norm": 0.23551054298877716,
      "learning_rate": 2.4828060876995462e-08,
      "log_odds_chosen": 1.4917556047439575,
      "log_odds_ratio": -0.2092304825782776,
      "logits/chosen": -0.7992951273918152,
      "logits/rejected": -1.956207036972046,
      "logps/chosen": -1.499756932258606,
      "logps/rejected": -2.800715923309326,
      "loss": 1.5506,
      "nll_loss": 1.5296664237976074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14997568726539612,
      "rewards/margins": 0.13009591400623322,
      "rewards/rejected": -0.2800716161727905,
      "step": 1046
    },
    {
      "epoch": 2.886434242319641,
      "grad_norm": 0.21869802474975586,
      "learning_rate": 2.371410439411248e-08,
      "log_odds_chosen": 1.826915979385376,
      "log_odds_ratio": -0.1546439379453659,
      "logits/chosen": -0.864876925945282,
      "logits/rejected": -2.1301004886627197,
      "logps/chosen": -1.3602914810180664,
      "logps/rejected": -2.9441142082214355,
      "loss": 1.4152,
      "nll_loss": 1.3997533321380615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13602915406227112,
      "rewards/margins": 0.15838226675987244,
      "rewards/rejected": -0.29441142082214355,
      "step": 1047
    },
    {
      "epoch": 2.889195719710045,
      "grad_norm": 0.22462975978851318,
      "learning_rate": 2.262559558016325e-08,
      "log_odds_chosen": 1.7097482681274414,
      "log_odds_ratio": -0.1781245321035385,
      "logits/chosen": -0.8304237723350525,
      "logits/rejected": -2.108201265335083,
      "logps/chosen": -1.3457623720169067,
      "logps/rejected": -2.819329261779785,
      "loss": 1.4148,
      "nll_loss": 1.397024154663086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13457626104354858,
      "rewards/margins": 0.14735668897628784,
      "rewards/rejected": -0.2819329500198364,
      "step": 1048
    },
    {
      "epoch": 2.8919571971004485,
      "grad_norm": 0.22334124147891998,
      "learning_rate": 2.1562545621259534e-08,
      "log_odds_chosen": 1.6744170188903809,
      "log_odds_ratio": -0.1740560382604599,
      "logits/chosen": -0.7684246301651001,
      "logits/rejected": -1.8581528663635254,
      "logps/chosen": -1.316230297088623,
      "logps/rejected": -2.7437851428985596,
      "loss": 1.4007,
      "nll_loss": 1.3832471370697021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1316230446100235,
      "rewards/margins": 0.14275550842285156,
      "rewards/rejected": -0.2743785083293915,
      "step": 1049
    },
    {
      "epoch": 2.8947186744908526,
      "grad_norm": 0.22718796133995056,
      "learning_rate": 2.052496544188487e-08,
      "log_odds_chosen": 1.427244782447815,
      "log_odds_ratio": -0.2224445343017578,
      "logits/chosen": -0.7508878111839294,
      "logits/rejected": -2.012288808822632,
      "logps/chosen": -1.4001238346099854,
      "logps/rejected": -2.6197235584259033,
      "loss": 1.4666,
      "nll_loss": 1.444330096244812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14001239836215973,
      "rewards/margins": 0.12195995450019836,
      "rewards/rejected": -0.2619723379611969,
      "step": 1050
    },
    {
      "epoch": 2.8974801518812567,
      "grad_norm": 0.24057117104530334,
      "learning_rate": 1.9512865704780504e-08,
      "log_odds_chosen": 1.6308375597000122,
      "log_odds_ratio": -0.19242976605892181,
      "logits/chosen": -0.7897810339927673,
      "logits/rejected": -1.8339242935180664,
      "logps/chosen": -1.4479725360870361,
      "logps/rejected": -2.8747165203094482,
      "loss": 1.5115,
      "nll_loss": 1.4922785758972168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14479725062847137,
      "rewards/margins": 0.14267440140247345,
      "rewards/rejected": -0.2874716520309448,
      "step": 1051
    },
    {
      "epoch": 2.9002416292716604,
      "grad_norm": 0.24796703457832336,
      "learning_rate": 1.852625681083742e-08,
      "log_odds_chosen": 1.5950390100479126,
      "log_odds_ratio": -0.20395228266716003,
      "logits/chosen": -0.9735223054885864,
      "logits/rejected": -2.0237746238708496,
      "logps/chosen": -1.4253889322280884,
      "logps/rejected": -2.802295207977295,
      "loss": 1.501,
      "nll_loss": 1.4805935621261597,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1425389051437378,
      "rewards/margins": 0.13769064843654633,
      "rewards/rejected": -0.2802295386791229,
      "step": 1052
    },
    {
      "epoch": 2.903003106662064,
      "grad_norm": 0.25104832649230957,
      "learning_rate": 1.7565148898988916e-08,
      "log_odds_chosen": 1.5715587139129639,
      "log_odds_ratio": -0.19690975546836853,
      "logits/chosen": -0.8224033713340759,
      "logits/rejected": -1.7730584144592285,
      "logps/chosen": -1.4370899200439453,
      "logps/rejected": -2.799690008163452,
      "loss": 1.4958,
      "nll_loss": 1.4761466979980469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437089890241623,
      "rewards/margins": 0.1362600475549698,
      "rewards/rejected": -0.2799690365791321,
      "step": 1053
    },
    {
      "epoch": 2.905764584052468,
      "grad_norm": 0.22561626136302948,
      "learning_rate": 1.6629551846104874e-08,
      "log_odds_chosen": 1.4102277755737305,
      "log_odds_ratio": -0.24327263236045837,
      "logits/chosen": -0.7862374782562256,
      "logits/rejected": -1.792546272277832,
      "logps/chosen": -1.4360134601593018,
      "logps/rejected": -2.6559813022613525,
      "loss": 1.4941,
      "nll_loss": 1.469818353652954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14360132813453674,
      "rewards/margins": 0.12199681997299194,
      "rewards/rejected": -0.2655981481075287,
      "step": 1054
    },
    {
      "epoch": 2.908526061442872,
      "grad_norm": 0.2336493730545044,
      "learning_rate": 1.571947526689349e-08,
      "log_odds_chosen": 1.5892236232757568,
      "log_odds_ratio": -0.1958022266626358,
      "logits/chosen": -0.8617162108421326,
      "logits/rejected": -1.8406635522842407,
      "logps/chosen": -1.4430006742477417,
      "logps/rejected": -2.8268589973449707,
      "loss": 1.5114,
      "nll_loss": 1.4918040037155151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14430007338523865,
      "rewards/margins": 0.1383858323097229,
      "rewards/rejected": -0.28268590569496155,
      "step": 1055
    },
    {
      "epoch": 2.911287538833276,
      "grad_norm": 0.23663151264190674,
      "learning_rate": 1.483492851379914e-08,
      "log_odds_chosen": 1.7302970886230469,
      "log_odds_ratio": -0.16981743276119232,
      "logits/chosen": -0.8490622639656067,
      "logits/rejected": -2.0886123180389404,
      "logps/chosen": -1.3956736326217651,
      "logps/rejected": -2.897519588470459,
      "loss": 1.4492,
      "nll_loss": 1.4322402477264404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13956737518310547,
      "rewards/margins": 0.15018458664417267,
      "rewards/rejected": -0.28975194692611694,
      "step": 1056
    },
    {
      "epoch": 2.9140490162236796,
      "grad_norm": 0.24135632812976837,
      "learning_rate": 1.3975920676908838e-08,
      "log_odds_chosen": 1.4057644605636597,
      "log_odds_ratio": -0.22804616391658783,
      "logits/chosen": -0.6639127135276794,
      "logits/rejected": -1.8646632432937622,
      "logps/chosen": -1.440869688987732,
      "logps/rejected": -2.6512222290039062,
      "loss": 1.5137,
      "nll_loss": 1.4908647537231445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14408698678016663,
      "rewards/margins": 0.12103521823883057,
      "rewards/rejected": -0.2651222050189972,
      "step": 1057
    },
    {
      "epoch": 2.9168104936140837,
      "grad_norm": 0.2815915644168854,
      "learning_rate": 1.3142460583856487e-08,
      "log_odds_chosen": 1.5973938703536987,
      "log_odds_ratio": -0.18859460949897766,
      "logits/chosen": -0.8560981750488281,
      "logits/rejected": -1.7154216766357422,
      "logps/chosen": -1.403993010520935,
      "logps/rejected": -2.7832603454589844,
      "loss": 1.4698,
      "nll_loss": 1.4509804248809814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1403992921113968,
      "rewards/margins": 0.13792674243450165,
      "rewards/rejected": -0.27832603454589844,
      "step": 1058
    },
    {
      "epoch": 2.9195719710044874,
      "grad_norm": 0.26606282591819763,
      "learning_rate": 1.2334556799734887e-08,
      "log_odds_chosen": 1.3971562385559082,
      "log_odds_ratio": -0.2267698049545288,
      "logits/chosen": -0.8830655217170715,
      "logits/rejected": -1.6509592533111572,
      "logps/chosen": -1.3806607723236084,
      "logps/rejected": -2.5692031383514404,
      "loss": 1.4562,
      "nll_loss": 1.4335050582885742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13806608319282532,
      "rewards/margins": 0.11885424703359604,
      "rewards/rejected": -0.25692033767700195,
      "step": 1059
    },
    {
      "epoch": 2.922333448394891,
      "grad_norm": 0.22054573893547058,
      "learning_rate": 1.1552217627004426e-08,
      "log_odds_chosen": 1.5220046043395996,
      "log_odds_ratio": -0.2108788937330246,
      "logits/chosen": -0.7800368666648865,
      "logits/rejected": -1.8905466794967651,
      "logps/chosen": -1.3639496564865112,
      "logps/rejected": -2.6639370918273926,
      "loss": 1.4512,
      "nll_loss": 1.4301084280014038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13639497756958008,
      "rewards/margins": 0.12999877333641052,
      "rewards/rejected": -0.2663937211036682,
      "step": 1060
    },
    {
      "epoch": 2.925094925785295,
      "grad_norm": 0.2264561504125595,
      "learning_rate": 1.079545110541147e-08,
      "log_odds_chosen": 1.7285921573638916,
      "log_odds_ratio": -0.173441544175148,
      "logits/chosen": -0.7664073705673218,
      "logits/rejected": -2.037234306335449,
      "logps/chosen": -1.389496088027954,
      "logps/rejected": -2.8904924392700195,
      "loss": 1.4607,
      "nll_loss": 1.4433540105819702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13894961774349213,
      "rewards/margins": 0.15009962022304535,
      "rewards/rejected": -0.2890492379665375,
      "step": 1061
    },
    {
      "epoch": 2.927856403175699,
      "grad_norm": 0.214057058095932,
      "learning_rate": 1.006426501190233e-08,
      "log_odds_chosen": 1.740164875984192,
      "log_odds_ratio": -0.1689753532409668,
      "logits/chosen": -0.7992459535598755,
      "logits/rejected": -2.1515536308288574,
      "logps/chosen": -1.3622679710388184,
      "logps/rejected": -2.8680598735809326,
      "loss": 1.4201,
      "nll_loss": 1.4032260179519653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13622678816318512,
      "rewards/margins": 0.15057921409606934,
      "rewards/rejected": -0.28680598735809326,
      "step": 1062
    },
    {
      "epoch": 2.930617880566103,
      "grad_norm": 0.23196722567081451,
      "learning_rate": 9.358666860545817e-09,
      "log_odds_chosen": 1.8300212621688843,
      "log_odds_ratio": -0.1529330015182495,
      "logits/chosen": -0.7448672652244568,
      "logits/rejected": -2.0953400135040283,
      "logps/chosen": -1.3911778926849365,
      "logps/rejected": -2.9877777099609375,
      "loss": 1.449,
      "nll_loss": 1.4337555170059204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1391177773475647,
      "rewards/margins": 0.1596599817276001,
      "rewards/rejected": -0.2987777292728424,
      "step": 1063
    },
    {
      "epoch": 2.9333793579565066,
      "grad_norm": 0.23241263628005981,
      "learning_rate": 8.67866390245442e-09,
      "log_odds_chosen": 1.4628114700317383,
      "log_odds_ratio": -0.2144569754600525,
      "logits/chosen": -0.8802769184112549,
      "logits/rejected": -1.8510754108428955,
      "logps/chosen": -1.3962527513504028,
      "logps/rejected": -2.648005723953247,
      "loss": 1.4754,
      "nll_loss": 1.4539076089859009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13962529599666595,
      "rewards/margins": 0.12517526745796204,
      "rewards/rejected": -0.2648005783557892,
      "step": 1064
    },
    {
      "epoch": 2.9361408353469107,
      "grad_norm": 0.2281099408864975,
      "learning_rate": 8.024263125710751e-09,
      "log_odds_chosen": 1.6584781408309937,
      "log_odds_ratio": -0.19228924810886383,
      "logits/chosen": -0.8245370388031006,
      "logits/rejected": -2.0083014965057373,
      "logps/chosen": -1.4000036716461182,
      "logps/rejected": -2.8395965099334717,
      "loss": 1.4715,
      "nll_loss": 1.452258586883545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1400003731250763,
      "rewards/margins": 0.14395931363105774,
      "rewards/rejected": -0.28395968675613403,
      "step": 1065
    },
    {
      "epoch": 2.9389023127373144,
      "grad_norm": 0.22550606727600098,
      "learning_rate": 7.3954712552953835e-09,
      "log_odds_chosen": 1.671879768371582,
      "log_odds_ratio": -0.18240472674369812,
      "logits/chosen": -0.7968007326126099,
      "logits/rejected": -2.037839889526367,
      "logps/chosen": -1.4791769981384277,
      "logps/rejected": -2.9491400718688965,
      "loss": 1.5383,
      "nll_loss": 1.5200833082199097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14791768789291382,
      "rewards/margins": 0.14699633419513702,
      "rewards/rejected": -0.29491403698921204,
      "step": 1066
    },
    {
      "epoch": 2.941663790127718,
      "grad_norm": 0.2156606912612915,
      "learning_rate": 6.792294753017181e-09,
      "log_odds_chosen": 1.5692713260650635,
      "log_odds_ratio": -0.21755948662757874,
      "logits/chosen": -0.8175379633903503,
      "logits/rejected": -1.7547521591186523,
      "logps/chosen": -1.3667752742767334,
      "logps/rejected": -2.7211294174194336,
      "loss": 1.4277,
      "nll_loss": 1.4059330224990845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13667753338813782,
      "rewards/margins": 0.13543540239334106,
      "rewards/rejected": -0.2721129357814789,
      "step": 1067
    },
    {
      "epoch": 2.944425267518122,
      "grad_norm": 0.2185620218515396,
      "learning_rate": 6.214739817448634e-09,
      "log_odds_chosen": 1.5329852104187012,
      "log_odds_ratio": -0.22458839416503906,
      "logits/chosen": -0.7700731754302979,
      "logits/rejected": -1.848755955696106,
      "logps/chosen": -1.4170128107070923,
      "logps/rejected": -2.7483105659484863,
      "loss": 1.4819,
      "nll_loss": 1.459450125694275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14170128107070923,
      "rewards/margins": 0.1331297755241394,
      "rewards/rejected": -0.27483102679252625,
      "step": 1068
    },
    {
      "epoch": 2.9471867449085263,
      "grad_norm": 0.22919808328151703,
      "learning_rate": 5.662812383859795e-09,
      "log_odds_chosen": 1.7428557872772217,
      "log_odds_ratio": -0.16942371428012848,
      "logits/chosen": -0.7553185224533081,
      "logits/rejected": -1.9580934047698975,
      "logps/chosen": -1.3186566829681396,
      "logps/rejected": -2.807377576828003,
      "loss": 1.3891,
      "nll_loss": 1.3721272945404053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1318656951189041,
      "rewards/margins": 0.14887209236621857,
      "rewards/rejected": -0.2807377576828003,
      "step": 1069
    },
    {
      "epoch": 2.94994822229893,
      "grad_norm": 0.20699773728847504,
      "learning_rate": 5.136518124159162e-09,
      "log_odds_chosen": 1.5876624584197998,
      "log_odds_ratio": -0.2104184627532959,
      "logits/chosen": -0.7132779359817505,
      "logits/rejected": -1.9351344108581543,
      "logps/chosen": -1.360245704650879,
      "logps/rejected": -2.7215259075164795,
      "loss": 1.4345,
      "nll_loss": 1.413459062576294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1360245794057846,
      "rewards/margins": 0.1361280381679535,
      "rewards/rejected": -0.2721526324748993,
      "step": 1070
    },
    {
      "epoch": 2.9527096996893336,
      "grad_norm": 0.2266066074371338,
      "learning_rate": 4.63586244683456e-09,
      "log_odds_chosen": 1.3620774745941162,
      "log_odds_ratio": -0.23232533037662506,
      "logits/chosen": -0.811676025390625,
      "logits/rejected": -1.820404052734375,
      "logps/chosen": -1.4501413106918335,
      "logps/rejected": -2.6185903549194336,
      "loss": 1.5106,
      "nll_loss": 1.487402319908142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14501415193080902,
      "rewards/margins": 0.11684489995241165,
      "rewards/rejected": -0.2618590295314789,
      "step": 1071
    },
    {
      "epoch": 2.9554711770797377,
      "grad_norm": 0.2652856111526489,
      "learning_rate": 4.160850496897906e-09,
      "log_odds_chosen": 1.6856169700622559,
      "log_odds_ratio": -0.1874770075082779,
      "logits/chosen": -0.8645150661468506,
      "logits/rejected": -1.93236243724823,
      "logps/chosen": -1.4419993162155151,
      "logps/rejected": -2.9198200702667236,
      "loss": 1.5035,
      "nll_loss": 1.4847571849822998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.144199937582016,
      "rewards/margins": 0.1477820724248886,
      "rewards/rejected": -0.2919819951057434,
      "step": 1072
    },
    {
      "epoch": 2.9582326544701414,
      "grad_norm": 0.21664103865623474,
      "learning_rate": 3.7114871558313614e-09,
      "log_odds_chosen": 1.7259833812713623,
      "log_odds_ratio": -0.1650668829679489,
      "logits/chosen": -0.8545508980751038,
      "logits/rejected": -2.1027960777282715,
      "logps/chosen": -1.424027681350708,
      "logps/rejected": -2.9264843463897705,
      "loss": 1.4887,
      "nll_loss": 1.4721843004226685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1424027681350708,
      "rewards/margins": 0.15024566650390625,
      "rewards/rejected": -0.29264843463897705,
      "step": 1073
    },
    {
      "epoch": 2.9609941318605455,
      "grad_norm": 0.22490212321281433,
      "learning_rate": 3.287777041539042e-09,
      "log_odds_chosen": 1.437325119972229,
      "log_odds_ratio": -0.22529327869415283,
      "logits/chosen": -0.8498947620391846,
      "logits/rejected": -1.7718011140823364,
      "logps/chosen": -1.3232507705688477,
      "logps/rejected": -2.540151596069336,
      "loss": 1.4037,
      "nll_loss": 1.3812105655670166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13232508301734924,
      "rewards/margins": 0.12169007956981659,
      "rewards/rejected": -0.25401514768600464,
      "step": 1074
    },
    {
      "epoch": 2.963755609250949,
      "grad_norm": 0.2170701026916504,
      "learning_rate": 2.8897245082978863e-09,
      "log_odds_chosen": 1.7161061763763428,
      "log_odds_ratio": -0.17984658479690552,
      "logits/chosen": -0.8351077437400818,
      "logits/rejected": -2.1513471603393555,
      "logps/chosen": -1.3885916471481323,
      "logps/rejected": -2.882119655609131,
      "loss": 1.4441,
      "nll_loss": 1.426088571548462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13885916769504547,
      "rewards/margins": 0.1493528038263321,
      "rewards/rejected": -0.2882119417190552,
      "step": 1075
    },
    {
      "epoch": 2.9665170866413533,
      "grad_norm": 0.22006747126579285,
      "learning_rate": 2.5173336467135266e-09,
      "log_odds_chosen": 1.820219874382019,
      "log_odds_ratio": -0.15693305432796478,
      "logits/chosen": -0.8700850009918213,
      "logits/rejected": -2.169057607650757,
      "logps/chosen": -1.352325201034546,
      "logps/rejected": -2.9299466609954834,
      "loss": 1.4083,
      "nll_loss": 1.392561435699463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13523253798484802,
      "rewards/margins": 0.15776214003562927,
      "rewards/rejected": -0.2929946780204773,
      "step": 1076
    },
    {
      "epoch": 2.969278564031757,
      "grad_norm": 0.2348693460226059,
      "learning_rate": 2.170608283677822e-09,
      "log_odds_chosen": 1.7549494504928589,
      "log_odds_ratio": -0.1712295114994049,
      "logits/chosen": -0.9456162452697754,
      "logits/rejected": -1.9611583948135376,
      "logps/chosen": -1.3380199670791626,
      "logps/rejected": -2.8467626571655273,
      "loss": 1.4134,
      "nll_loss": 1.3962781429290771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13380199670791626,
      "rewards/margins": 0.1508743017911911,
      "rewards/rejected": -0.28467628359794617,
      "step": 1077
    },
    {
      "epoch": 2.9720400414221606,
      "grad_norm": 0.23374752700328827,
      "learning_rate": 1.8495519823308329e-09,
      "log_odds_chosen": 1.8946998119354248,
      "log_odds_ratio": -0.1446288675069809,
      "logits/chosen": -0.8362556099891663,
      "logits/rejected": -2.1337828636169434,
      "logps/chosen": -1.4020692110061646,
      "logps/rejected": -3.0605435371398926,
      "loss": 1.4662,
      "nll_loss": 1.4517593383789062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14020691812038422,
      "rewards/margins": 0.16584742069244385,
      "rewards/rejected": -0.30605435371398926,
      "step": 1078
    },
    {
      "epoch": 2.9748015188125647,
      "grad_norm": 0.2259787917137146,
      "learning_rate": 1.5541680420227968e-09,
      "log_odds_chosen": 1.4082696437835693,
      "log_odds_ratio": -0.22261746227741241,
      "logits/chosen": -0.9028100371360779,
      "logits/rejected": -1.8519865274429321,
      "logps/chosen": -1.3738371133804321,
      "logps/rejected": -2.56370210647583,
      "loss": 1.4394,
      "nll_loss": 1.4171141386032104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13738371431827545,
      "rewards/margins": 0.11898650228977203,
      "rewards/rejected": -0.2563702166080475,
      "step": 1079
    },
    {
      "epoch": 2.977562996202969,
      "grad_norm": 0.22851480543613434,
      "learning_rate": 1.284459498280266e-09,
      "log_odds_chosen": 1.6992191076278687,
      "log_odds_ratio": -0.1761513650417328,
      "logits/chosen": -0.7371349334716797,
      "logits/rejected": -1.8762682676315308,
      "logps/chosen": -1.3590335845947266,
      "logps/rejected": -2.823000431060791,
      "loss": 1.4165,
      "nll_loss": 1.3989176750183105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13590337336063385,
      "rewards/margins": 0.1463966816663742,
      "rewards/rejected": -0.28230005502700806,
      "step": 1080
    },
    {
      "epoch": 2.9803244735933725,
      "grad_norm": 0.22858591377735138,
      "learning_rate": 1.0404291227764097e-09,
      "log_odds_chosen": 1.6860997676849365,
      "log_odds_ratio": -0.1761903464794159,
      "logits/chosen": -0.8258135914802551,
      "logits/rejected": -2.1534647941589355,
      "logps/chosen": -1.435535192489624,
      "logps/rejected": -2.90619158744812,
      "loss": 1.4886,
      "nll_loss": 1.4709571599960327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14355352520942688,
      "rewards/margins": 0.1470656543970108,
      "rewards/rejected": -0.2906191945075989,
      "step": 1081
    },
    {
      "epoch": 2.983085950983776,
      "grad_norm": 0.2309102714061737,
      "learning_rate": 8.220794233004814e-10,
      "log_odds_chosen": 1.8477295637130737,
      "log_odds_ratio": -0.17313013970851898,
      "logits/chosen": -0.8398474454879761,
      "logits/rejected": -1.861135721206665,
      "logps/chosen": -1.325697898864746,
      "logps/rejected": -2.925485134124756,
      "loss": 1.41,
      "nll_loss": 1.3926753997802734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13256977498531342,
      "rewards/margins": 0.1599787175655365,
      "rewards/rejected": -0.2925485074520111,
      "step": 1082
    },
    {
      "epoch": 2.9858474283741803,
      "grad_norm": 0.2285986989736557,
      "learning_rate": 6.294126437336734e-10,
      "log_odds_chosen": 1.8615604639053345,
      "log_odds_ratio": -0.14680801331996918,
      "logits/chosen": -0.87724369764328,
      "logits/rejected": -2.150709390640259,
      "logps/chosen": -1.4179985523223877,
      "logps/rejected": -3.0496137142181396,
      "loss": 1.476,
      "nll_loss": 1.4613220691680908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14179985225200653,
      "rewards/margins": 0.1631615310907364,
      "rewards/rejected": -0.3049613833427429,
      "step": 1083
    },
    {
      "epoch": 2.988608905764584,
      "grad_norm": 0.21875019371509552,
      "learning_rate": 4.624307640249681e-10,
      "log_odds_chosen": 1.664400339126587,
      "log_odds_ratio": -0.18604637682437897,
      "logits/chosen": -0.8599823117256165,
      "logits/rejected": -2.165078639984131,
      "logps/chosen": -1.4403799772262573,
      "logps/rejected": -2.8875458240509033,
      "loss": 1.4962,
      "nll_loss": 1.4775702953338623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14403800666332245,
      "rewards/margins": 0.14471659064292908,
      "rewards/rejected": -0.2887546122074127,
      "step": 1084
    },
    {
      "epoch": 2.991370383154988,
      "grad_norm": 0.21976594626903534,
      "learning_rate": 3.2113550017198734e-10,
      "log_odds_chosen": 1.7429512739181519,
      "log_odds_ratio": -0.17201046645641327,
      "logits/chosen": -0.8370999097824097,
      "logits/rejected": -2.065781831741333,
      "logps/chosen": -1.3308002948760986,
      "logps/rejected": -2.826251268386841,
      "loss": 1.4156,
      "nll_loss": 1.3984227180480957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13308003544807434,
      "rewards/margins": 0.1495451182126999,
      "rewards/rejected": -0.28262513875961304,
      "step": 1085
    },
    {
      "epoch": 2.9941318605453917,
      "grad_norm": 0.22135642170906067,
      "learning_rate": 2.0552830420184077e-10,
      "log_odds_chosen": 1.7063344717025757,
      "log_odds_ratio": -0.2038329541683197,
      "logits/chosen": -0.7838236093521118,
      "logits/rejected": -1.8901429176330566,
      "logps/chosen": -1.2961207628250122,
      "logps/rejected": -2.761850357055664,
      "loss": 1.3611,
      "nll_loss": 1.34067964553833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12961207330226898,
      "rewards/margins": 0.14657296240329742,
      "rewards/rejected": -0.2761850357055664,
      "step": 1086
    },
    {
      "epoch": 2.996893337935796,
      "grad_norm": 0.21547023952007294,
      "learning_rate": 1.1561036415752568e-10,
      "log_odds_chosen": 1.8393921852111816,
      "log_odds_ratio": -0.16761940717697144,
      "logits/chosen": -0.7874873280525208,
      "logits/rejected": -1.8532605171203613,
      "logps/chosen": -1.335709571838379,
      "logps/rejected": -2.9297051429748535,
      "loss": 1.4178,
      "nll_loss": 1.4010545015335083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13357096910476685,
      "rewards/margins": 0.15939952433109283,
      "rewards/rejected": -0.2929704785346985,
      "step": 1087
    },
    {
      "epoch": 2.9996548153261995,
      "grad_norm": 0.21358518302440643,
      "learning_rate": 5.138260408488194e-11,
      "log_odds_chosen": 1.7693160772323608,
      "log_odds_ratio": -0.1748848706483841,
      "logits/chosen": -0.7919652462005615,
      "logits/rejected": -2.1071107387542725,
      "logps/chosen": -1.4612958431243896,
      "logps/rejected": -3.023159980773926,
      "loss": 1.5195,
      "nll_loss": 1.501983642578125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461295783519745,
      "rewards/margins": 0.15618643164634705,
      "rewards/rejected": -0.30231598019599915,
      "step": 1088
    },
    {
      "epoch": 3.0,
      "grad_norm": 0.5550550222396851,
      "learning_rate": 1.2845684023432559e-11,
      "log_odds_chosen": 2.2474207878112793,
      "log_odds_ratio": -0.10045277327299118,
      "logits/chosen": -0.9216854572296143,
      "logits/rejected": -1.9990668296813965,
      "logps/chosen": -1.146590232849121,
      "logps/rejected": -3.0597341060638428,
      "loss": 1.224,
      "nll_loss": 1.2139885425567627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11465902626514435,
      "rewards/margins": 0.19131438434123993,
      "rewards/rejected": -0.3059734106063843,
      "step": 1089
    }
  ],
  "logging_steps": 1,
  "max_steps": 1089,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}