{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 1270,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015760441292356187,
      "grad_norm": 0.8194138407707214,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.35938164591789246,
      "log_odds_ratio": -0.5514221787452698,
      "logits/chosen": -0.6073167324066162,
      "logits/rejected": 0.20061472058296204,
      "logps/chosen": -2.176460027694702,
      "logps/rejected": -2.502065420150757,
      "loss": 2.4622,
      "nll_loss": 2.407036304473877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21764598786830902,
      "rewards/margins": 0.03256051242351532,
      "rewards/rejected": -0.25020653009414673,
      "step": 1
    },
    {
      "epoch": 0.0031520882584712374,
      "grad_norm": 0.7936525940895081,
      "learning_rate": 3.1496062992125985e-08,
      "log_odds_chosen": 0.35638517141342163,
      "log_odds_ratio": -0.5405049324035645,
      "logits/chosen": -0.6276549100875854,
      "logits/rejected": 0.3162096440792084,
      "logps/chosen": -2.069308280944824,
      "logps/rejected": -2.385652780532837,
      "loss": 2.343,
      "nll_loss": 2.288930892944336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20693084597587585,
      "rewards/margins": 0.031634461134672165,
      "rewards/rejected": -0.23856529593467712,
      "step": 2
    },
    {
      "epoch": 0.004728132387706856,
      "grad_norm": 0.9091457724571228,
      "learning_rate": 6.299212598425197e-08,
      "log_odds_chosen": 0.7094058394432068,
      "log_odds_ratio": -0.4360072910785675,
      "logits/chosen": -0.7982139587402344,
      "logits/rejected": 0.22547274827957153,
      "logps/chosen": -2.2025773525238037,
      "logps/rejected": -2.8602232933044434,
      "loss": 2.4665,
      "nll_loss": 2.4228932857513428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2202577441930771,
      "rewards/margins": 0.06576459109783173,
      "rewards/rejected": -0.2860223054885864,
      "step": 3
    },
    {
      "epoch": 0.006304176516942475,
      "grad_norm": 0.7839515209197998,
      "learning_rate": 9.448818897637795e-08,
      "log_odds_chosen": 0.6190818548202515,
      "log_odds_ratio": -0.45430099964141846,
      "logits/chosen": -0.5776420831680298,
      "logits/rejected": 0.3159808814525604,
      "logps/chosen": -2.197322130203247,
      "logps/rejected": -2.7688748836517334,
      "loss": 2.4613,
      "nll_loss": 2.4159181118011475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21973221004009247,
      "rewards/margins": 0.057155244052410126,
      "rewards/rejected": -0.2768874764442444,
      "step": 4
    },
    {
      "epoch": 0.007880220646178092,
      "grad_norm": 0.8797410726547241,
      "learning_rate": 1.2598425196850394e-07,
      "log_odds_chosen": 0.6505839228630066,
      "log_odds_ratio": -0.43997013568878174,
      "logits/chosen": -0.6981326937675476,
      "logits/rejected": 0.13587594032287598,
      "logps/chosen": -2.136472702026367,
      "logps/rejected": -2.7251079082489014,
      "loss": 2.395,
      "nll_loss": 2.3510234355926514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2136472761631012,
      "rewards/margins": 0.05886353179812431,
      "rewards/rejected": -0.272510826587677,
      "step": 5
    },
    {
      "epoch": 0.009456264775413711,
      "grad_norm": 0.8133273720741272,
      "learning_rate": 1.5748031496062992e-07,
      "log_odds_chosen": 0.39265337586402893,
      "log_odds_ratio": -0.5214492678642273,
      "logits/chosen": -0.5345965027809143,
      "logits/rejected": 0.3466244637966156,
      "logps/chosen": -2.0630874633789062,
      "logps/rejected": -2.4150893688201904,
      "loss": 2.3692,
      "nll_loss": 2.317031145095825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2063087522983551,
      "rewards/margins": 0.035200200974941254,
      "rewards/rejected": -0.24150894582271576,
      "step": 6
    },
    {
      "epoch": 0.01103230890464933,
      "grad_norm": 0.7818942666053772,
      "learning_rate": 1.889763779527559e-07,
      "log_odds_chosen": 0.7252346277236938,
      "log_odds_ratio": -0.4240948557853699,
      "logits/chosen": -0.5406574606895447,
      "logits/rejected": 0.29844143986701965,
      "logps/chosen": -2.129410982131958,
      "logps/rejected": -2.7971620559692383,
      "loss": 2.3917,
      "nll_loss": 2.34930419921875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21294109523296356,
      "rewards/margins": 0.06677509844303131,
      "rewards/rejected": -0.2797161936759949,
      "step": 7
    },
    {
      "epoch": 0.01260835303388495,
      "grad_norm": 1.0168143510818481,
      "learning_rate": 2.2047244094488187e-07,
      "log_odds_chosen": 0.9424365162849426,
      "log_odds_ratio": -0.34798142313957214,
      "logits/chosen": -0.6615124344825745,
      "logits/rejected": 0.22647148370742798,
      "logps/chosen": -2.2724151611328125,
      "logps/rejected": -3.144073963165283,
      "loss": 2.5346,
      "nll_loss": 2.4997568130493164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22724151611328125,
      "rewards/margins": 0.08716587722301483,
      "rewards/rejected": -0.3144074082374573,
      "step": 8
    },
    {
      "epoch": 0.014184397163120567,
      "grad_norm": 0.7796529531478882,
      "learning_rate": 2.519685039370079e-07,
      "log_odds_chosen": 0.4133250415325165,
      "log_odds_ratio": -0.5154058337211609,
      "logits/chosen": -0.5772379636764526,
      "logits/rejected": 0.26968488097190857,
      "logps/chosen": -2.124163866043091,
      "logps/rejected": -2.496925115585327,
      "loss": 2.3769,
      "nll_loss": 2.325348377227783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2124163955450058,
      "rewards/margins": 0.03727611154317856,
      "rewards/rejected": -0.24969248473644257,
      "step": 9
    },
    {
      "epoch": 0.015760441292356184,
      "grad_norm": 0.8118637800216675,
      "learning_rate": 2.8346456692913386e-07,
      "log_odds_chosen": 0.4085013270378113,
      "log_odds_ratio": -0.5165191292762756,
      "logits/chosen": -0.6135594844818115,
      "logits/rejected": 0.11062480509281158,
      "logps/chosen": -2.1375293731689453,
      "logps/rejected": -2.5072574615478516,
      "loss": 2.4048,
      "nll_loss": 2.3531653881073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21375294029712677,
      "rewards/margins": 0.03697281330823898,
      "rewards/rejected": -0.25072574615478516,
      "step": 10
    },
    {
      "epoch": 0.017336485421591805,
      "grad_norm": 0.8751649856567383,
      "learning_rate": 3.1496062992125984e-07,
      "log_odds_chosen": 0.4661106765270233,
      "log_odds_ratio": -0.49116337299346924,
      "logits/chosen": -0.5825808644294739,
      "logits/rejected": 0.18361972272396088,
      "logps/chosen": -2.239102840423584,
      "logps/rejected": -2.662409782409668,
      "loss": 2.5002,
      "nll_loss": 2.451051950454712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22391027212142944,
      "rewards/margins": 0.042330723255872726,
      "rewards/rejected": -0.2662409842014313,
      "step": 11
    },
    {
      "epoch": 0.018912529550827423,
      "grad_norm": 1.0198794603347778,
      "learning_rate": 3.464566929133858e-07,
      "log_odds_chosen": 0.6043168902397156,
      "log_odds_ratio": -0.4442984163761139,
      "logits/chosen": -0.756458580493927,
      "logits/rejected": 0.36159712076187134,
      "logps/chosen": -2.3110899925231934,
      "logps/rejected": -2.8696181774139404,
      "loss": 2.5785,
      "nll_loss": 2.5340256690979004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23110897839069366,
      "rewards/margins": 0.055852800607681274,
      "rewards/rejected": -0.2869618237018585,
      "step": 12
    },
    {
      "epoch": 0.02048857368006304,
      "grad_norm": 0.9937220215797424,
      "learning_rate": 3.779527559055118e-07,
      "log_odds_chosen": 0.543169379234314,
      "log_odds_ratio": -0.48258134722709656,
      "logits/chosen": -0.8145802021026611,
      "logits/rejected": 0.4112021327018738,
      "logps/chosen": -2.206066608428955,
      "logps/rejected": -2.700916290283203,
      "loss": 2.4702,
      "nll_loss": 2.421976089477539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22060665488243103,
      "rewards/margins": 0.0494849756360054,
      "rewards/rejected": -0.27009162306785583,
      "step": 13
    },
    {
      "epoch": 0.02206461780929866,
      "grad_norm": 0.8126177787780762,
      "learning_rate": 4.0944881889763777e-07,
      "log_odds_chosen": 0.5306293368339539,
      "log_odds_ratio": -0.5026512145996094,
      "logits/chosen": -0.6288756728172302,
      "logits/rejected": 0.034559354186058044,
      "logps/chosen": -2.1632676124572754,
      "logps/rejected": -2.653379440307617,
      "loss": 2.425,
      "nll_loss": 2.3747172355651855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2163267731666565,
      "rewards/margins": 0.04901116341352463,
      "rewards/rejected": -0.2653379440307617,
      "step": 14
    },
    {
      "epoch": 0.02364066193853428,
      "grad_norm": 0.9252188801765442,
      "learning_rate": 4.4094488188976375e-07,
      "log_odds_chosen": 0.4157894253730774,
      "log_odds_ratio": -0.5104730129241943,
      "logits/chosen": -0.7391470074653625,
      "logits/rejected": 0.21298080682754517,
      "logps/chosen": -2.185275077819824,
      "logps/rejected": -2.563283920288086,
      "loss": 2.4617,
      "nll_loss": 2.4106781482696533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21852751076221466,
      "rewards/margins": 0.03780088573694229,
      "rewards/rejected": -0.25632840394973755,
      "step": 15
    },
    {
      "epoch": 0.0252167060677699,
      "grad_norm": 0.9326465129852295,
      "learning_rate": 4.7244094488188973e-07,
      "log_odds_chosen": 0.42695683240890503,
      "log_odds_ratio": -0.5144686102867126,
      "logits/chosen": -0.7023543119430542,
      "logits/rejected": 0.20582355558872223,
      "logps/chosen": -2.171863079071045,
      "logps/rejected": -2.559831380844116,
      "loss": 2.4416,
      "nll_loss": 2.3901824951171875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2171863466501236,
      "rewards/margins": 0.0387967973947525,
      "rewards/rejected": -0.2559831440448761,
      "step": 16
    },
    {
      "epoch": 0.026792750197005517,
      "grad_norm": 0.7562754154205322,
      "learning_rate": 5.039370078740158e-07,
      "log_odds_chosen": 0.6844577789306641,
      "log_odds_ratio": -0.42741507291793823,
      "logits/chosen": -0.5460200309753418,
      "logits/rejected": 0.18708395957946777,
      "logps/chosen": -2.098813056945801,
      "logps/rejected": -2.723797082901001,
      "loss": 2.3725,
      "nll_loss": 2.3297088146209717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20988130569458008,
      "rewards/margins": 0.062498390674591064,
      "rewards/rejected": -0.27237969636917114,
      "step": 17
    },
    {
      "epoch": 0.028368794326241134,
      "grad_norm": 0.8840509653091431,
      "learning_rate": 5.354330708661418e-07,
      "log_odds_chosen": 0.3034907281398773,
      "log_odds_ratio": -0.5711926221847534,
      "logits/chosen": -0.6161488890647888,
      "logits/rejected": 0.47962120175361633,
      "logps/chosen": -2.1487925052642822,
      "logps/rejected": -2.4217610359191895,
      "loss": 2.4355,
      "nll_loss": 2.3784008026123047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21487924456596375,
      "rewards/margins": 0.0272968839854002,
      "rewards/rejected": -0.2421761304140091,
      "step": 18
    },
    {
      "epoch": 0.029944838455476755,
      "grad_norm": 0.8342726230621338,
      "learning_rate": 5.669291338582677e-07,
      "log_odds_chosen": 0.6132441163063049,
      "log_odds_ratio": -0.4440116584300995,
      "logits/chosen": -0.7522751688957214,
      "logits/rejected": 0.13862034678459167,
      "logps/chosen": -2.0390007495880127,
      "logps/rejected": -2.5903594493865967,
      "loss": 2.3287,
      "nll_loss": 2.2843337059020996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20390008389949799,
      "rewards/margins": 0.055135857313871384,
      "rewards/rejected": -0.25903594493865967,
      "step": 19
    },
    {
      "epoch": 0.03152088258471237,
      "grad_norm": 0.8772868514060974,
      "learning_rate": 5.984251968503937e-07,
      "log_odds_chosen": 0.6135118007659912,
      "log_odds_ratio": -0.460088849067688,
      "logits/chosen": -0.5715646147727966,
      "logits/rejected": 0.18610909581184387,
      "logps/chosen": -2.1998162269592285,
      "logps/rejected": -2.76267147064209,
      "loss": 2.483,
      "nll_loss": 2.436969757080078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2199816256761551,
      "rewards/margins": 0.05628550797700882,
      "rewards/rejected": -0.2762671411037445,
      "step": 20
    },
    {
      "epoch": 0.03309692671394799,
      "grad_norm": 0.9545782208442688,
      "learning_rate": 6.299212598425197e-07,
      "log_odds_chosen": 0.3836444914340973,
      "log_odds_ratio": -0.5285393595695496,
      "logits/chosen": -0.6870481371879578,
      "logits/rejected": 0.3343619704246521,
      "logps/chosen": -2.276167154312134,
      "logps/rejected": -2.6286871433258057,
      "loss": 2.5437,
      "nll_loss": 2.490856885910034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22761669754981995,
      "rewards/margins": 0.03525201976299286,
      "rewards/rejected": -0.262868732213974,
      "step": 21
    },
    {
      "epoch": 0.03467297084318361,
      "grad_norm": 0.8653661012649536,
      "learning_rate": 6.614173228346457e-07,
      "log_odds_chosen": 0.3887983560562134,
      "log_odds_ratio": -0.5376300811767578,
      "logits/chosen": -0.7233757972717285,
      "logits/rejected": 0.4169387221336365,
      "logps/chosen": -2.1862027645111084,
      "logps/rejected": -2.5423219203948975,
      "loss": 2.4644,
      "nll_loss": 2.410614490509033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21862027049064636,
      "rewards/margins": 0.03561191260814667,
      "rewards/rejected": -0.25423216819763184,
      "step": 22
    },
    {
      "epoch": 0.036249014972419225,
      "grad_norm": 0.8709145784378052,
      "learning_rate": 6.929133858267716e-07,
      "log_odds_chosen": 0.4194765090942383,
      "log_odds_ratio": -0.5103728771209717,
      "logits/chosen": -0.7145715355873108,
      "logits/rejected": 0.29370206594467163,
      "logps/chosen": -2.231127977371216,
      "logps/rejected": -2.613429307937622,
      "loss": 2.4803,
      "nll_loss": 2.429227352142334,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2231128066778183,
      "rewards/margins": 0.03823015093803406,
      "rewards/rejected": -0.2613429129123688,
      "step": 23
    },
    {
      "epoch": 0.037825059101654845,
      "grad_norm": 0.8538913130760193,
      "learning_rate": 7.244094488188977e-07,
      "log_odds_chosen": 0.5760198831558228,
      "log_odds_ratio": -0.4734610915184021,
      "logits/chosen": -0.41316404938697815,
      "logits/rejected": 0.3446556031703949,
      "logps/chosen": -2.180354595184326,
      "logps/rejected": -2.710923671722412,
      "loss": 2.4467,
      "nll_loss": 2.3993289470672607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21803542971611023,
      "rewards/margins": 0.053056925535202026,
      "rewards/rejected": -0.27109238505363464,
      "step": 24
    },
    {
      "epoch": 0.039401103230890466,
      "grad_norm": 0.9757054448127747,
      "learning_rate": 7.559055118110236e-07,
      "log_odds_chosen": 0.35894954204559326,
      "log_odds_ratio": -0.5444148778915405,
      "logits/chosen": -0.7573986053466797,
      "logits/rejected": 0.08482818305492401,
      "logps/chosen": -2.2291455268859863,
      "logps/rejected": -2.5626137256622314,
      "loss": 2.4993,
      "nll_loss": 2.4448294639587402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22291454672813416,
      "rewards/margins": 0.033346809446811676,
      "rewards/rejected": -0.25626134872436523,
      "step": 25
    },
    {
      "epoch": 0.04097714736012608,
      "grad_norm": 0.9316413998603821,
      "learning_rate": 7.874015748031496e-07,
      "log_odds_chosen": 0.4797845780849457,
      "log_odds_ratio": -0.4976828992366791,
      "logits/chosen": -0.8583399653434753,
      "logits/rejected": 0.1226923018693924,
      "logps/chosen": -2.17708420753479,
      "logps/rejected": -2.617652177810669,
      "loss": 2.4456,
      "nll_loss": 2.3958029747009277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21770845353603363,
      "rewards/margins": 0.044056810438632965,
      "rewards/rejected": -0.2617652416229248,
      "step": 26
    },
    {
      "epoch": 0.0425531914893617,
      "grad_norm": 0.8912092447280884,
      "learning_rate": 8.188976377952755e-07,
      "log_odds_chosen": 0.8782732486724854,
      "log_odds_ratio": -0.38095948100090027,
      "logits/chosen": -0.6175351142883301,
      "logits/rejected": 0.3754832446575165,
      "logps/chosen": -2.209217071533203,
      "logps/rejected": -3.024287223815918,
      "loss": 2.4801,
      "nll_loss": 2.4420361518859863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22092171013355255,
      "rewards/margins": 0.08150701969861984,
      "rewards/rejected": -0.3024287223815918,
      "step": 27
    },
    {
      "epoch": 0.04412923561859732,
      "grad_norm": 0.9370722770690918,
      "learning_rate": 8.503937007874016e-07,
      "log_odds_chosen": 0.38346901535987854,
      "log_odds_ratio": -0.5323441624641418,
      "logits/chosen": -0.6895672082901001,
      "logits/rejected": 0.3445851802825928,
      "logps/chosen": -2.1683366298675537,
      "logps/rejected": -2.516519546508789,
      "loss": 2.4565,
      "nll_loss": 2.4032399654388428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2168336808681488,
      "rewards/margins": 0.03481828421354294,
      "rewards/rejected": -0.25165194272994995,
      "step": 28
    },
    {
      "epoch": 0.045705279747832936,
      "grad_norm": 0.8310018181800842,
      "learning_rate": 8.818897637795275e-07,
      "log_odds_chosen": 0.3637988865375519,
      "log_odds_ratio": -0.5361331105232239,
      "logits/chosen": -0.5572834610939026,
      "logits/rejected": 0.25494274497032166,
      "logps/chosen": -2.240579843521118,
      "logps/rejected": -2.57593035697937,
      "loss": 2.5104,
      "nll_loss": 2.4568018913269043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22405798733234406,
      "rewards/margins": 0.03353505954146385,
      "rewards/rejected": -0.257593035697937,
      "step": 29
    },
    {
      "epoch": 0.04728132387706856,
      "grad_norm": 1.0258733034133911,
      "learning_rate": 9.133858267716535e-07,
      "log_odds_chosen": 0.6511096954345703,
      "log_odds_ratio": -0.4371214210987091,
      "logits/chosen": -0.6823744177818298,
      "logits/rejected": 0.13312608003616333,
      "logps/chosen": -2.2421658039093018,
      "logps/rejected": -2.842336654663086,
      "loss": 2.5426,
      "nll_loss": 2.49886155128479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22421656548976898,
      "rewards/margins": 0.06001708656549454,
      "rewards/rejected": -0.28423362970352173,
      "step": 30
    },
    {
      "epoch": 0.04885736800630418,
      "grad_norm": 0.9316391944885254,
      "learning_rate": 9.448818897637795e-07,
      "log_odds_chosen": 0.6505148410797119,
      "log_odds_ratio": -0.4398355185985565,
      "logits/chosen": -0.631649911403656,
      "logits/rejected": 0.41568225622177124,
      "logps/chosen": -2.274989604949951,
      "logps/rejected": -2.876878261566162,
      "loss": 2.5297,
      "nll_loss": 2.485715389251709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22749897837638855,
      "rewards/margins": 0.06018887460231781,
      "rewards/rejected": -0.28768783807754517,
      "step": 31
    },
    {
      "epoch": 0.0504334121355398,
      "grad_norm": 0.7761370539665222,
      "learning_rate": 9.763779527559055e-07,
      "log_odds_chosen": 0.8358607292175293,
      "log_odds_ratio": -0.3823116719722748,
      "logits/chosen": -0.5432754158973694,
      "logits/rejected": 0.14401675760746002,
      "logps/chosen": -2.1370320320129395,
      "logps/rejected": -2.906792402267456,
      "loss": 2.3966,
      "nll_loss": 2.3583412170410156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2137032151222229,
      "rewards/margins": 0.07697603851556778,
      "rewards/rejected": -0.2906792461872101,
      "step": 32
    },
    {
      "epoch": 0.05200945626477541,
      "grad_norm": 0.7844627499580383,
      "learning_rate": 1.0078740157480315e-06,
      "log_odds_chosen": 0.6228745579719543,
      "log_odds_ratio": -0.4355998933315277,
      "logits/chosen": -0.5304853320121765,
      "logits/rejected": 0.25808629393577576,
      "logps/chosen": -2.138329029083252,
      "logps/rejected": -2.7067372798919678,
      "loss": 2.3954,
      "nll_loss": 2.351879119873047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21383287012577057,
      "rewards/margins": 0.056840844452381134,
      "rewards/rejected": -0.2706737220287323,
      "step": 33
    },
    {
      "epoch": 0.05358550039401103,
      "grad_norm": 0.8664770126342773,
      "learning_rate": 1.0393700787401573e-06,
      "log_odds_chosen": 0.3209899663925171,
      "log_odds_ratio": -0.5560799837112427,
      "logits/chosen": -0.529889702796936,
      "logits/rejected": 0.25037455558776855,
      "logps/chosen": -2.1071910858154297,
      "logps/rejected": -2.391819715499878,
      "loss": 2.3922,
      "nll_loss": 2.3365979194641113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21071910858154297,
      "rewards/margins": 0.028462877497076988,
      "rewards/rejected": -0.2391819953918457,
      "step": 34
    },
    {
      "epoch": 0.055161544523246654,
      "grad_norm": 1.0934066772460938,
      "learning_rate": 1.0708661417322836e-06,
      "log_odds_chosen": 0.39015841484069824,
      "log_odds_ratio": -0.5191880464553833,
      "logits/chosen": -0.7120888233184814,
      "logits/rejected": 0.5324327945709229,
      "logps/chosen": -2.3131232261657715,
      "logps/rejected": -2.6696863174438477,
      "loss": 2.587,
      "nll_loss": 2.535099506378174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2313123196363449,
      "rewards/margins": 0.03565630316734314,
      "rewards/rejected": -0.26696863770484924,
      "step": 35
    },
    {
      "epoch": 0.05673758865248227,
      "grad_norm": 0.9424160122871399,
      "learning_rate": 1.1023622047244094e-06,
      "log_odds_chosen": 0.6048139333724976,
      "log_odds_ratio": -0.4379541575908661,
      "logits/chosen": -0.8271058201789856,
      "logits/rejected": 0.32734885811805725,
      "logps/chosen": -2.1414098739624023,
      "logps/rejected": -2.6914708614349365,
      "loss": 2.4076,
      "nll_loss": 2.3637936115264893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21414096653461456,
      "rewards/margins": 0.055006109178066254,
      "rewards/rejected": -0.2691470980644226,
      "step": 36
    },
    {
      "epoch": 0.05831363278171789,
      "grad_norm": 0.8267931342124939,
      "learning_rate": 1.1338582677165354e-06,
      "log_odds_chosen": 0.511355996131897,
      "log_odds_ratio": -0.48152798414230347,
      "logits/chosen": -0.6461195945739746,
      "logits/rejected": 0.38446712493896484,
      "logps/chosen": -2.110015392303467,
      "logps/rejected": -2.5689022541046143,
      "loss": 2.3946,
      "nll_loss": 2.346407651901245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21100151538848877,
      "rewards/margins": 0.04588867723941803,
      "rewards/rejected": -0.256890207529068,
      "step": 37
    },
    {
      "epoch": 0.05988967691095351,
      "grad_norm": 0.9441820383071899,
      "learning_rate": 1.1653543307086612e-06,
      "log_odds_chosen": 0.4044821858406067,
      "log_odds_ratio": -0.5209547281265259,
      "logits/chosen": -0.7661936283111572,
      "logits/rejected": 0.18499918282032013,
      "logps/chosen": -2.1512036323547363,
      "logps/rejected": -2.5139970779418945,
      "loss": 2.4393,
      "nll_loss": 2.3872311115264893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2151203751564026,
      "rewards/margins": 0.036279335618019104,
      "rewards/rejected": -0.2513996958732605,
      "step": 38
    },
    {
      "epoch": 0.061465721040189124,
      "grad_norm": 1.0256890058517456,
      "learning_rate": 1.1968503937007875e-06,
      "log_odds_chosen": 0.16465213894844055,
      "log_odds_ratio": -0.6248909831047058,
      "logits/chosen": -0.6871511340141296,
      "logits/rejected": 0.38549405336380005,
      "logps/chosen": -2.2681920528411865,
      "logps/rejected": -2.4200401306152344,
      "loss": 2.5632,
      "nll_loss": 2.5007407665252686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2268192023038864,
      "rewards/margins": 0.015184784308075905,
      "rewards/rejected": -0.24200400710105896,
      "step": 39
    },
    {
      "epoch": 0.06304176516942474,
      "grad_norm": 0.8143957853317261,
      "learning_rate": 1.2283464566929133e-06,
      "log_odds_chosen": 0.38844311237335205,
      "log_odds_ratio": -0.5253883004188538,
      "logits/chosen": -0.675887405872345,
      "logits/rejected": 0.2226046621799469,
      "logps/chosen": -2.0888419151306152,
      "logps/rejected": -2.4366869926452637,
      "loss": 2.3744,
      "nll_loss": 2.321840763092041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20888419449329376,
      "rewards/margins": 0.034784507006406784,
      "rewards/rejected": -0.24366869032382965,
      "step": 40
    },
    {
      "epoch": 0.06461780929866036,
      "grad_norm": 0.8119162917137146,
      "learning_rate": 1.2598425196850393e-06,
      "log_odds_chosen": 0.20214533805847168,
      "log_odds_ratio": -0.6077091693878174,
      "logits/chosen": -0.6068657636642456,
      "logits/rejected": 0.30688706040382385,
      "logps/chosen": -2.137021780014038,
      "logps/rejected": -2.3217053413391113,
      "loss": 2.4076,
      "nll_loss": 2.3468475341796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21370220184326172,
      "rewards/margins": 0.018468374386429787,
      "rewards/rejected": -0.23217056691646576,
      "step": 41
    },
    {
      "epoch": 0.06619385342789598,
      "grad_norm": 0.8659912943840027,
      "learning_rate": 1.2913385826771652e-06,
      "log_odds_chosen": 0.29553914070129395,
      "log_odds_ratio": -0.5623306035995483,
      "logits/chosen": -0.7116361260414124,
      "logits/rejected": 0.290731281042099,
      "logps/chosen": -2.154006004333496,
      "logps/rejected": -2.42051362991333,
      "loss": 2.4304,
      "nll_loss": 2.3741416931152344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21540062129497528,
      "rewards/margins": 0.02665073424577713,
      "rewards/rejected": -0.24205134809017181,
      "step": 42
    },
    {
      "epoch": 0.0677698975571316,
      "grad_norm": 0.8237044215202332,
      "learning_rate": 1.3228346456692914e-06,
      "log_odds_chosen": 0.16473819315433502,
      "log_odds_ratio": -0.6206480264663696,
      "logits/chosen": -0.5807620286941528,
      "logits/rejected": 0.22030681371688843,
      "logps/chosen": -2.1601130962371826,
      "logps/rejected": -2.30804705619812,
      "loss": 2.4377,
      "nll_loss": 2.3756585121154785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21601131558418274,
      "rewards/margins": 0.014793392270803452,
      "rewards/rejected": -0.2308046966791153,
      "step": 43
    },
    {
      "epoch": 0.06934594168636722,
      "grad_norm": 0.9198518991470337,
      "learning_rate": 1.3543307086614172e-06,
      "log_odds_chosen": 0.5057801008224487,
      "log_odds_ratio": -0.486497700214386,
      "logits/chosen": -0.7227851152420044,
      "logits/rejected": 0.4384616017341614,
      "logps/chosen": -2.154522657394409,
      "logps/rejected": -2.6166176795959473,
      "loss": 2.4283,
      "nll_loss": 2.3796355724334717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21545226871967316,
      "rewards/margins": 0.04620949178934097,
      "rewards/rejected": -0.2616617679595947,
      "step": 44
    },
    {
      "epoch": 0.07092198581560284,
      "grad_norm": 0.7992345094680786,
      "learning_rate": 1.3858267716535433e-06,
      "log_odds_chosen": 0.6110447645187378,
      "log_odds_ratio": -0.4478919804096222,
      "logits/chosen": -0.5170164108276367,
      "logits/rejected": 0.38270628452301025,
      "logps/chosen": -2.089824676513672,
      "logps/rejected": -2.644026756286621,
      "loss": 2.3858,
      "nll_loss": 2.3409948348999023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20898248255252838,
      "rewards/margins": 0.05542018264532089,
      "rewards/rejected": -0.26440268754959106,
      "step": 45
    },
    {
      "epoch": 0.07249802994483845,
      "grad_norm": 1.0367867946624756,
      "learning_rate": 1.417322834645669e-06,
      "log_odds_chosen": 0.6249645352363586,
      "log_odds_ratio": -0.4582594037055969,
      "logits/chosen": -0.7600032687187195,
      "logits/rejected": 0.12511876225471497,
      "logps/chosen": -2.15745210647583,
      "logps/rejected": -2.7306129932403564,
      "loss": 2.4426,
      "nll_loss": 2.396728992462158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.215745210647583,
      "rewards/margins": 0.05731611326336861,
      "rewards/rejected": -0.2730613350868225,
      "step": 46
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 0.8897382020950317,
      "learning_rate": 1.4488188976377953e-06,
      "log_odds_chosen": 0.3489275276660919,
      "log_odds_ratio": -0.5626947283744812,
      "logits/chosen": -0.8047929406166077,
      "logits/rejected": 0.2395666539669037,
      "logps/chosen": -2.175475835800171,
      "logps/rejected": -2.4979097843170166,
      "loss": 2.4499,
      "nll_loss": 2.3936004638671875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21754758059978485,
      "rewards/margins": 0.032243404537439346,
      "rewards/rejected": -0.2497909665107727,
      "step": 47
    },
    {
      "epoch": 0.07565011820330969,
      "grad_norm": 1.0149489641189575,
      "learning_rate": 1.4803149606299211e-06,
      "log_odds_chosen": 0.43032294511795044,
      "log_odds_ratio": -0.5186169147491455,
      "logits/chosen": -0.6099531054496765,
      "logits/rejected": 0.2379540652036667,
      "logps/chosen": -2.2526159286499023,
      "logps/rejected": -2.6506638526916504,
      "loss": 2.5361,
      "nll_loss": 2.4842634201049805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2252615988254547,
      "rewards/margins": 0.03980477899312973,
      "rewards/rejected": -0.26506638526916504,
      "step": 48
    },
    {
      "epoch": 0.07722616233254531,
      "grad_norm": 0.8126904368400574,
      "learning_rate": 1.5118110236220472e-06,
      "log_odds_chosen": 0.5315326452255249,
      "log_odds_ratio": -0.47535181045532227,
      "logits/chosen": -0.6085718870162964,
      "logits/rejected": 0.533661425113678,
      "logps/chosen": -2.1099607944488525,
      "logps/rejected": -2.5920519828796387,
      "loss": 2.355,
      "nll_loss": 2.3075127601623535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2109961062669754,
      "rewards/margins": 0.04820908233523369,
      "rewards/rejected": -0.2592051923274994,
      "step": 49
    },
    {
      "epoch": 0.07880220646178093,
      "grad_norm": 0.8765050768852234,
      "learning_rate": 1.543307086614173e-06,
      "log_odds_chosen": 0.32944872975349426,
      "log_odds_ratio": -0.5625548362731934,
      "logits/chosen": -0.6740020513534546,
      "logits/rejected": 0.2358410358428955,
      "logps/chosen": -2.1824560165405273,
      "logps/rejected": -2.4877140522003174,
      "loss": 2.4638,
      "nll_loss": 2.407498836517334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21824561059474945,
      "rewards/margins": 0.030525820329785347,
      "rewards/rejected": -0.24877141416072845,
      "step": 50
    },
    {
      "epoch": 0.08037825059101655,
      "grad_norm": 0.8369614481925964,
      "learning_rate": 1.5748031496062992e-06,
      "log_odds_chosen": 0.6268947720527649,
      "log_odds_ratio": -0.4370884597301483,
      "logits/chosen": -0.5486613512039185,
      "logits/rejected": 0.3741488754749298,
      "logps/chosen": -2.047851800918579,
      "logps/rejected": -2.6127448081970215,
      "loss": 2.3196,
      "nll_loss": 2.2759056091308594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20478516817092896,
      "rewards/margins": 0.05648929998278618,
      "rewards/rejected": -0.2612744867801666,
      "step": 51
    },
    {
      "epoch": 0.08195429472025216,
      "grad_norm": 0.848452627658844,
      "learning_rate": 1.6062992125984253e-06,
      "log_odds_chosen": 0.5586876273155212,
      "log_odds_ratio": -0.46114546060562134,
      "logits/chosen": -0.4986473619937897,
      "logits/rejected": 0.39973095059394836,
      "logps/chosen": -2.234095335006714,
      "logps/rejected": -2.7459263801574707,
      "loss": 2.4913,
      "nll_loss": 2.445152997970581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22340954840183258,
      "rewards/margins": 0.05118311941623688,
      "rewards/rejected": -0.27459266781806946,
      "step": 52
    },
    {
      "epoch": 0.08353033884948778,
      "grad_norm": 0.8976253271102905,
      "learning_rate": 1.637795275590551e-06,
      "log_odds_chosen": 0.31935352087020874,
      "log_odds_ratio": -0.5499067902565002,
      "logits/chosen": -0.5869264602661133,
      "logits/rejected": 0.4152706563472748,
      "logps/chosen": -2.1620757579803467,
      "logps/rejected": -2.449148654937744,
      "loss": 2.4373,
      "nll_loss": 2.38226580619812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2162075787782669,
      "rewards/margins": 0.028707284480333328,
      "rewards/rejected": -0.24491485953330994,
      "step": 53
    },
    {
      "epoch": 0.0851063829787234,
      "grad_norm": 0.8512571454048157,
      "learning_rate": 1.6692913385826771e-06,
      "log_odds_chosen": 0.288684606552124,
      "log_odds_ratio": -0.5614148378372192,
      "logits/chosen": -0.6818708181381226,
      "logits/rejected": 0.2810644507408142,
      "logps/chosen": -2.0992839336395264,
      "logps/rejected": -2.356931209564209,
      "loss": 2.3834,
      "nll_loss": 2.327263832092285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20992840826511383,
      "rewards/margins": 0.02576470375061035,
      "rewards/rejected": -0.23569311201572418,
      "step": 54
    },
    {
      "epoch": 0.08668242710795902,
      "grad_norm": 0.939967930316925,
      "learning_rate": 1.7007874015748031e-06,
      "log_odds_chosen": 0.40921398997306824,
      "log_odds_ratio": -0.5136203765869141,
      "logits/chosen": -0.6111175417900085,
      "logits/rejected": 0.1454327255487442,
      "logps/chosen": -2.2065820693969727,
      "logps/rejected": -2.579739570617676,
      "loss": 2.4918,
      "nll_loss": 2.4404354095458984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22065819799900055,
      "rewards/margins": 0.037315767258405685,
      "rewards/rejected": -0.2579739987850189,
      "step": 55
    },
    {
      "epoch": 0.08825847123719464,
      "grad_norm": 1.0213046073913574,
      "learning_rate": 1.7322834645669292e-06,
      "log_odds_chosen": 0.27949830889701843,
      "log_odds_ratio": -0.5767449140548706,
      "logits/chosen": -0.5307231545448303,
      "logits/rejected": 0.4212001860141754,
      "logps/chosen": -2.3242146968841553,
      "logps/rejected": -2.5813708305358887,
      "loss": 2.5978,
      "nll_loss": 2.540128707885742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23242147266864777,
      "rewards/margins": 0.025715606287121773,
      "rewards/rejected": -0.2581370770931244,
      "step": 56
    },
    {
      "epoch": 0.08983451536643026,
      "grad_norm": 0.7659716010093689,
      "learning_rate": 1.763779527559055e-06,
      "log_odds_chosen": 0.36814042925834656,
      "log_odds_ratio": -0.5291614532470703,
      "logits/chosen": -0.5885671973228455,
      "logits/rejected": 0.36917299032211304,
      "logps/chosen": -2.1409497261047363,
      "logps/rejected": -2.473041296005249,
      "loss": 2.4074,
      "nll_loss": 2.35445499420166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21409496665000916,
      "rewards/margins": 0.03320915997028351,
      "rewards/rejected": -0.24730412662029266,
      "step": 57
    },
    {
      "epoch": 0.09141055949566587,
      "grad_norm": 0.8884453773498535,
      "learning_rate": 1.795275590551181e-06,
      "log_odds_chosen": 0.3462219834327698,
      "log_odds_ratio": -0.5492972135543823,
      "logits/chosen": -0.58001309633255,
      "logits/rejected": 0.07347771525382996,
      "logps/chosen": -2.1368823051452637,
      "logps/rejected": -2.4556267261505127,
      "loss": 2.4149,
      "nll_loss": 2.359976053237915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2136882245540619,
      "rewards/margins": 0.031874436885118484,
      "rewards/rejected": -0.24556267261505127,
      "step": 58
    },
    {
      "epoch": 0.09298660362490149,
      "grad_norm": 0.9240288734436035,
      "learning_rate": 1.826771653543307e-06,
      "log_odds_chosen": 0.7683625221252441,
      "log_odds_ratio": -0.3947170376777649,
      "logits/chosen": -0.5223172903060913,
      "logits/rejected": 0.41549327969551086,
      "logps/chosen": -2.2466952800750732,
      "logps/rejected": -2.959190607070923,
      "loss": 2.4924,
      "nll_loss": 2.452920436859131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22466953098773956,
      "rewards/margins": 0.0712495669722557,
      "rewards/rejected": -0.2959190905094147,
      "step": 59
    },
    {
      "epoch": 0.09456264775413711,
      "grad_norm": 0.9423351883888245,
      "learning_rate": 1.858267716535433e-06,
      "log_odds_chosen": 0.5180408954620361,
      "log_odds_ratio": -0.48297566175460815,
      "logits/chosen": -0.5866909623146057,
      "logits/rejected": 0.16357183456420898,
      "logps/chosen": -2.1894171237945557,
      "logps/rejected": -2.6666948795318604,
      "loss": 2.4597,
      "nll_loss": 2.4114181995391846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21894173324108124,
      "rewards/margins": 0.04772777110338211,
      "rewards/rejected": -0.26666948199272156,
      "step": 60
    },
    {
      "epoch": 0.09613869188337273,
      "grad_norm": 0.9445580840110779,
      "learning_rate": 1.889763779527559e-06,
      "log_odds_chosen": 0.3669084906578064,
      "log_odds_ratio": -0.5591684579849243,
      "logits/chosen": -0.5270159840583801,
      "logits/rejected": 0.23815563321113586,
      "logps/chosen": -2.309187412261963,
      "logps/rejected": -2.6533546447753906,
      "loss": 2.5673,
      "nll_loss": 2.511385917663574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2309187352657318,
      "rewards/margins": 0.03441672772169113,
      "rewards/rejected": -0.26533547043800354,
      "step": 61
    },
    {
      "epoch": 0.09771473601260836,
      "grad_norm": 1.0885119438171387,
      "learning_rate": 1.9212598425196847e-06,
      "log_odds_chosen": 0.24820619821548462,
      "log_odds_ratio": -0.5866947174072266,
      "logits/chosen": -0.6077854037284851,
      "logits/rejected": 0.2984403371810913,
      "logps/chosen": -2.3013339042663574,
      "logps/rejected": -2.5299530029296875,
      "loss": 2.5716,
      "nll_loss": 2.512897253036499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23013338446617126,
      "rewards/margins": 0.022861897945404053,
      "rewards/rejected": -0.2529952824115753,
      "step": 62
    },
    {
      "epoch": 0.09929078014184398,
      "grad_norm": 0.8842782378196716,
      "learning_rate": 1.952755905511811e-06,
      "log_odds_chosen": 0.6375288963317871,
      "log_odds_ratio": -0.43413716554641724,
      "logits/chosen": -0.7599831819534302,
      "logits/rejected": 0.21808521449565887,
      "logps/chosen": -2.001779556274414,
      "logps/rejected": -2.5689616203308105,
      "loss": 2.2902,
      "nll_loss": 2.2467427253723145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20017798244953156,
      "rewards/margins": 0.05671820789575577,
      "rewards/rejected": -0.2568961977958679,
      "step": 63
    },
    {
      "epoch": 0.1008668242710796,
      "grad_norm": 0.8473801612854004,
      "learning_rate": 1.9842519685039368e-06,
      "log_odds_chosen": 0.7632179260253906,
      "log_odds_ratio": -0.4094430208206177,
      "logits/chosen": -0.6404256820678711,
      "logits/rejected": 0.3081037402153015,
      "logps/chosen": -2.0274438858032227,
      "logps/rejected": -2.7193148136138916,
      "loss": 2.3124,
      "nll_loss": 2.2714569568634033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20274437963962555,
      "rewards/margins": 0.06918711960315704,
      "rewards/rejected": -0.271931529045105,
      "step": 64
    },
    {
      "epoch": 0.1024428684003152,
      "grad_norm": 0.8429259657859802,
      "learning_rate": 2.015748031496063e-06,
      "log_odds_chosen": 0.6748821139335632,
      "log_odds_ratio": -0.41991835832595825,
      "logits/chosen": -0.6493370532989502,
      "logits/rejected": 0.1356709748506546,
      "logps/chosen": -2.115241765975952,
      "logps/rejected": -2.729600429534912,
      "loss": 2.3877,
      "nll_loss": 2.3457255363464355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21152418851852417,
      "rewards/margins": 0.061435848474502563,
      "rewards/rejected": -0.27296003699302673,
      "step": 65
    },
    {
      "epoch": 0.10401891252955082,
      "grad_norm": 0.8441055417060852,
      "learning_rate": 2.047244094488189e-06,
      "log_odds_chosen": 0.48374417424201965,
      "log_odds_ratio": -0.49196910858154297,
      "logits/chosen": -0.6177844405174255,
      "logits/rejected": 0.2689531445503235,
      "logps/chosen": -2.2088959217071533,
      "logps/rejected": -2.6486873626708984,
      "loss": 2.4956,
      "nll_loss": 2.446399688720703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2208895981311798,
      "rewards/margins": 0.04397913068532944,
      "rewards/rejected": -0.26486870646476746,
      "step": 66
    },
    {
      "epoch": 0.10559495665878645,
      "grad_norm": 0.8736575245857239,
      "learning_rate": 2.0787401574803147e-06,
      "log_odds_chosen": 0.5440582633018494,
      "log_odds_ratio": -0.4641622006893158,
      "logits/chosen": -0.5756601691246033,
      "logits/rejected": 0.4042266011238098,
      "logps/chosen": -2.1569972038269043,
      "logps/rejected": -2.6500601768493652,
      "loss": 2.4113,
      "nll_loss": 2.3649327754974365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2156997174024582,
      "rewards/margins": 0.04930633306503296,
      "rewards/rejected": -0.26500603556632996,
      "step": 67
    },
    {
      "epoch": 0.10717100078802207,
      "grad_norm": 1.0219923257827759,
      "learning_rate": 2.1102362204724405e-06,
      "log_odds_chosen": 0.38612306118011475,
      "log_odds_ratio": -0.5304967761039734,
      "logits/chosen": -0.7021428942680359,
      "logits/rejected": 0.2923763394355774,
      "logps/chosen": -2.1771302223205566,
      "logps/rejected": -2.529010057449341,
      "loss": 2.4909,
      "nll_loss": 2.437896966934204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21771301329135895,
      "rewards/margins": 0.0351879820227623,
      "rewards/rejected": -0.25290098786354065,
      "step": 68
    },
    {
      "epoch": 0.10874704491725769,
      "grad_norm": 1.0103472471237183,
      "learning_rate": 2.141732283464567e-06,
      "log_odds_chosen": 0.2671879529953003,
      "log_odds_ratio": -0.5846587419509888,
      "logits/chosen": -0.39670664072036743,
      "logits/rejected": 0.3414006531238556,
      "logps/chosen": -2.2567994594573975,
      "logps/rejected": -2.502344846725464,
      "loss": 2.5488,
      "nll_loss": 2.4903371334075928,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22567996382713318,
      "rewards/margins": 0.02455453760921955,
      "rewards/rejected": -0.2502345144748688,
      "step": 69
    },
    {
      "epoch": 0.11032308904649331,
      "grad_norm": 0.8793603777885437,
      "learning_rate": 2.173228346456693e-06,
      "log_odds_chosen": 0.4352240562438965,
      "log_odds_ratio": -0.505509614944458,
      "logits/chosen": -0.5309305787086487,
      "logits/rejected": 0.3599890172481537,
      "logps/chosen": -2.1654765605926514,
      "logps/rejected": -2.5600106716156006,
      "loss": 2.4516,
      "nll_loss": 2.4010517597198486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2165476679801941,
      "rewards/margins": 0.039453424513339996,
      "rewards/rejected": -0.2560010850429535,
      "step": 70
    },
    {
      "epoch": 0.11189913317572892,
      "grad_norm": 1.018066644668579,
      "learning_rate": 2.204724409448819e-06,
      "log_odds_chosen": 0.4932715892791748,
      "log_odds_ratio": -0.49797213077545166,
      "logits/chosen": -0.7851399183273315,
      "logits/rejected": 0.06452173739671707,
      "logps/chosen": -2.137401819229126,
      "logps/rejected": -2.58742618560791,
      "loss": 2.4109,
      "nll_loss": 2.3611340522766113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21374019980430603,
      "rewards/margins": 0.04500245302915573,
      "rewards/rejected": -0.25874263048171997,
      "step": 71
    },
    {
      "epoch": 0.11347517730496454,
      "grad_norm": 0.8898210525512695,
      "learning_rate": 2.2362204724409446e-06,
      "log_odds_chosen": 0.2936049997806549,
      "log_odds_ratio": -0.5717579126358032,
      "logits/chosen": -0.5596552491188049,
      "logits/rejected": 0.31953391432762146,
      "logps/chosen": -2.1422340869903564,
      "logps/rejected": -2.4082069396972656,
      "loss": 2.4216,
      "nll_loss": 2.3644003868103027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21422341465950012,
      "rewards/margins": 0.026597272604703903,
      "rewards/rejected": -0.24082070589065552,
      "step": 72
    },
    {
      "epoch": 0.11505122143420016,
      "grad_norm": 0.8397022485733032,
      "learning_rate": 2.267716535433071e-06,
      "log_odds_chosen": 0.37629467248916626,
      "log_odds_ratio": -0.526333749294281,
      "logits/chosen": -0.5056473612785339,
      "logits/rejected": 0.19583015143871307,
      "logps/chosen": -2.1670448780059814,
      "logps/rejected": -2.50854754447937,
      "loss": 2.4448,
      "nll_loss": 2.392120599746704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21670450270175934,
      "rewards/margins": 0.034150246530771255,
      "rewards/rejected": -0.2508547604084015,
      "step": 73
    },
    {
      "epoch": 0.11662726556343578,
      "grad_norm": 0.9695731401443481,
      "learning_rate": 2.2992125984251967e-06,
      "log_odds_chosen": 0.37078186869621277,
      "log_odds_ratio": -0.5291284918785095,
      "logits/chosen": -0.6568418741226196,
      "logits/rejected": 0.33567366003990173,
      "logps/chosen": -2.0653738975524902,
      "logps/rejected": -2.3936045169830322,
      "loss": 2.3679,
      "nll_loss": 2.3150012493133545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2065374106168747,
      "rewards/margins": 0.03282305598258972,
      "rewards/rejected": -0.23936045169830322,
      "step": 74
    },
    {
      "epoch": 0.1182033096926714,
      "grad_norm": 0.9203644394874573,
      "learning_rate": 2.3307086614173225e-06,
      "log_odds_chosen": 0.6003991961479187,
      "log_odds_ratio": -0.4472671151161194,
      "logits/chosen": -0.5809924006462097,
      "logits/rejected": 0.31674546003341675,
      "logps/chosen": -2.1095938682556152,
      "logps/rejected": -2.6563313007354736,
      "loss": 2.3819,
      "nll_loss": 2.337153434753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21095938980579376,
      "rewards/margins": 0.05467373877763748,
      "rewards/rejected": -0.26563313603401184,
      "step": 75
    },
    {
      "epoch": 0.11977935382190702,
      "grad_norm": 0.8541455864906311,
      "learning_rate": 2.3622047244094483e-06,
      "log_odds_chosen": 0.5599644184112549,
      "log_odds_ratio": -0.47028374671936035,
      "logits/chosen": -0.513896107673645,
      "logits/rejected": 0.44032493233680725,
      "logps/chosen": -2.139996290206909,
      "logps/rejected": -2.651939630508423,
      "loss": 2.3981,
      "nll_loss": 2.3510262966156006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21399962902069092,
      "rewards/margins": 0.05119434371590614,
      "rewards/rejected": -0.26519396901130676,
      "step": 76
    },
    {
      "epoch": 0.12135539795114263,
      "grad_norm": 0.9734304547309875,
      "learning_rate": 2.393700787401575e-06,
      "log_odds_chosen": 0.5413910150527954,
      "log_odds_ratio": -0.47939157485961914,
      "logits/chosen": -0.5768397450447083,
      "logits/rejected": 0.2747974097728729,
      "logps/chosen": -2.185297727584839,
      "logps/rejected": -2.681164026260376,
      "loss": 2.4654,
      "nll_loss": 2.4174275398254395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21852977573871613,
      "rewards/margins": 0.04958663508296013,
      "rewards/rejected": -0.26811644434928894,
      "step": 77
    },
    {
      "epoch": 0.12293144208037825,
      "grad_norm": 0.7510354518890381,
      "learning_rate": 2.425196850393701e-06,
      "log_odds_chosen": 0.4635729193687439,
      "log_odds_ratio": -0.5051595568656921,
      "logits/chosen": -0.3848555088043213,
      "logits/rejected": 0.07427231967449188,
      "logps/chosen": -2.1879336833953857,
      "logps/rejected": -2.612847089767456,
      "loss": 2.4399,
      "nll_loss": 2.389348268508911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21879339218139648,
      "rewards/margins": 0.04249133914709091,
      "rewards/rejected": -0.2612847089767456,
      "step": 78
    },
    {
      "epoch": 0.12450748620961387,
      "grad_norm": 0.9081109762191772,
      "learning_rate": 2.4566929133858266e-06,
      "log_odds_chosen": 0.5393069982528687,
      "log_odds_ratio": -0.46829110383987427,
      "logits/chosen": -0.5025306940078735,
      "logits/rejected": 0.21235036849975586,
      "logps/chosen": -2.1488659381866455,
      "logps/rejected": -2.639071464538574,
      "loss": 2.4115,
      "nll_loss": 2.3646249771118164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21488657593727112,
      "rewards/margins": 0.04902056232094765,
      "rewards/rejected": -0.26390716433525085,
      "step": 79
    },
    {
      "epoch": 0.12608353033884948,
      "grad_norm": 0.8297442197799683,
      "learning_rate": 2.488188976377953e-06,
      "log_odds_chosen": 0.644790530204773,
      "log_odds_ratio": -0.43552231788635254,
      "logits/chosen": -0.5664839744567871,
      "logits/rejected": 0.24572807550430298,
      "logps/chosen": -2.059072256088257,
      "logps/rejected": -2.6416335105895996,
      "loss": 2.321,
      "nll_loss": 2.2774367332458496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20590724050998688,
      "rewards/margins": 0.05825610086321831,
      "rewards/rejected": -0.2641633450984955,
      "step": 80
    },
    {
      "epoch": 0.1276595744680851,
      "grad_norm": 0.7417708039283752,
      "learning_rate": 2.5196850393700787e-06,
      "log_odds_chosen": 0.4702707529067993,
      "log_odds_ratio": -0.5074190497398376,
      "logits/chosen": -0.4309433698654175,
      "logits/rejected": 0.25342458486557007,
      "logps/chosen": -2.1097593307495117,
      "logps/rejected": -2.5410802364349365,
      "loss": 2.3641,
      "nll_loss": 2.31331729888916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21097594499588013,
      "rewards/margins": 0.04313211143016815,
      "rewards/rejected": -0.2541080713272095,
      "step": 81
    },
    {
      "epoch": 0.12923561859732072,
      "grad_norm": 0.7992884516716003,
      "learning_rate": 2.5511811023622045e-06,
      "log_odds_chosen": 0.6740604639053345,
      "log_odds_ratio": -0.43822091817855835,
      "logits/chosen": -0.5042348504066467,
      "logits/rejected": 0.23487010598182678,
      "logps/chosen": -2.0556299686431885,
      "logps/rejected": -2.6750376224517822,
      "loss": 2.3225,
      "nll_loss": 2.2787139415740967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2055629938840866,
      "rewards/margins": 0.061940789222717285,
      "rewards/rejected": -0.2675037682056427,
      "step": 82
    },
    {
      "epoch": 0.13081166272655634,
      "grad_norm": 0.7936323285102844,
      "learning_rate": 2.5826771653543303e-06,
      "log_odds_chosen": 0.33946871757507324,
      "log_odds_ratio": -0.5478945970535278,
      "logits/chosen": -0.3788740634918213,
      "logits/rejected": 0.251629114151001,
      "logps/chosen": -2.0991952419281006,
      "logps/rejected": -2.4062960147857666,
      "loss": 2.3638,
      "nll_loss": 2.309030532836914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2099195122718811,
      "rewards/margins": 0.030710091814398766,
      "rewards/rejected": -0.2406296283006668,
      "step": 83
    },
    {
      "epoch": 0.13238770685579196,
      "grad_norm": 0.9158631563186646,
      "learning_rate": 2.6141732283464566e-06,
      "log_odds_chosen": 0.6110659837722778,
      "log_odds_ratio": -0.4645346403121948,
      "logits/chosen": -0.5568953156471252,
      "logits/rejected": 0.15132026374340057,
      "logps/chosen": -2.069227933883667,
      "logps/rejected": -2.6256966590881348,
      "loss": 2.3621,
      "nll_loss": 2.315664052963257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20692278444766998,
      "rewards/margins": 0.0556468665599823,
      "rewards/rejected": -0.2625696659088135,
      "step": 84
    },
    {
      "epoch": 0.13396375098502758,
      "grad_norm": 0.7886312007904053,
      "learning_rate": 2.645669291338583e-06,
      "log_odds_chosen": 0.525355875492096,
      "log_odds_ratio": -0.4657125771045685,
      "logits/chosen": -0.37237846851348877,
      "logits/rejected": 0.3347388207912445,
      "logps/chosen": -2.123147487640381,
      "logps/rejected": -2.598121166229248,
      "loss": 2.3755,
      "nll_loss": 2.3288934230804443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21231473982334137,
      "rewards/margins": 0.04749736934900284,
      "rewards/rejected": -0.2598121166229248,
      "step": 85
    },
    {
      "epoch": 0.1355397951142632,
      "grad_norm": 0.8968652486801147,
      "learning_rate": 2.6771653543307086e-06,
      "log_odds_chosen": 0.5338146686553955,
      "log_odds_ratio": -0.46815913915634155,
      "logits/chosen": -0.6325598955154419,
      "logits/rejected": 0.32120490074157715,
      "logps/chosen": -2.123342752456665,
      "logps/rejected": -2.605854034423828,
      "loss": 2.3892,
      "nll_loss": 2.342399835586548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2123342901468277,
      "rewards/margins": 0.04825112968683243,
      "rewards/rejected": -0.2605854272842407,
      "step": 86
    },
    {
      "epoch": 0.13711583924349882,
      "grad_norm": 0.7727012634277344,
      "learning_rate": 2.7086614173228344e-06,
      "log_odds_chosen": 0.3835631310939789,
      "log_odds_ratio": -0.5259324312210083,
      "logits/chosen": -0.3659380078315735,
      "logits/rejected": 0.1596633940935135,
      "logps/chosen": -2.104921340942383,
      "logps/rejected": -2.4491097927093506,
      "loss": 2.3625,
      "nll_loss": 2.309898614883423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21049213409423828,
      "rewards/margins": 0.034418847411870956,
      "rewards/rejected": -0.24491098523139954,
      "step": 87
    },
    {
      "epoch": 0.13869188337273444,
      "grad_norm": 0.9119899868965149,
      "learning_rate": 2.7401574803149607e-06,
      "log_odds_chosen": 0.7017119526863098,
      "log_odds_ratio": -0.42803943157196045,
      "logits/chosen": -0.6377043724060059,
      "logits/rejected": 0.3170080780982971,
      "logps/chosen": -2.032348871231079,
      "logps/rejected": -2.671203136444092,
      "loss": 2.3239,
      "nll_loss": 2.2810964584350586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20323489606380463,
      "rewards/margins": 0.0638853907585144,
      "rewards/rejected": -0.2671203017234802,
      "step": 88
    },
    {
      "epoch": 0.14026792750197006,
      "grad_norm": 0.9140077233314514,
      "learning_rate": 2.7716535433070865e-06,
      "log_odds_chosen": 0.48430994153022766,
      "log_odds_ratio": -0.4811188280582428,
      "logits/chosen": -0.46536654233932495,
      "logits/rejected": 0.35523223876953125,
      "logps/chosen": -2.1665518283843994,
      "logps/rejected": -2.6059086322784424,
      "loss": 2.4371,
      "nll_loss": 2.3890204429626465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2166551947593689,
      "rewards/margins": 0.04393570125102997,
      "rewards/rejected": -0.26059091091156006,
      "step": 89
    },
    {
      "epoch": 0.14184397163120568,
      "grad_norm": 0.8198168277740479,
      "learning_rate": 2.8031496062992123e-06,
      "log_odds_chosen": 0.40435484051704407,
      "log_odds_ratio": -0.5177271962165833,
      "logits/chosen": -0.3415748178958893,
      "logits/rejected": 0.31405988335609436,
      "logps/chosen": -2.2057156562805176,
      "logps/rejected": -2.5755887031555176,
      "loss": 2.4455,
      "nll_loss": 2.3937432765960693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2205715775489807,
      "rewards/margins": 0.036987315863370895,
      "rewards/rejected": -0.2575588822364807,
      "step": 90
    },
    {
      "epoch": 0.1434200157604413,
      "grad_norm": 0.7300707101821899,
      "learning_rate": 2.834645669291338e-06,
      "log_odds_chosen": 0.5259378552436829,
      "log_odds_ratio": -0.4800778329372406,
      "logits/chosen": -0.3468290865421295,
      "logits/rejected": 0.19894689321517944,
      "logps/chosen": -1.9790496826171875,
      "logps/rejected": -2.4511303901672363,
      "loss": 2.2363,
      "nll_loss": 2.188265085220337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19790495932102203,
      "rewards/margins": 0.04720810800790787,
      "rewards/rejected": -0.2451130747795105,
      "step": 91
    },
    {
      "epoch": 0.1449960598896769,
      "grad_norm": 0.7919360995292664,
      "learning_rate": 2.8661417322834644e-06,
      "log_odds_chosen": 0.5125716924667358,
      "log_odds_ratio": -0.4782637357711792,
      "logits/chosen": -0.4530889689922333,
      "logits/rejected": 0.2999853193759918,
      "logps/chosen": -2.0266165733337402,
      "logps/rejected": -2.486846685409546,
      "loss": 2.299,
      "nll_loss": 2.251190185546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2026616334915161,
      "rewards/margins": 0.046023011207580566,
      "rewards/rejected": -0.24868465960025787,
      "step": 92
    },
    {
      "epoch": 0.14657210401891252,
      "grad_norm": 0.8349723815917969,
      "learning_rate": 2.8976377952755906e-06,
      "log_odds_chosen": 0.4769258499145508,
      "log_odds_ratio": -0.5019131898880005,
      "logits/chosen": -0.3912510275840759,
      "logits/rejected": 0.2896612584590912,
      "logps/chosen": -2.136798858642578,
      "logps/rejected": -2.5724754333496094,
      "loss": 2.3948,
      "nll_loss": 2.344578266143799,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21367989480495453,
      "rewards/margins": 0.043567635118961334,
      "rewards/rejected": -0.25724753737449646,
      "step": 93
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 0.7782561182975769,
      "learning_rate": 2.9291338582677165e-06,
      "log_odds_chosen": 0.46800780296325684,
      "log_odds_ratio": -0.4987240731716156,
      "logits/chosen": -0.3847990334033966,
      "logits/rejected": 0.16628870368003845,
      "logps/chosen": -2.1040778160095215,
      "logps/rejected": -2.523435592651367,
      "loss": 2.3559,
      "nll_loss": 2.306053400039673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2104077935218811,
      "rewards/margins": 0.04193580150604248,
      "rewards/rejected": -0.2523435950279236,
      "step": 94
    },
    {
      "epoch": 0.14972419227738376,
      "grad_norm": 0.8788174986839294,
      "learning_rate": 2.9606299212598423e-06,
      "log_odds_chosen": 0.6574824452400208,
      "log_odds_ratio": -0.4303269386291504,
      "logits/chosen": -0.41694146394729614,
      "logits/rejected": 0.33073097467422485,
      "logps/chosen": -2.1245923042297363,
      "logps/rejected": -2.725464344024658,
      "loss": 2.3872,
      "nll_loss": 2.3441953659057617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21245922148227692,
      "rewards/margins": 0.06008722260594368,
      "rewards/rejected": -0.2725464701652527,
      "step": 95
    },
    {
      "epoch": 0.15130023640661938,
      "grad_norm": 0.7738668322563171,
      "learning_rate": 2.9921259842519685e-06,
      "log_odds_chosen": 0.6411773562431335,
      "log_odds_ratio": -0.4322814643383026,
      "logits/chosen": -0.45478928089141846,
      "logits/rejected": 0.2121216505765915,
      "logps/chosen": -2.096010684967041,
      "logps/rejected": -2.6793506145477295,
      "loss": 2.3287,
      "nll_loss": 2.285444498062134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20960105955600739,
      "rewards/margins": 0.05833399295806885,
      "rewards/rejected": -0.26793503761291504,
      "step": 96
    },
    {
      "epoch": 0.152876280535855,
      "grad_norm": 0.733166515827179,
      "learning_rate": 3.0236220472440943e-06,
      "log_odds_chosen": 0.35148048400878906,
      "log_odds_ratio": -0.5345249176025391,
      "logits/chosen": -0.4715975224971771,
      "logits/rejected": 0.29987144470214844,
      "logps/chosen": -2.0591440200805664,
      "logps/rejected": -2.3701171875,
      "loss": 2.3035,
      "nll_loss": 2.25003719329834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20591440796852112,
      "rewards/margins": 0.031097283586859703,
      "rewards/rejected": -0.23701170086860657,
      "step": 97
    },
    {
      "epoch": 0.15445232466509062,
      "grad_norm": 0.7708275318145752,
      "learning_rate": 3.05511811023622e-06,
      "log_odds_chosen": 0.43126457929611206,
      "log_odds_ratio": -0.5121496319770813,
      "logits/chosen": -0.2948766052722931,
      "logits/rejected": 0.15091395378112793,
      "logps/chosen": -2.189704656600952,
      "logps/rejected": -2.583603858947754,
      "loss": 2.4482,
      "nll_loss": 2.3969645500183105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21897049248218536,
      "rewards/margins": 0.03938991576433182,
      "rewards/rejected": -0.2583604156970978,
      "step": 98
    },
    {
      "epoch": 0.15602836879432624,
      "grad_norm": 0.8191169500350952,
      "learning_rate": 3.086614173228346e-06,
      "log_odds_chosen": 0.22818513214588165,
      "log_odds_ratio": -0.5883071422576904,
      "logits/chosen": -0.5125383138656616,
      "logits/rejected": 0.10425034165382385,
      "logps/chosen": -2.1129236221313477,
      "logps/rejected": -2.3160712718963623,
      "loss": 2.3859,
      "nll_loss": 2.3271055221557617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2112923562526703,
      "rewards/margins": 0.020314780995249748,
      "rewards/rejected": -0.23160713911056519,
      "step": 99
    },
    {
      "epoch": 0.15760441292356187,
      "grad_norm": 0.7705802321434021,
      "learning_rate": 3.1181102362204722e-06,
      "log_odds_chosen": 0.4004606008529663,
      "log_odds_ratio": -0.5196990370750427,
      "logits/chosen": -0.46414798498153687,
      "logits/rejected": 0.17729449272155762,
      "logps/chosen": -2.0139927864074707,
      "logps/rejected": -2.3726866245269775,
      "loss": 2.2862,
      "nll_loss": 2.234262466430664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20139926671981812,
      "rewards/margins": 0.035869404673576355,
      "rewards/rejected": -0.23726868629455566,
      "step": 100
    },
    {
      "epoch": 0.15918045705279749,
      "grad_norm": 0.7530562877655029,
      "learning_rate": 3.1496062992125985e-06,
      "log_odds_chosen": 0.43779271841049194,
      "log_odds_ratio": -0.5091973543167114,
      "logits/chosen": -0.3823246657848358,
      "logits/rejected": 0.2523665428161621,
      "logps/chosen": -2.008505344390869,
      "logps/rejected": -2.4028048515319824,
      "loss": 2.2858,
      "nll_loss": 2.2348995208740234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2008505016565323,
      "rewards/margins": 0.03942997381091118,
      "rewards/rejected": -0.24028049409389496,
      "step": 101
    },
    {
      "epoch": 0.1607565011820331,
      "grad_norm": 0.6781676411628723,
      "learning_rate": 3.1811023622047243e-06,
      "log_odds_chosen": 0.4132150709629059,
      "log_odds_ratio": -0.5262648463249207,
      "logits/chosen": -0.3420214354991913,
      "logits/rejected": 0.1129278838634491,
      "logps/chosen": -1.9598687887191772,
      "logps/rejected": -2.329220771789551,
      "loss": 2.2327,
      "nll_loss": 2.18007493019104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19598689675331116,
      "rewards/margins": 0.03693518415093422,
      "rewards/rejected": -0.23292206227779388,
      "step": 102
    },
    {
      "epoch": 0.16233254531126873,
      "grad_norm": 0.6758076548576355,
      "learning_rate": 3.2125984251968505e-06,
      "log_odds_chosen": 0.4648427963256836,
      "log_odds_ratio": -0.5020236968994141,
      "logits/chosen": -0.2520603537559509,
      "logits/rejected": 0.437377393245697,
      "logps/chosen": -1.937645673751831,
      "logps/rejected": -2.347379684448242,
      "loss": 2.2144,
      "nll_loss": 2.1642251014709473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1937645673751831,
      "rewards/margins": 0.040973398834466934,
      "rewards/rejected": -0.23473796248435974,
      "step": 103
    },
    {
      "epoch": 0.16390858944050432,
      "grad_norm": 0.7805861234664917,
      "learning_rate": 3.2440944881889763e-06,
      "log_odds_chosen": 0.4882758557796478,
      "log_odds_ratio": -0.5017855763435364,
      "logits/chosen": -0.20399755239486694,
      "logits/rejected": 0.5055577754974365,
      "logps/chosen": -2.1117475032806396,
      "logps/rejected": -2.558027744293213,
      "loss": 2.3732,
      "nll_loss": 2.3230180740356445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21117472648620605,
      "rewards/margins": 0.044628046452999115,
      "rewards/rejected": -0.25580278038978577,
      "step": 104
    },
    {
      "epoch": 0.16548463356973994,
      "grad_norm": 0.6345953345298767,
      "learning_rate": 3.275590551181102e-06,
      "log_odds_chosen": 0.4859236478805542,
      "log_odds_ratio": -0.4872024357318878,
      "logits/chosen": -0.3714447617530823,
      "logits/rejected": 0.3421481251716614,
      "logps/chosen": -1.947684407234192,
      "logps/rejected": -2.3787930011749268,
      "loss": 2.2301,
      "nll_loss": 2.1814191341400146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19476845860481262,
      "rewards/margins": 0.04311085492372513,
      "rewards/rejected": -0.23787930607795715,
      "step": 105
    },
    {
      "epoch": 0.16706067769897556,
      "grad_norm": 0.6725803017616272,
      "learning_rate": 3.307086614173228e-06,
      "log_odds_chosen": 0.4803786873817444,
      "log_odds_ratio": -0.494854211807251,
      "logits/chosen": -0.38588568568229675,
      "logits/rejected": 0.39132970571517944,
      "logps/chosen": -1.9737625122070312,
      "logps/rejected": -2.3936586380004883,
      "loss": 2.2348,
      "nll_loss": 2.1852762699127197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19737625122070312,
      "rewards/margins": 0.041989609599113464,
      "rewards/rejected": -0.23936587572097778,
      "step": 106
    },
    {
      "epoch": 0.16863672182821118,
      "grad_norm": 0.6772481203079224,
      "learning_rate": 3.3385826771653542e-06,
      "log_odds_chosen": 0.4360727071762085,
      "log_odds_ratio": -0.5130836963653564,
      "logits/chosen": -0.22578240931034088,
      "logits/rejected": 0.21718156337738037,
      "logps/chosen": -2.05010986328125,
      "logps/rejected": -2.44297456741333,
      "loss": 2.3066,
      "nll_loss": 2.2553317546844482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20501098036766052,
      "rewards/margins": 0.03928646445274353,
      "rewards/rejected": -0.24429744482040405,
      "step": 107
    },
    {
      "epoch": 0.1702127659574468,
      "grad_norm": 0.6258271932601929,
      "learning_rate": 3.37007874015748e-06,
      "log_odds_chosen": 0.37870439887046814,
      "log_odds_ratio": -0.5316060185432434,
      "logits/chosen": -0.20526322722434998,
      "logits/rejected": 0.22974106669425964,
      "logps/chosen": -2.0181500911712646,
      "logps/rejected": -2.354243278503418,
      "loss": 2.2772,
      "nll_loss": 2.22402024269104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20181502401828766,
      "rewards/margins": 0.03360932320356369,
      "rewards/rejected": -0.23542433977127075,
      "step": 108
    },
    {
      "epoch": 0.17178881008668243,
      "grad_norm": 0.6688246130943298,
      "learning_rate": 3.4015748031496063e-06,
      "log_odds_chosen": 0.468779593706131,
      "log_odds_ratio": -0.4883277416229248,
      "logits/chosen": -0.12419804185628891,
      "logits/rejected": 0.2353970855474472,
      "logps/chosen": -2.091757297515869,
      "logps/rejected": -2.511913299560547,
      "loss": 2.3467,
      "nll_loss": 2.2978553771972656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20917576551437378,
      "rewards/margins": 0.042015574872493744,
      "rewards/rejected": -0.25119131803512573,
      "step": 109
    },
    {
      "epoch": 0.17336485421591805,
      "grad_norm": 0.6432668566703796,
      "learning_rate": 3.433070866141732e-06,
      "log_odds_chosen": 0.5067487955093384,
      "log_odds_ratio": -0.4804004430770874,
      "logits/chosen": -0.24377617239952087,
      "logits/rejected": 0.29760029911994934,
      "logps/chosen": -1.9592363834381104,
      "logps/rejected": -2.402676582336426,
      "loss": 2.1931,
      "nll_loss": 2.1450953483581543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19592364132404327,
      "rewards/margins": 0.04434400796890259,
      "rewards/rejected": -0.24026766419410706,
      "step": 110
    },
    {
      "epoch": 0.17494089834515367,
      "grad_norm": 0.7077887654304504,
      "learning_rate": 3.4645669291338583e-06,
      "log_odds_chosen": 0.5833423733711243,
      "log_odds_ratio": -0.4554339051246643,
      "logits/chosen": -0.22579139471054077,
      "logits/rejected": 0.24488507211208344,
      "logps/chosen": -2.0807836055755615,
      "logps/rejected": -2.604956865310669,
      "loss": 2.3392,
      "nll_loss": 2.2936172485351562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20807836949825287,
      "rewards/margins": 0.0524173304438591,
      "rewards/rejected": -0.26049572229385376,
      "step": 111
    },
    {
      "epoch": 0.1765169424743893,
      "grad_norm": 0.6585553288459778,
      "learning_rate": 3.496062992125984e-06,
      "log_odds_chosen": 0.3966713547706604,
      "log_odds_ratio": -0.5204365253448486,
      "logits/chosen": -0.16819843649864197,
      "logits/rejected": 0.3559090793132782,
      "logps/chosen": -2.107595205307007,
      "logps/rejected": -2.462603807449341,
      "loss": 2.3602,
      "nll_loss": 2.308168411254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2107595056295395,
      "rewards/margins": 0.035500891506671906,
      "rewards/rejected": -0.24626041948795319,
      "step": 112
    },
    {
      "epoch": 0.1780929866036249,
      "grad_norm": 0.6397602558135986,
      "learning_rate": 3.52755905511811e-06,
      "log_odds_chosen": 0.35250332951545715,
      "log_odds_ratio": -0.5411746501922607,
      "logits/chosen": -0.3075207769870758,
      "logits/rejected": 0.21061550080776215,
      "logps/chosen": -1.9689809083938599,
      "logps/rejected": -2.2822649478912354,
      "loss": 2.2367,
      "nll_loss": 2.1825618743896484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19689807295799255,
      "rewards/margins": 0.03132840245962143,
      "rewards/rejected": -0.22822648286819458,
      "step": 113
    },
    {
      "epoch": 0.17966903073286053,
      "grad_norm": 0.593964159488678,
      "learning_rate": 3.559055118110236e-06,
      "log_odds_chosen": 0.32149726152420044,
      "log_odds_ratio": -0.5685874223709106,
      "logits/chosen": -0.049146618694067,
      "logits/rejected": 0.2939941883087158,
      "logps/chosen": -2.0819308757781982,
      "logps/rejected": -2.375070333480835,
      "loss": 2.3471,
      "nll_loss": 2.290268898010254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2081930786371231,
      "rewards/margins": 0.029313962906599045,
      "rewards/rejected": -0.23750706017017365,
      "step": 114
    },
    {
      "epoch": 0.18124507486209615,
      "grad_norm": 0.5885688066482544,
      "learning_rate": 3.590551181102362e-06,
      "log_odds_chosen": 0.4819232225418091,
      "log_odds_ratio": -0.49382418394088745,
      "logits/chosen": -0.16471779346466064,
      "logits/rejected": 0.19575665891170502,
      "logps/chosen": -1.9886581897735596,
      "logps/rejected": -2.418565273284912,
      "loss": 2.2384,
      "nll_loss": 2.1889805793762207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1988658308982849,
      "rewards/margins": 0.04299072176218033,
      "rewards/rejected": -0.24185653030872345,
      "step": 115
    },
    {
      "epoch": 0.18282111899133174,
      "grad_norm": 0.5882607102394104,
      "learning_rate": 3.622047244094488e-06,
      "log_odds_chosen": 0.38364386558532715,
      "log_odds_ratio": -0.5387082695960999,
      "logits/chosen": -0.14860567450523376,
      "logits/rejected": 0.33340632915496826,
      "logps/chosen": -2.0445148944854736,
      "logps/rejected": -2.388934373855591,
      "loss": 2.2878,
      "nll_loss": 2.233961582183838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20445148646831512,
      "rewards/margins": 0.03444194793701172,
      "rewards/rejected": -0.23889344930648804,
      "step": 116
    },
    {
      "epoch": 0.18439716312056736,
      "grad_norm": 0.6176384091377258,
      "learning_rate": 3.653543307086614e-06,
      "log_odds_chosen": 0.5844452977180481,
      "log_odds_ratio": -0.4591362476348877,
      "logits/chosen": -0.27243772149086,
      "logits/rejected": 0.04437294602394104,
      "logps/chosen": -1.9680087566375732,
      "logps/rejected": -2.489656925201416,
      "loss": 2.2336,
      "nll_loss": 2.1876473426818848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19680088758468628,
      "rewards/margins": 0.05216481536626816,
      "rewards/rejected": -0.24896571040153503,
      "step": 117
    },
    {
      "epoch": 0.18597320724980299,
      "grad_norm": 0.5718716979026794,
      "learning_rate": 3.68503937007874e-06,
      "log_odds_chosen": 0.3214079439640045,
      "log_odds_ratio": -0.5579454302787781,
      "logits/chosen": -0.010645353235304356,
      "logits/rejected": 0.30515971779823303,
      "logps/chosen": -1.9919353723526,
      "logps/rejected": -2.2776200771331787,
      "loss": 2.254,
      "nll_loss": 2.1981608867645264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19919352233409882,
      "rewards/margins": 0.028568467125296593,
      "rewards/rejected": -0.22776199877262115,
      "step": 118
    },
    {
      "epoch": 0.1875492513790386,
      "grad_norm": 0.5524156093597412,
      "learning_rate": 3.716535433070866e-06,
      "log_odds_chosen": 0.3362745940685272,
      "log_odds_ratio": -0.5541406869888306,
      "logits/chosen": -0.08601650595664978,
      "logits/rejected": 0.28649064898490906,
      "logps/chosen": -1.9557974338531494,
      "logps/rejected": -2.2547686100006104,
      "loss": 2.2373,
      "nll_loss": 2.1819238662719727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19557975232601166,
      "rewards/margins": 0.029897117987275124,
      "rewards/rejected": -0.22547689080238342,
      "step": 119
    },
    {
      "epoch": 0.18912529550827423,
      "grad_norm": 0.5379864573478699,
      "learning_rate": 3.748031496062992e-06,
      "log_odds_chosen": 0.6901645064353943,
      "log_odds_ratio": -0.42435628175735474,
      "logits/chosen": -0.12895634770393372,
      "logits/rejected": 0.31545358896255493,
      "logps/chosen": -1.8600318431854248,
      "logps/rejected": -2.4725053310394287,
      "loss": 2.1111,
      "nll_loss": 2.0686473846435547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1860032081604004,
      "rewards/margins": 0.06124735623598099,
      "rewards/rejected": -0.2472505420446396,
      "step": 120
    },
    {
      "epoch": 0.19070133963750985,
      "grad_norm": 0.5051740407943726,
      "learning_rate": 3.779527559055118e-06,
      "log_odds_chosen": 0.7074211835861206,
      "log_odds_ratio": -0.4035435616970062,
      "logits/chosen": -0.1733936369419098,
      "logits/rejected": 0.12479071319103241,
      "logps/chosen": -1.961637020111084,
      "logps/rejected": -2.5950496196746826,
      "loss": 2.1988,
      "nll_loss": 2.158412218093872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19616371393203735,
      "rewards/margins": 0.06334124505519867,
      "rewards/rejected": -0.25950494408607483,
      "step": 121
    },
    {
      "epoch": 0.19227738376674547,
      "grad_norm": 0.5642458200454712,
      "learning_rate": 3.8110236220472436e-06,
      "log_odds_chosen": 0.5054293870925903,
      "log_odds_ratio": -0.4897348880767822,
      "logits/chosen": -0.2511146068572998,
      "logits/rejected": 0.25759318470954895,
      "logps/chosen": -1.9019826650619507,
      "logps/rejected": -2.3513503074645996,
      "loss": 2.144,
      "nll_loss": 2.0950286388397217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19019825756549835,
      "rewards/margins": 0.04493675008416176,
      "rewards/rejected": -0.2351350039243698,
      "step": 122
    },
    {
      "epoch": 0.1938534278959811,
      "grad_norm": 0.546146035194397,
      "learning_rate": 3.8425196850393695e-06,
      "log_odds_chosen": 0.424507737159729,
      "log_odds_ratio": -0.5058774352073669,
      "logits/chosen": 0.04497145488858223,
      "logits/rejected": 0.20520615577697754,
      "logps/chosen": -1.9872440099716187,
      "logps/rejected": -2.363257884979248,
      "loss": 2.2507,
      "nll_loss": 2.2000772953033447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1987243890762329,
      "rewards/margins": 0.037601370364427567,
      "rewards/rejected": -0.23632577061653137,
      "step": 123
    },
    {
      "epoch": 0.1954294720252167,
      "grad_norm": 0.496383935213089,
      "learning_rate": 3.874015748031496e-06,
      "log_odds_chosen": 0.43429940938949585,
      "log_odds_ratio": -0.5118638277053833,
      "logits/chosen": -0.17999807000160217,
      "logits/rejected": 0.2584393322467804,
      "logps/chosen": -1.8671247959136963,
      "logps/rejected": -2.2387583255767822,
      "loss": 2.1116,
      "nll_loss": 2.0604166984558105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18671248853206635,
      "rewards/margins": 0.03716333210468292,
      "rewards/rejected": -0.22387583553791046,
      "step": 124
    },
    {
      "epoch": 0.19700551615445233,
      "grad_norm": 0.5127514600753784,
      "learning_rate": 3.905511811023622e-06,
      "log_odds_chosen": 0.47707968950271606,
      "log_odds_ratio": -0.49198096990585327,
      "logits/chosen": -0.09298758208751678,
      "logits/rejected": 0.04988850653171539,
      "logps/chosen": -1.9040323495864868,
      "logps/rejected": -2.3241515159606934,
      "loss": 2.1671,
      "nll_loss": 2.117922306060791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19040323793888092,
      "rewards/margins": 0.04201190173625946,
      "rewards/rejected": -0.23241513967514038,
      "step": 125
    },
    {
      "epoch": 0.19858156028368795,
      "grad_norm": 0.5426633358001709,
      "learning_rate": 3.937007874015748e-06,
      "log_odds_chosen": 0.6238756775856018,
      "log_odds_ratio": -0.4400244355201721,
      "logits/chosen": -0.0192108117043972,
      "logits/rejected": 0.3376918137073517,
      "logps/chosen": -1.9439184665679932,
      "logps/rejected": -2.5035502910614014,
      "loss": 2.1878,
      "nll_loss": 2.1438095569610596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1943918615579605,
      "rewards/margins": 0.0559631809592247,
      "rewards/rejected": -0.2503550350666046,
      "step": 126
    },
    {
      "epoch": 0.20015760441292357,
      "grad_norm": 0.5588484406471252,
      "learning_rate": 3.9685039370078736e-06,
      "log_odds_chosen": 0.567841112613678,
      "log_odds_ratio": -0.46976813673973083,
      "logits/chosen": 0.013626057654619217,
      "logits/rejected": 0.3258940875530243,
      "logps/chosen": -1.9778722524642944,
      "logps/rejected": -2.4898061752319336,
      "loss": 2.2598,
      "nll_loss": 2.21280574798584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19778724014759064,
      "rewards/margins": 0.05119336396455765,
      "rewards/rejected": -0.2489805966615677,
      "step": 127
    },
    {
      "epoch": 0.2017336485421592,
      "grad_norm": 0.5180553793907166,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.2811572551727295,
      "log_odds_ratio": -0.5696135759353638,
      "logits/chosen": 0.08110462874174118,
      "logits/rejected": 0.487108051776886,
      "logps/chosen": -2.0393309593200684,
      "logps/rejected": -2.288515090942383,
      "loss": 2.2809,
      "nll_loss": 2.2239131927490234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20393308997154236,
      "rewards/margins": 0.02491840347647667,
      "rewards/rejected": -0.22885149717330933,
      "step": 128
    },
    {
      "epoch": 0.2033096926713948,
      "grad_norm": 0.5022251009941101,
      "learning_rate": 3.999992445477635e-06,
      "log_odds_chosen": 0.4684014916419983,
      "log_odds_ratio": -0.49844875931739807,
      "logits/chosen": 0.03743650019168854,
      "logits/rejected": 0.18749424815177917,
      "logps/chosen": -2.027505874633789,
      "logps/rejected": -2.4474422931671143,
      "loss": 2.2515,
      "nll_loss": 2.2016143798828125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.202750563621521,
      "rewards/margins": 0.041993647813797,
      "rewards/rejected": -0.2447442263364792,
      "step": 129
    },
    {
      "epoch": 0.2048857368006304,
      "grad_norm": 0.49709925055503845,
      "learning_rate": 3.999969781967615e-06,
      "log_odds_chosen": 0.30073270201683044,
      "log_odds_ratio": -0.5600437521934509,
      "logits/chosen": 0.047653649002313614,
      "logits/rejected": 0.35839155316352844,
      "logps/chosen": -2.123664617538452,
      "logps/rejected": -2.3944878578186035,
      "loss": 2.3459,
      "nll_loss": 2.2898778915405273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21236644685268402,
      "rewards/margins": 0.027082331478595734,
      "rewards/rejected": -0.23944880068302155,
      "step": 130
    },
    {
      "epoch": 0.20646178092986603,
      "grad_norm": 0.44330260157585144,
      "learning_rate": 3.99993200964115e-06,
      "log_odds_chosen": 0.3670748472213745,
      "log_odds_ratio": -0.5292910933494568,
      "logits/chosen": -0.04872098192572594,
      "logits/rejected": 0.13453775644302368,
      "logps/chosen": -1.9400205612182617,
      "logps/rejected": -2.261190414428711,
      "loss": 2.1811,
      "nll_loss": 2.1281275749206543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19400206208229065,
      "rewards/margins": 0.03211698681116104,
      "rewards/rejected": -0.2261190563440323,
      "step": 131
    },
    {
      "epoch": 0.20803782505910165,
      "grad_norm": 0.5485524535179138,
      "learning_rate": 3.99987912878359e-06,
      "log_odds_chosen": 0.1374671310186386,
      "log_odds_ratio": -0.634438693523407,
      "logits/chosen": 0.07270228117704391,
      "logits/rejected": 0.4620732367038727,
      "logps/chosen": -1.9368000030517578,
      "logps/rejected": -2.0512306690216064,
      "loss": 2.2276,
      "nll_loss": 2.1641933917999268,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19368000328540802,
      "rewards/margins": 0.011443082243204117,
      "rewards/rejected": -0.20512308180332184,
      "step": 132
    },
    {
      "epoch": 0.20961386918833727,
      "grad_norm": 0.4629489481449127,
      "learning_rate": 3.999811139794429e-06,
      "log_odds_chosen": 0.40406906604766846,
      "log_odds_ratio": -0.5154622793197632,
      "logits/chosen": 0.050318021327257156,
      "logits/rejected": 0.28723451495170593,
      "logps/chosen": -1.9487131834030151,
      "logps/rejected": -2.304403305053711,
      "loss": 2.1784,
      "nll_loss": 2.1268701553344727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19487132132053375,
      "rewards/margins": 0.03556900843977928,
      "rewards/rejected": -0.23044034838676453,
      "step": 133
    },
    {
      "epoch": 0.2111899133175729,
      "grad_norm": 0.4633968472480774,
      "learning_rate": 3.999728043187288e-06,
      "log_odds_chosen": 0.35710862278938293,
      "log_odds_ratio": -0.5401404500007629,
      "logits/chosen": 0.08867964148521423,
      "logits/rejected": 0.26978808641433716,
      "logps/chosen": -1.9941068887710571,
      "logps/rejected": -2.3115949630737305,
      "loss": 2.2544,
      "nll_loss": 2.2004222869873047,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19941069185733795,
      "rewards/margins": 0.03174880892038345,
      "rewards/rejected": -0.231159508228302,
      "step": 134
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 0.45478522777557373,
      "learning_rate": 3.999629839589922e-06,
      "log_odds_chosen": 0.26061490178108215,
      "log_odds_ratio": -0.5747462511062622,
      "logits/chosen": 0.1319434493780136,
      "logits/rejected": 0.09344829618930817,
      "logps/chosen": -2.0818891525268555,
      "logps/rejected": -2.3141493797302246,
      "loss": 2.3355,
      "nll_loss": 2.2779908180236816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20818890631198883,
      "rewards/margins": 0.02322601154446602,
      "rewards/rejected": -0.23141492903232574,
      "step": 135
    },
    {
      "epoch": 0.21434200157604413,
      "grad_norm": 0.4266544282436371,
      "learning_rate": 3.999516529744215e-06,
      "log_odds_chosen": 0.3937363922595978,
      "log_odds_ratio": -0.5289265513420105,
      "logits/chosen": 0.1900571584701538,
      "logits/rejected": 0.15361058712005615,
      "logps/chosen": -1.9516725540161133,
      "logps/rejected": -2.301271438598633,
      "loss": 2.1982,
      "nll_loss": 2.1452577114105225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19516725838184357,
      "rewards/margins": 0.03495991230010986,
      "rewards/rejected": -0.23012715578079224,
      "step": 136
    },
    {
      "epoch": 0.21591804570527975,
      "grad_norm": 0.43369004130363464,
      "learning_rate": 3.999388114506166e-06,
      "log_odds_chosen": 0.4344315826892853,
      "log_odds_ratio": -0.5094961524009705,
      "logits/chosen": 0.058895863592624664,
      "logits/rejected": 0.163179412484169,
      "logps/chosen": -1.844701886177063,
      "logps/rejected": -2.226527690887451,
      "loss": 2.0858,
      "nll_loss": 2.0348169803619385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18447020649909973,
      "rewards/margins": 0.03818256035447121,
      "rewards/rejected": -0.22265274822711945,
      "step": 137
    },
    {
      "epoch": 0.21749408983451538,
      "grad_norm": 0.42203620076179504,
      "learning_rate": 3.999244594845892e-06,
      "log_odds_chosen": 0.37491941452026367,
      "log_odds_ratio": -0.5347194671630859,
      "logits/chosen": 0.09584309160709381,
      "logits/rejected": 0.18220439553260803,
      "logps/chosen": -1.8879350423812866,
      "logps/rejected": -2.219787120819092,
      "loss": 2.1374,
      "nll_loss": 2.0839574337005615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18879351019859314,
      "rewards/margins": 0.03318518400192261,
      "rewards/rejected": -0.22197869420051575,
      "step": 138
    },
    {
      "epoch": 0.219070133963751,
      "grad_norm": 0.4215342104434967,
      "learning_rate": 3.999085971847616e-06,
      "log_odds_chosen": 0.2666381895542145,
      "log_odds_ratio": -0.579487144947052,
      "logits/chosen": 0.12761875987052917,
      "logits/rejected": 0.00037235766649246216,
      "logps/chosen": -1.945099115371704,
      "logps/rejected": -2.181492805480957,
      "loss": 2.1896,
      "nll_loss": 2.1316585540771484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19450990855693817,
      "rewards/margins": 0.023639345541596413,
      "rewards/rejected": -0.21814927458763123,
      "step": 139
    },
    {
      "epoch": 0.22064617809298662,
      "grad_norm": 0.4192501902580261,
      "learning_rate": 3.998912246709658e-06,
      "log_odds_chosen": 0.47762030363082886,
      "log_odds_ratio": -0.5048978328704834,
      "logits/chosen": 0.03178280219435692,
      "logits/rejected": 0.035381902009248734,
      "logps/chosen": -1.8466227054595947,
      "logps/rejected": -2.268432855606079,
      "loss": 2.0897,
      "nll_loss": 2.039208173751831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18466226756572723,
      "rewards/margins": 0.04218100756406784,
      "rewards/rejected": -0.22684328258037567,
      "step": 140
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 0.4541744589805603,
      "learning_rate": 3.9987234207444295e-06,
      "log_odds_chosen": 0.39642781019210815,
      "log_odds_ratio": -0.5283527374267578,
      "logits/chosen": 0.0909489318728447,
      "logits/rejected": 0.18739546835422516,
      "logps/chosen": -1.9560786485671997,
      "logps/rejected": -2.3097827434539795,
      "loss": 2.2049,
      "nll_loss": 2.1520698070526123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19560788571834564,
      "rewards/margins": 0.03537040203809738,
      "rewards/rejected": -0.23097828030586243,
      "step": 141
    },
    {
      "epoch": 0.22379826635145783,
      "grad_norm": 0.4312290847301483,
      "learning_rate": 3.998519495378419e-06,
      "log_odds_chosen": 0.3546152114868164,
      "log_odds_ratio": -0.5386707782745361,
      "logits/chosen": 0.18091727793216705,
      "logits/rejected": 0.33730435371398926,
      "logps/chosen": -2.0447652339935303,
      "logps/rejected": -2.3624267578125,
      "loss": 2.2652,
      "nll_loss": 2.2112996578216553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2044765055179596,
      "rewards/margins": 0.03176615759730339,
      "rewards/rejected": -0.2362426519393921,
      "step": 142
    },
    {
      "epoch": 0.22537431048069345,
      "grad_norm": 0.38687628507614136,
      "learning_rate": 3.998300472152187e-06,
      "log_odds_chosen": 0.2729724645614624,
      "log_odds_ratio": -0.5763713121414185,
      "logits/chosen": 0.03658361732959747,
      "logits/rejected": -0.03532174229621887,
      "logps/chosen": -1.8041529655456543,
      "logps/rejected": -2.040553092956543,
      "loss": 2.0568,
      "nll_loss": 1.9991142749786377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18041527271270752,
      "rewards/margins": 0.023640040308237076,
      "rewards/rejected": -0.2040553241968155,
      "step": 143
    },
    {
      "epoch": 0.22695035460992907,
      "grad_norm": 0.42334818840026855,
      "learning_rate": 3.998066352720347e-06,
      "log_odds_chosen": 0.4992850124835968,
      "log_odds_ratio": -0.481873095035553,
      "logits/chosen": 0.238363116979599,
      "logits/rejected": 0.23998966813087463,
      "logps/chosen": -1.9472233057022095,
      "logps/rejected": -2.3916687965393066,
      "loss": 2.1797,
      "nll_loss": 2.131523847579956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19472233951091766,
      "rewards/margins": 0.04444453865289688,
      "rewards/rejected": -0.23916688561439514,
      "step": 144
    },
    {
      "epoch": 0.2285263987391647,
      "grad_norm": 0.4056847393512726,
      "learning_rate": 3.997817138851562e-06,
      "log_odds_chosen": 0.3399352431297302,
      "log_odds_ratio": -0.5512011051177979,
      "logits/chosen": 0.1913917064666748,
      "logits/rejected": 0.20310351252555847,
      "logps/chosen": -2.0232315063476562,
      "logps/rejected": -2.327948570251465,
      "loss": 2.2587,
      "nll_loss": 2.203575372695923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20232316851615906,
      "rewards/margins": 0.030471697449684143,
      "rewards/rejected": -0.2327948659658432,
      "step": 145
    },
    {
      "epoch": 0.23010244286840031,
      "grad_norm": 0.390330046415329,
      "learning_rate": 3.997552832428522e-06,
      "log_odds_chosen": 0.4053119421005249,
      "log_odds_ratio": -0.5154557228088379,
      "logits/chosen": 0.05899511277675629,
      "logits/rejected": 0.17203162610530853,
      "logps/chosen": -1.8117233514785767,
      "logps/rejected": -2.1613264083862305,
      "loss": 2.0757,
      "nll_loss": 2.0241761207580566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18117234110832214,
      "rewards/margins": 0.03496028482913971,
      "rewards/rejected": -0.21613261103630066,
      "step": 146
    },
    {
      "epoch": 0.23167848699763594,
      "grad_norm": 0.3808334171772003,
      "learning_rate": 3.9972734354479366e-06,
      "log_odds_chosen": 0.36333751678466797,
      "log_odds_ratio": -0.5398846864700317,
      "logits/chosen": 0.1746935397386551,
      "logits/rejected": -0.06057439744472504,
      "logps/chosen": -1.7978198528289795,
      "logps/rejected": -2.109157085418701,
      "loss": 2.0488,
      "nll_loss": 1.994767189025879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1797819882631302,
      "rewards/margins": 0.031133724376559258,
      "rewards/rejected": -0.2109157145023346,
      "step": 147
    },
    {
      "epoch": 0.23325453112687156,
      "grad_norm": 0.4227812588214874,
      "learning_rate": 3.996978950020517e-06,
      "log_odds_chosen": 0.23635518550872803,
      "log_odds_ratio": -0.602321207523346,
      "logits/chosen": 0.30303770303726196,
      "logits/rejected": 0.26232045888900757,
      "logps/chosen": -2.013943910598755,
      "logps/rejected": -2.220210313796997,
      "loss": 2.2625,
      "nll_loss": 2.2022464275360107,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20139439404010773,
      "rewards/margins": 0.020626649260520935,
      "rewards/rejected": -0.22202105820178986,
      "step": 148
    },
    {
      "epoch": 0.23483057525610718,
      "grad_norm": 0.36287862062454224,
      "learning_rate": 3.996669378370959e-06,
      "log_odds_chosen": 0.601196825504303,
      "log_odds_ratio": -0.4497024714946747,
      "logits/chosen": 0.16156260669231415,
      "logits/rejected": 0.15708498656749725,
      "logps/chosen": -1.7733263969421387,
      "logps/rejected": -2.2993738651275635,
      "loss": 2.017,
      "nll_loss": 1.9719862937927246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17733265459537506,
      "rewards/margins": 0.052604734897613525,
      "rewards/rejected": -0.2299373894929886,
      "step": 149
    },
    {
      "epoch": 0.2364066193853428,
      "grad_norm": 0.4217900335788727,
      "learning_rate": 3.996344722837929e-06,
      "log_odds_chosen": 0.4981028735637665,
      "log_odds_ratio": -0.48042774200439453,
      "logits/chosen": 0.14997795224189758,
      "logits/rejected": 0.20861023664474487,
      "logps/chosen": -1.9517134428024292,
      "logps/rejected": -2.393977403640747,
      "loss": 2.1797,
      "nll_loss": 2.1316077709198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19517134130001068,
      "rewards/margins": 0.04422639310359955,
      "rewards/rejected": -0.23939774930477142,
      "step": 150
    },
    {
      "epoch": 0.23798266351457842,
      "grad_norm": 0.3475205898284912,
      "learning_rate": 3.996004985874043e-06,
      "log_odds_chosen": 0.2393593043088913,
      "log_odds_ratio": -0.5837403535842896,
      "logits/chosen": 0.2634306252002716,
      "logits/rejected": 0.15408116579055786,
      "logps/chosen": -1.8215903043746948,
      "logps/rejected": -2.025744676589966,
      "loss": 2.0775,
      "nll_loss": 2.0191402435302734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18215903639793396,
      "rewards/margins": 0.020415417850017548,
      "rewards/rejected": -0.2025744616985321,
      "step": 151
    },
    {
      "epoch": 0.23955870764381404,
      "grad_norm": 0.3991820216178894,
      "learning_rate": 3.995650170045855e-06,
      "log_odds_chosen": 0.27918171882629395,
      "log_odds_ratio": -0.5713668465614319,
      "logits/chosen": 0.19363005459308624,
      "logits/rejected": 0.11847929656505585,
      "logps/chosen": -1.8987635374069214,
      "logps/rejected": -2.141249656677246,
      "loss": 2.1421,
      "nll_loss": 2.0849664211273193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18987634778022766,
      "rewards/margins": 0.024248624220490456,
      "rewards/rejected": -0.21412499248981476,
      "step": 152
    },
    {
      "epoch": 0.24113475177304963,
      "grad_norm": 0.39748474955558777,
      "learning_rate": 3.995280278033825e-06,
      "log_odds_chosen": 0.16328397393226624,
      "log_odds_ratio": -0.6251372694969177,
      "logits/chosen": 0.26617446541786194,
      "logits/rejected": 0.23933932185173035,
      "logps/chosen": -2.0087883472442627,
      "logps/rejected": -2.1514151096343994,
      "loss": 2.2561,
      "nll_loss": 2.1936352252960205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20087884366512299,
      "rewards/margins": 0.014262663200497627,
      "rewards/rejected": -0.21514150500297546,
      "step": 153
    },
    {
      "epoch": 0.24271079590228525,
      "grad_norm": 0.3707138001918793,
      "learning_rate": 3.994895312632314e-06,
      "log_odds_chosen": 0.6531109809875488,
      "log_odds_ratio": -0.4291936159133911,
      "logits/chosen": 0.3400770127773285,
      "logits/rejected": -0.1956482231616974,
      "logps/chosen": -1.8738765716552734,
      "logps/rejected": -2.452014446258545,
      "loss": 2.1097,
      "nll_loss": 2.066800594329834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1873876452445984,
      "rewards/margins": 0.05781375616788864,
      "rewards/rejected": -0.24520139396190643,
      "step": 154
    },
    {
      "epoch": 0.24428684003152087,
      "grad_norm": 0.3588172495365143,
      "learning_rate": 3.994495276749549e-06,
      "log_odds_chosen": 0.5185203552246094,
      "log_odds_ratio": -0.47327202558517456,
      "logits/chosen": 0.23319880664348602,
      "logits/rejected": 0.1430215835571289,
      "logps/chosen": -1.8845221996307373,
      "logps/rejected": -2.341402053833008,
      "loss": 2.1094,
      "nll_loss": 2.062074661254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18845221400260925,
      "rewards/margins": 0.045687995851039886,
      "rewards/rejected": -0.23414020240306854,
      "step": 155
    },
    {
      "epoch": 0.2458628841607565,
      "grad_norm": 0.38072529435157776,
      "learning_rate": 3.994080173407612e-06,
      "log_odds_chosen": 0.2687567174434662,
      "log_odds_ratio": -0.5712822079658508,
      "logits/chosen": 0.2882783114910126,
      "logits/rejected": 0.13753741979599,
      "logps/chosen": -1.923572301864624,
      "logps/rejected": -2.1562812328338623,
      "loss": 2.1602,
      "nll_loss": 2.10306453704834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19235721230506897,
      "rewards/margins": 0.023270905017852783,
      "rewards/rejected": -0.21562813222408295,
      "step": 156
    },
    {
      "epoch": 0.24743892828999212,
      "grad_norm": 0.37946414947509766,
      "learning_rate": 3.993650005742409e-06,
      "log_odds_chosen": 0.27943286299705505,
      "log_odds_ratio": -0.5697324275970459,
      "logits/chosen": 0.22721070051193237,
      "logits/rejected": 0.17388474941253662,
      "logps/chosen": -1.9122453927993774,
      "logps/rejected": -2.155641555786133,
      "loss": 2.151,
      "nll_loss": 2.0940608978271484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19122453033924103,
      "rewards/margins": 0.02433963678777218,
      "rewards/rejected": -0.21556417644023895,
      "step": 157
    },
    {
      "epoch": 0.24901497241922774,
      "grad_norm": 0.3543010652065277,
      "learning_rate": 3.993204777003652e-06,
      "log_odds_chosen": 0.3954124450683594,
      "log_odds_ratio": -0.5226207375526428,
      "logits/chosen": 0.22299621999263763,
      "logits/rejected": 0.03944576531648636,
      "logps/chosen": -1.8209710121154785,
      "logps/rejected": -2.1653072834014893,
      "loss": 2.0513,
      "nll_loss": 1.9990218877792358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18209709227085114,
      "rewards/margins": 0.034433621913194656,
      "rewards/rejected": -0.2165307253599167,
      "step": 158
    },
    {
      "epoch": 0.25059101654846333,
      "grad_norm": 0.3736223876476288,
      "learning_rate": 3.992744490554832e-06,
      "log_odds_chosen": 0.38352489471435547,
      "log_odds_ratio": -0.5264863967895508,
      "logits/chosen": 0.3949679136276245,
      "logits/rejected": 0.19439592957496643,
      "logps/chosen": -1.8858449459075928,
      "logps/rejected": -2.221181631088257,
      "loss": 2.1334,
      "nll_loss": 2.08070969581604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18858450651168823,
      "rewards/margins": 0.033533647656440735,
      "rewards/rejected": -0.22211815416812897,
      "step": 159
    },
    {
      "epoch": 0.25216706067769895,
      "grad_norm": 0.3478967845439911,
      "learning_rate": 3.992269149873192e-06,
      "log_odds_chosen": 0.2978421151638031,
      "log_odds_ratio": -0.5619373917579651,
      "logits/chosen": 0.1769585907459259,
      "logits/rejected": 0.05092085525393486,
      "logps/chosen": -1.8340367078781128,
      "logps/rejected": -2.087353467941284,
      "loss": 2.0725,
      "nll_loss": 2.016280174255371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18340367078781128,
      "rewards/margins": 0.02533169835805893,
      "rewards/rejected": -0.2087353765964508,
      "step": 160
    },
    {
      "epoch": 0.25374310480693457,
      "grad_norm": 0.34116631746292114,
      "learning_rate": 3.991778758549705e-06,
      "log_odds_chosen": 0.2225290983915329,
      "log_odds_ratio": -0.5962461829185486,
      "logits/chosen": 0.4033081829547882,
      "logits/rejected": 0.2022503912448883,
      "logps/chosen": -1.9565954208374023,
      "logps/rejected": -2.15372633934021,
      "loss": 2.1758,
      "nll_loss": 2.1161608695983887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1956595480442047,
      "rewards/margins": 0.019713083282113075,
      "rewards/rejected": -0.21537262201309204,
      "step": 161
    },
    {
      "epoch": 0.2553191489361702,
      "grad_norm": 0.36636391282081604,
      "learning_rate": 3.9912733202890415e-06,
      "log_odds_chosen": 0.3752463161945343,
      "log_odds_ratio": -0.5345346927642822,
      "logits/chosen": 0.28393152356147766,
      "logits/rejected": 0.05336347967386246,
      "logps/chosen": -1.8976832628250122,
      "logps/rejected": -2.226705312728882,
      "loss": 2.1285,
      "nll_loss": 2.0750443935394287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18976832926273346,
      "rewards/margins": 0.03290221095085144,
      "rewards/rejected": -0.2226705402135849,
      "step": 162
    },
    {
      "epoch": 0.2568951930654058,
      "grad_norm": 0.3403889834880829,
      "learning_rate": 3.990752838909548e-06,
      "log_odds_chosen": 0.20657047629356384,
      "log_odds_ratio": -0.6020048260688782,
      "logits/chosen": 0.36604052782058716,
      "logits/rejected": 0.1470409482717514,
      "logps/chosen": -1.9812164306640625,
      "logps/rejected": -2.162238121032715,
      "loss": 2.2041,
      "nll_loss": 2.143928289413452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19812165200710297,
      "rewards/margins": 0.018102185800671577,
      "rewards/rejected": -0.2162238508462906,
      "step": 163
    },
    {
      "epoch": 0.25847123719464143,
      "grad_norm": 0.3617730736732483,
      "learning_rate": 3.990217318343213e-06,
      "log_odds_chosen": 0.6242728233337402,
      "log_odds_ratio": -0.4343191981315613,
      "logits/chosen": 0.33031949400901794,
      "logits/rejected": -0.01348424144089222,
      "logps/chosen": -2.0581400394439697,
      "logps/rejected": -2.6227612495422363,
      "loss": 2.2678,
      "nll_loss": 2.22434139251709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20581400394439697,
      "rewards/margins": 0.056462109088897705,
      "rewards/rejected": -0.2622761130332947,
      "step": 164
    },
    {
      "epoch": 0.26004728132387706,
      "grad_norm": 0.3566831946372986,
      "learning_rate": 3.989666762635637e-06,
      "log_odds_chosen": 0.2842005491256714,
      "log_odds_ratio": -0.5646076798439026,
      "logits/chosen": 0.3204185962677002,
      "logits/rejected": 0.14812150597572327,
      "logps/chosen": -1.8993555307388306,
      "logps/rejected": -2.145724296569824,
      "loss": 2.1492,
      "nll_loss": 2.0927205085754395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18993555009365082,
      "rewards/margins": 0.024636875838041306,
      "rewards/rejected": -0.21457241475582123,
      "step": 165
    },
    {
      "epoch": 0.2616233254531127,
      "grad_norm": 0.34587928652763367,
      "learning_rate": 3.9891011759460056e-06,
      "log_odds_chosen": 0.38712218403816223,
      "log_odds_ratio": -0.5237506628036499,
      "logits/chosen": 0.30775323510169983,
      "logits/rejected": 0.1324882060289383,
      "logps/chosen": -1.8900407552719116,
      "logps/rejected": -2.2226169109344482,
      "loss": 2.1266,
      "nll_loss": 2.0742549896240234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1890040785074234,
      "rewards/margins": 0.03325759619474411,
      "rewards/rejected": -0.2222616821527481,
      "step": 166
    },
    {
      "epoch": 0.2631993695823483,
      "grad_norm": 0.32432565093040466,
      "learning_rate": 3.988520562547057e-06,
      "log_odds_chosen": 0.4879869818687439,
      "log_odds_ratio": -0.49468889832496643,
      "logits/chosen": 0.3148024082183838,
      "logits/rejected": 0.025401366874575615,
      "logps/chosen": -1.8358017206192017,
      "logps/rejected": -2.2611985206604004,
      "loss": 2.0526,
      "nll_loss": 2.003171682357788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1835801899433136,
      "rewards/margins": 0.04253965616226196,
      "rewards/rejected": -0.22611983120441437,
      "step": 167
    },
    {
      "epoch": 0.2647754137115839,
      "grad_norm": 0.3420664668083191,
      "learning_rate": 3.987924926825047e-06,
      "log_odds_chosen": 0.5040227174758911,
      "log_odds_ratio": -0.4986487627029419,
      "logits/chosen": 0.3197277784347534,
      "logits/rejected": 0.055843938142061234,
      "logps/chosen": -1.9207936525344849,
      "logps/rejected": -2.368576765060425,
      "loss": 2.1598,
      "nll_loss": 2.1099777221679688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1920793652534485,
      "rewards/margins": 0.04477829486131668,
      "rewards/rejected": -0.23685765266418457,
      "step": 168
    },
    {
      "epoch": 0.26635145784081954,
      "grad_norm": 0.3396807909011841,
      "learning_rate": 3.98731427327972e-06,
      "log_odds_chosen": 0.4706634283065796,
      "log_odds_ratio": -0.4895702004432678,
      "logits/chosen": 0.2792813777923584,
      "logits/rejected": -0.0839325338602066,
      "logps/chosen": -1.8512392044067383,
      "logps/rejected": -2.2608284950256348,
      "loss": 2.0695,
      "nll_loss": 2.020576238632202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18512390553951263,
      "rewards/margins": 0.0409589484333992,
      "rewards/rejected": -0.22608286142349243,
      "step": 169
    },
    {
      "epoch": 0.26792750197005516,
      "grad_norm": 0.32379212975502014,
      "learning_rate": 3.986688606524273e-06,
      "log_odds_chosen": 0.3634461760520935,
      "log_odds_ratio": -0.543432354927063,
      "logits/chosen": 0.3800685703754425,
      "logits/rejected": -0.3125303387641907,
      "logps/chosen": -1.8691346645355225,
      "logps/rejected": -2.190423011779785,
      "loss": 2.0918,
      "nll_loss": 2.0374691486358643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18691349029541016,
      "rewards/margins": 0.032128822058439255,
      "rewards/rejected": -0.21904230117797852,
      "step": 170
    },
    {
      "epoch": 0.2695035460992908,
      "grad_norm": 0.3257015347480774,
      "learning_rate": 3.986047931285315e-06,
      "log_odds_chosen": 0.6643016338348389,
      "log_odds_ratio": -0.42536190152168274,
      "logits/chosen": 0.2640674412250519,
      "logits/rejected": -0.16231495141983032,
      "logps/chosen": -1.7854793071746826,
      "logps/rejected": -2.365164279937744,
      "loss": 1.9975,
      "nll_loss": 1.9549524784088135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1785479485988617,
      "rewards/margins": 0.057968489825725555,
      "rewards/rejected": -0.23651644587516785,
      "step": 171
    },
    {
      "epoch": 0.2710795902285264,
      "grad_norm": 0.3187534511089325,
      "learning_rate": 3.985392252402847e-06,
      "log_odds_chosen": 0.21815888583660126,
      "log_odds_ratio": -0.5928647518157959,
      "logits/chosen": 0.24203622341156006,
      "logits/rejected": -0.04228699579834938,
      "logps/chosen": -1.77311110496521,
      "logps/rejected": -1.9570823907852173,
      "loss": 2.0022,
      "nll_loss": 1.942939043045044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17731110751628876,
      "rewards/margins": 0.018397128209471703,
      "rewards/rejected": -0.1957082450389862,
      "step": 172
    },
    {
      "epoch": 0.272655634357762,
      "grad_norm": 0.32601413130760193,
      "learning_rate": 3.984721574830206e-06,
      "log_odds_chosen": 0.2795952558517456,
      "log_odds_ratio": -0.566702127456665,
      "logits/chosen": 0.32929977774620056,
      "logits/rejected": -0.1160498857498169,
      "logps/chosen": -1.8864023685455322,
      "logps/rejected": -2.128418445587158,
      "loss": 2.1073,
      "nll_loss": 2.050626039505005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18864023685455322,
      "rewards/margins": 0.02420159988105297,
      "rewards/rejected": -0.21284182369709015,
      "step": 173
    },
    {
      "epoch": 0.27423167848699764,
      "grad_norm": 0.30667030811309814,
      "learning_rate": 3.984035903634041e-06,
      "log_odds_chosen": 0.3065723478794098,
      "log_odds_ratio": -0.560809850692749,
      "logits/chosen": 0.37695005536079407,
      "logits/rejected": -0.13857263326644897,
      "logps/chosen": -1.8567309379577637,
      "logps/rejected": -2.1228272914886475,
      "loss": 2.0624,
      "nll_loss": 2.0063276290893555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1856730878353119,
      "rewards/margins": 0.026609646156430244,
      "rewards/rejected": -0.21228273212909698,
      "step": 174
    },
    {
      "epoch": 0.27580772261623326,
      "grad_norm": 0.32427388429641724,
      "learning_rate": 3.983335243994273e-06,
      "log_odds_chosen": 0.47853925824165344,
      "log_odds_ratio": -0.4986327290534973,
      "logits/chosen": 0.38745561242103577,
      "logits/rejected": -0.0722002163529396,
      "logps/chosen": -1.8594340085983276,
      "logps/rejected": -2.2790563106536865,
      "loss": 2.0772,
      "nll_loss": 2.0272879600524902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18594340980052948,
      "rewards/margins": 0.04196225106716156,
      "rewards/rejected": -0.22790564596652985,
      "step": 175
    },
    {
      "epoch": 0.2773837667454689,
      "grad_norm": 0.3393646776676178,
      "learning_rate": 3.982619601204049e-06,
      "log_odds_chosen": 0.3384026885032654,
      "log_odds_ratio": -0.5461087226867676,
      "logits/chosen": 0.32837018370628357,
      "logits/rejected": -0.0760173350572586,
      "logps/chosen": -1.928531289100647,
      "logps/rejected": -2.2252705097198486,
      "loss": 2.1483,
      "nll_loss": 2.0936667919158936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1928531527519226,
      "rewards/margins": 0.029673907905817032,
      "rewards/rejected": -0.22252705693244934,
      "step": 176
    },
    {
      "epoch": 0.2789598108747045,
      "grad_norm": 0.30985409021377563,
      "learning_rate": 3.9818889806697085e-06,
      "log_odds_chosen": 0.417074590921402,
      "log_odds_ratio": -0.5178275108337402,
      "logits/chosen": 0.44880637526512146,
      "logits/rejected": -0.11007529497146606,
      "logps/chosen": -1.9631050825119019,
      "logps/rejected": -2.3332266807556152,
      "loss": 2.1563,
      "nll_loss": 2.1044764518737793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19631052017211914,
      "rewards/margins": 0.037012144923210144,
      "rewards/rejected": -0.2333226501941681,
      "step": 177
    },
    {
      "epoch": 0.2805358550039401,
      "grad_norm": 0.3177052438259125,
      "learning_rate": 3.98114338791074e-06,
      "log_odds_chosen": 0.545357882976532,
      "log_odds_ratio": -0.4829763174057007,
      "logits/chosen": 0.3017576038837433,
      "logits/rejected": -0.15456542372703552,
      "logps/chosen": -1.8234596252441406,
      "logps/rejected": -2.3049373626708984,
      "loss": 2.047,
      "nll_loss": 1.9987092018127441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18234597146511078,
      "rewards/margins": 0.0481477752327919,
      "rewards/rejected": -0.2304937243461609,
      "step": 178
    },
    {
      "epoch": 0.28211189913317575,
      "grad_norm": 0.31176838278770447,
      "learning_rate": 3.980382828559742e-06,
      "log_odds_chosen": 0.3622802197933197,
      "log_odds_ratio": -0.5349768400192261,
      "logits/chosen": 0.3076533377170563,
      "logits/rejected": -0.15830329060554504,
      "logps/chosen": -1.8622955083847046,
      "logps/rejected": -2.1759631633758545,
      "loss": 2.0629,
      "nll_loss": 2.009382963180542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18622955679893494,
      "rewards/margins": 0.03136677294969559,
      "rewards/rejected": -0.21759635210037231,
      "step": 179
    },
    {
      "epoch": 0.28368794326241137,
      "grad_norm": 0.3124650716781616,
      "learning_rate": 3.9796073083623774e-06,
      "log_odds_chosen": 0.37124893069267273,
      "log_odds_ratio": -0.5417748093605042,
      "logits/chosen": 0.32632479071617126,
      "logits/rejected": -0.19913263618946075,
      "logps/chosen": -1.9367098808288574,
      "logps/rejected": -2.2628095149993896,
      "loss": 2.1354,
      "nll_loss": 2.08120059967041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19367098808288574,
      "rewards/margins": 0.0326099693775177,
      "rewards/rejected": -0.22628095746040344,
      "step": 180
    },
    {
      "epoch": 0.285263987391647,
      "grad_norm": 0.3195948302745819,
      "learning_rate": 3.978816833177329e-06,
      "log_odds_chosen": 0.3703266978263855,
      "log_odds_ratio": -0.5306106805801392,
      "logits/chosen": 0.42076432704925537,
      "logits/rejected": -0.02614203840494156,
      "logps/chosen": -1.8599867820739746,
      "logps/rejected": -2.1812684535980225,
      "loss": 2.0678,
      "nll_loss": 2.014716386795044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18599866330623627,
      "rewards/margins": 0.03212819620966911,
      "rewards/rejected": -0.21812686324119568,
      "step": 181
    },
    {
      "epoch": 0.2868400315208826,
      "grad_norm": 0.3143973648548126,
      "learning_rate": 3.978011408976261e-06,
      "log_odds_chosen": 0.33120378851890564,
      "log_odds_ratio": -0.5492135286331177,
      "logits/chosen": 0.3848988711833954,
      "logits/rejected": -0.14226217567920685,
      "logps/chosen": -1.8885282278060913,
      "logps/rejected": -2.1768221855163574,
      "loss": 2.1045,
      "nll_loss": 2.049586296081543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18885281682014465,
      "rewards/margins": 0.02882939763367176,
      "rewards/rejected": -0.21768222749233246,
      "step": 182
    },
    {
      "epoch": 0.28841607565011823,
      "grad_norm": 0.36059871315956116,
      "learning_rate": 3.9771910418437674e-06,
      "log_odds_chosen": 0.2940455675125122,
      "log_odds_ratio": -0.5578598976135254,
      "logits/chosen": 0.38924431800842285,
      "logits/rejected": 0.05839519202709198,
      "logps/chosen": -1.926640272140503,
      "logps/rejected": -2.183753728866577,
      "loss": 2.1334,
      "nll_loss": 2.0775983333587646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1926640272140503,
      "rewards/margins": 0.02571135386824608,
      "rewards/rejected": -0.21837536990642548,
      "step": 183
    },
    {
      "epoch": 0.2899921197793538,
      "grad_norm": 0.31750741600990295,
      "learning_rate": 3.976355737977332e-06,
      "log_odds_chosen": 0.3702757656574249,
      "log_odds_ratio": -0.5352032780647278,
      "logits/chosen": 0.39942264556884766,
      "logits/rejected": -0.12641888856887817,
      "logps/chosen": -1.8630297183990479,
      "logps/rejected": -2.1844112873077393,
      "loss": 2.0737,
      "nll_loss": 2.0201988220214844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18630295991897583,
      "rewards/margins": 0.032138168811798096,
      "rewards/rejected": -0.2184411585330963,
      "step": 184
    },
    {
      "epoch": 0.2915681639085894,
      "grad_norm": 0.2944638133049011,
      "learning_rate": 3.975505503687274e-06,
      "log_odds_chosen": 0.31336691975593567,
      "log_odds_ratio": -0.5550580620765686,
      "logits/chosen": 0.43454480171203613,
      "logits/rejected": 0.02699423022568226,
      "logps/chosen": -1.8394372463226318,
      "logps/rejected": -2.110544204711914,
      "loss": 2.0454,
      "nll_loss": 1.9898747205734253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1839437335729599,
      "rewards/margins": 0.027110688388347626,
      "rewards/rejected": -0.21105441451072693,
      "step": 185
    },
    {
      "epoch": 0.29314420803782504,
      "grad_norm": 0.28209826350212097,
      "learning_rate": 3.974640345396708e-06,
      "log_odds_chosen": 0.5129460096359253,
      "log_odds_ratio": -0.4778994023799896,
      "logits/chosen": 0.2777142822742462,
      "logits/rejected": -0.47065022587776184,
      "logps/chosen": -1.806014060974121,
      "logps/rejected": -2.2508716583251953,
      "loss": 2.0079,
      "nll_loss": 1.9601199626922607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18060140311717987,
      "rewards/margins": 0.04448575899004936,
      "rewards/rejected": -0.22508715093135834,
      "step": 186
    },
    {
      "epoch": 0.29472025216706066,
      "grad_norm": 0.2899785339832306,
      "learning_rate": 3.9737602696414925e-06,
      "log_odds_chosen": 0.14901578426361084,
      "log_odds_ratio": -0.6323056221008301,
      "logits/chosen": 0.3966515362262726,
      "logits/rejected": -0.2466234564781189,
      "logps/chosen": -1.9014618396759033,
      "logps/rejected": -2.0296077728271484,
      "loss": 2.0863,
      "nll_loss": 2.0230603218078613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19014619290828705,
      "rewards/margins": 0.012814579531550407,
      "rewards/rejected": -0.2029607594013214,
      "step": 187
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 0.2821260690689087,
      "learning_rate": 3.972865283070179e-06,
      "log_odds_chosen": 0.5185416340827942,
      "log_odds_ratio": -0.4854019284248352,
      "logits/chosen": 0.35249727964401245,
      "logits/rejected": -0.3178330361843109,
      "logps/chosen": -1.7990273237228394,
      "logps/rejected": -2.2565367221832275,
      "loss": 2.0084,
      "nll_loss": 1.9598151445388794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17990271747112274,
      "rewards/margins": 0.0457509309053421,
      "rewards/rejected": -0.22565364837646484,
      "step": 188
    },
    {
      "epoch": 0.2978723404255319,
      "grad_norm": 0.2892349660396576,
      "learning_rate": 3.971955392443965e-06,
      "log_odds_chosen": 0.3767319917678833,
      "log_odds_ratio": -0.528398871421814,
      "logits/chosen": 0.39535295963287354,
      "logits/rejected": -0.14428094029426575,
      "logps/chosen": -1.8341965675354004,
      "logps/rejected": -2.162943124771118,
      "loss": 2.0321,
      "nll_loss": 1.9792832136154175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18341965973377228,
      "rewards/margins": 0.03287467360496521,
      "rewards/rejected": -0.21629434823989868,
      "step": 189
    },
    {
      "epoch": 0.2994483845547675,
      "grad_norm": 0.5001341104507446,
      "learning_rate": 3.971030604636637e-06,
      "log_odds_chosen": 0.2465718537569046,
      "log_odds_ratio": -0.5827715992927551,
      "logits/chosen": 0.4445749819278717,
      "logits/rejected": -0.11869044601917267,
      "logps/chosen": -1.8594155311584473,
      "logps/rejected": -2.070815324783325,
      "loss": 2.0563,
      "nll_loss": 1.9980722665786743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18594154715538025,
      "rewards/margins": 0.021139992401003838,
      "rewards/rejected": -0.20708155632019043,
      "step": 190
    },
    {
      "epoch": 0.30102442868400314,
      "grad_norm": 0.3018903434276581,
      "learning_rate": 3.970090926634526e-06,
      "log_odds_chosen": 0.31170162558555603,
      "log_odds_ratio": -0.5536643862724304,
      "logits/chosen": 0.49709048867225647,
      "logits/rejected": -0.3552049696445465,
      "logps/chosen": -1.9083786010742188,
      "logps/rejected": -2.1798431873321533,
      "loss": 2.1132,
      "nll_loss": 2.0578255653381348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19083787500858307,
      "rewards/margins": 0.027146445587277412,
      "rewards/rejected": -0.21798431873321533,
      "step": 191
    },
    {
      "epoch": 0.30260047281323876,
      "grad_norm": 0.3082922399044037,
      "learning_rate": 3.9691363655364526e-06,
      "log_odds_chosen": 0.41405007243156433,
      "log_odds_ratio": -0.5227676630020142,
      "logits/chosen": 0.4238288998603821,
      "logits/rejected": -0.18818999826908112,
      "logps/chosen": -1.8893095254898071,
      "logps/rejected": -2.254439353942871,
      "loss": 2.0958,
      "nll_loss": 2.0435335636138916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1889309585094452,
      "rewards/margins": 0.036512985825538635,
      "rewards/rejected": -0.22544392943382263,
      "step": 192
    },
    {
      "epoch": 0.3041765169424744,
      "grad_norm": 0.2771205008029938,
      "learning_rate": 3.968166928553666e-06,
      "log_odds_chosen": 0.32098525762557983,
      "log_odds_ratio": -0.5535095930099487,
      "logits/chosen": 0.44575732946395874,
      "logits/rejected": -0.3498935103416443,
      "logps/chosen": -1.8364524841308594,
      "logps/rejected": -2.111985445022583,
      "loss": 2.0581,
      "nll_loss": 2.002781391143799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18364526331424713,
      "rewards/margins": 0.02755330130457878,
      "rewards/rejected": -0.21119855344295502,
      "step": 193
    },
    {
      "epoch": 0.30575256107171,
      "grad_norm": 0.28915470838546753,
      "learning_rate": 3.967182623009804e-06,
      "log_odds_chosen": 0.4584325850009918,
      "log_odds_ratio": -0.4974234998226166,
      "logits/chosen": 0.39246881008148193,
      "logits/rejected": -0.22741518914699554,
      "logps/chosen": -1.8555591106414795,
      "logps/rejected": -2.257932424545288,
      "loss": 2.0439,
      "nll_loss": 1.9941781759262085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18555592000484467,
      "rewards/margins": 0.040237318724393845,
      "rewards/rejected": -0.22579322755336761,
      "step": 194
    },
    {
      "epoch": 0.3073286052009456,
      "grad_norm": 0.31561747193336487,
      "learning_rate": 3.966183456340821e-06,
      "log_odds_chosen": 0.20420649647712708,
      "log_odds_ratio": -0.5991290211677551,
      "logits/chosen": 0.4658154547214508,
      "logits/rejected": -0.2021852731704712,
      "logps/chosen": -1.907456874847412,
      "logps/rejected": -2.082449197769165,
      "loss": 2.1109,
      "nll_loss": 2.0510265827178955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19074568152427673,
      "rewards/margins": 0.01749921590089798,
      "rewards/rejected": -0.2082449197769165,
      "step": 195
    },
    {
      "epoch": 0.30890464933018125,
      "grad_norm": 0.31182610988616943,
      "learning_rate": 3.965169436094947e-06,
      "log_odds_chosen": 0.3316318392753601,
      "log_odds_ratio": -0.554113507270813,
      "logits/chosen": 0.46127960085868835,
      "logits/rejected": -0.33384498953819275,
      "logps/chosen": -1.7970610857009888,
      "logps/rejected": -2.084744453430176,
      "loss": 2.0109,
      "nll_loss": 1.955501914024353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17970611155033112,
      "rewards/margins": 0.028768306598067284,
      "rewards/rejected": -0.20847442746162415,
      "step": 196
    },
    {
      "epoch": 0.31048069345941687,
      "grad_norm": 0.2999548017978668,
      "learning_rate": 3.964140569932618e-06,
      "log_odds_chosen": 0.3349445164203644,
      "log_odds_ratio": -0.5604900121688843,
      "logits/chosen": 0.3906029164791107,
      "logits/rejected": -0.4700361490249634,
      "logps/chosen": -1.8970091342926025,
      "logps/rejected": -2.1866891384124756,
      "loss": 2.0889,
      "nll_loss": 2.032803773880005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1897009164094925,
      "rewards/margins": 0.028968006372451782,
      "rewards/rejected": -0.21866890788078308,
      "step": 197
    },
    {
      "epoch": 0.3120567375886525,
      "grad_norm": 0.28372958302497864,
      "learning_rate": 3.9630968656264285e-06,
      "log_odds_chosen": 0.43718868494033813,
      "log_odds_ratio": -0.5063585042953491,
      "logits/chosen": 0.41493895649909973,
      "logits/rejected": -0.177265927195549,
      "logps/chosen": -1.820644497871399,
      "logps/rejected": -2.1998634338378906,
      "loss": 2.0096,
      "nll_loss": 1.959004521369934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1820644587278366,
      "rewards/margins": 0.037921883165836334,
      "rewards/rejected": -0.21998633444309235,
      "step": 198
    },
    {
      "epoch": 0.3136327817178881,
      "grad_norm": 0.30846837162971497,
      "learning_rate": 3.962038331061065e-06,
      "log_odds_chosen": 0.19350767135620117,
      "log_odds_ratio": -0.623175859451294,
      "logits/chosen": 0.5208650231361389,
      "logits/rejected": -0.31634849309921265,
      "logps/chosen": -1.888583779335022,
      "logps/rejected": -2.0645432472229004,
      "loss": 2.1024,
      "nll_loss": 2.0401272773742676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18885837495326996,
      "rewards/margins": 0.017595931887626648,
      "rewards/rejected": -0.2064543068408966,
      "step": 199
    },
    {
      "epoch": 0.31520882584712373,
      "grad_norm": 0.30154353380203247,
      "learning_rate": 3.96096497423325e-06,
      "log_odds_chosen": 0.21430566906929016,
      "log_odds_ratio": -0.5975645184516907,
      "logits/chosen": 0.5608594417572021,
      "logits/rejected": 0.0020670928061008453,
      "logps/chosen": -1.7950149774551392,
      "logps/rejected": -1.978040099143982,
      "loss": 2.0287,
      "nll_loss": 1.9689255952835083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1795014888048172,
      "rewards/margins": 0.01830250397324562,
      "rewards/rejected": -0.19780398905277252,
      "step": 200
    },
    {
      "epoch": 0.31678486997635935,
      "grad_norm": 0.28481966257095337,
      "learning_rate": 3.959876803251684e-06,
      "log_odds_chosen": 0.3561375141143799,
      "log_odds_ratio": -0.5370092988014221,
      "logits/chosen": 0.3803846538066864,
      "logits/rejected": -0.2256171554327011,
      "logps/chosen": -1.7965466976165771,
      "logps/rejected": -2.102306365966797,
      "loss": 1.9801,
      "nll_loss": 1.9264239072799683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17965465784072876,
      "rewards/margins": 0.030575988814234734,
      "rewards/rejected": -0.21023064851760864,
      "step": 201
    },
    {
      "epoch": 0.31836091410559497,
      "grad_norm": 0.28623512387275696,
      "learning_rate": 3.958773826336977e-06,
      "log_odds_chosen": 0.23698343336582184,
      "log_odds_ratio": -0.5838424563407898,
      "logits/chosen": 0.49230071902275085,
      "logits/rejected": -0.10177897661924362,
      "logps/chosen": -1.84354829788208,
      "logps/rejected": -2.0470781326293945,
      "loss": 2.048,
      "nll_loss": 1.989640235900879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18435484170913696,
      "rewards/margins": 0.02035299874842167,
      "rewards/rejected": -0.20470783114433289,
      "step": 202
    },
    {
      "epoch": 0.3199369582348306,
      "grad_norm": 0.29585060477256775,
      "learning_rate": 3.957656051821592e-06,
      "log_odds_chosen": 0.25145307183265686,
      "log_odds_ratio": -0.5859741568565369,
      "logits/chosen": 0.539005696773529,
      "logits/rejected": -0.3879539668560028,
      "logps/chosen": -1.8778821229934692,
      "logps/rejected": -2.1008167266845703,
      "loss": 2.0589,
      "nll_loss": 2.0003015995025635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1877882331609726,
      "rewards/margins": 0.02229345589876175,
      "rewards/rejected": -0.21008168160915375,
      "step": 203
    },
    {
      "epoch": 0.3215130023640662,
      "grad_norm": 0.3141014575958252,
      "learning_rate": 3.956523488149783e-06,
      "log_odds_chosen": 0.35386207699775696,
      "log_odds_ratio": -0.5435526371002197,
      "logits/chosen": 0.5516231060028076,
      "logits/rejected": -0.11498698592185974,
      "logps/chosen": -1.9893256425857544,
      "logps/rejected": -2.303192138671875,
      "loss": 2.1781,
      "nll_loss": 2.1237025260925293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19893257319927216,
      "rewards/margins": 0.03138667345046997,
      "rewards/rejected": -0.23031923174858093,
      "step": 204
    },
    {
      "epoch": 0.32308904649330183,
      "grad_norm": 0.30985650420188904,
      "learning_rate": 3.9553761438775285e-06,
      "log_odds_chosen": 0.5383609533309937,
      "log_odds_ratio": -0.4667017161846161,
      "logits/chosen": 0.4218091368675232,
      "logits/rejected": -0.1754085123538971,
      "logps/chosen": -1.719486951828003,
      "logps/rejected": -2.184528112411499,
      "loss": 1.9139,
      "nll_loss": 1.8672513961791992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17194871604442596,
      "rewards/margins": 0.046504102647304535,
      "rewards/rejected": -0.2184528112411499,
      "step": 205
    },
    {
      "epoch": 0.32466509062253746,
      "grad_norm": 0.2680577039718628,
      "learning_rate": 3.954214027672465e-06,
      "log_odds_chosen": 0.5561447143554688,
      "log_odds_ratio": -0.46768665313720703,
      "logits/chosen": 0.3986890912055969,
      "logits/rejected": -0.5810946226119995,
      "logps/chosen": -1.7915854454040527,
      "logps/rejected": -2.280474901199341,
      "loss": 1.9719,
      "nll_loss": 1.9251141548156738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.179158553481102,
      "rewards/margins": 0.048888955265283585,
      "rewards/rejected": -0.22804749011993408,
      "step": 206
    },
    {
      "epoch": 0.3262411347517731,
      "grad_norm": 0.29800331592559814,
      "learning_rate": 3.953037148313825e-06,
      "log_odds_chosen": 0.2825692296028137,
      "log_odds_ratio": -0.5700769424438477,
      "logits/chosen": 0.5266653895378113,
      "logits/rejected": -0.22400972247123718,
      "logps/chosen": -1.8013321161270142,
      "logps/rejected": -2.048236131668091,
      "loss": 2.0184,
      "nll_loss": 1.9614076614379883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18013322353363037,
      "rewards/margins": 0.024690402671694756,
      "rewards/rejected": -0.20482362806797028,
      "step": 207
    },
    {
      "epoch": 0.32781717888100864,
      "grad_norm": 0.30844083428382874,
      "learning_rate": 3.951845514692371e-06,
      "log_odds_chosen": 0.36232998967170715,
      "log_odds_ratio": -0.5381972789764404,
      "logits/chosen": 0.4263172149658203,
      "logits/rejected": -0.17897063493728638,
      "logps/chosen": -1.856035828590393,
      "logps/rejected": -2.1717522144317627,
      "loss": 2.0606,
      "nll_loss": 2.0067667961120605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18560358881950378,
      "rewards/margins": 0.031571630388498306,
      "rewards/rejected": -0.2171752154827118,
      "step": 208
    },
    {
      "epoch": 0.32939322301024426,
      "grad_norm": 0.31168684363365173,
      "learning_rate": 3.950639135810325e-06,
      "log_odds_chosen": 0.36213281750679016,
      "log_odds_ratio": -0.5349076390266418,
      "logits/chosen": 0.5617266893386841,
      "logits/rejected": -0.08795854449272156,
      "logps/chosen": -1.9983832836151123,
      "logps/rejected": -2.31766414642334,
      "loss": 2.1698,
      "nll_loss": 2.1163430213928223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1998383104801178,
      "rewards/margins": 0.031928107142448425,
      "rewards/rejected": -0.23176643252372742,
      "step": 209
    },
    {
      "epoch": 0.3309692671394799,
      "grad_norm": 0.2974774241447449,
      "learning_rate": 3.9494180207813044e-06,
      "log_odds_chosen": 0.28252044320106506,
      "log_odds_ratio": -0.5777407884597778,
      "logits/chosen": 0.5435987710952759,
      "logits/rejected": -0.44823166728019714,
      "logps/chosen": -1.8489923477172852,
      "logps/rejected": -2.093599796295166,
      "loss": 2.0489,
      "nll_loss": 1.9911445379257202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1848992556333542,
      "rewards/margins": 0.024460751563310623,
      "rewards/rejected": -0.20935998857021332,
      "step": 210
    },
    {
      "epoch": 0.3325453112687155,
      "grad_norm": 0.29235726594924927,
      "learning_rate": 3.948182178830249e-06,
      "log_odds_chosen": 0.3494931757450104,
      "log_odds_ratio": -0.5623037815093994,
      "logits/chosen": 0.5011661648750305,
      "logits/rejected": -0.5379094481468201,
      "logps/chosen": -1.8723922967910767,
      "logps/rejected": -2.184168577194214,
      "loss": 2.0515,
      "nll_loss": 1.9952374696731567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18723922967910767,
      "rewards/margins": 0.03117763251066208,
      "rewards/rejected": -0.21841685473918915,
      "step": 211
    },
    {
      "epoch": 0.3341213553979511,
      "grad_norm": 0.30690592527389526,
      "learning_rate": 3.9469316192933545e-06,
      "log_odds_chosen": 0.5762660503387451,
      "log_odds_ratio": -0.4586283564567566,
      "logits/chosen": 0.40061119198799133,
      "logits/rejected": -0.19122405350208282,
      "logps/chosen": -1.7368558645248413,
      "logps/rejected": -2.236180305480957,
      "loss": 1.934,
      "nll_loss": 1.8881280422210693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17368561029434204,
      "rewards/margins": 0.04993244633078575,
      "rewards/rejected": -0.2236180454492569,
      "step": 212
    },
    {
      "epoch": 0.33569739952718675,
      "grad_norm": 0.2981858253479004,
      "learning_rate": 3.945666351618001e-06,
      "log_odds_chosen": 0.24784377217292786,
      "log_odds_ratio": -0.5805960893630981,
      "logits/chosen": 0.4917218089103699,
      "logits/rejected": -0.2779148519039154,
      "logps/chosen": -1.7779290676116943,
      "logps/rejected": -1.9896657466888428,
      "loss": 1.9798,
      "nll_loss": 1.921708583831787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17779293656349182,
      "rewards/margins": 0.021173639222979546,
      "rewards/rejected": -0.19896656274795532,
      "step": 213
    },
    {
      "epoch": 0.33727344365642237,
      "grad_norm": 0.30391067266464233,
      "learning_rate": 3.9443863853626825e-06,
      "log_odds_chosen": 0.34453117847442627,
      "log_odds_ratio": -0.5483933091163635,
      "logits/chosen": 0.5546002388000488,
      "logits/rejected": -0.3275584876537323,
      "logps/chosen": -1.7622315883636475,
      "logps/rejected": -2.0517117977142334,
      "loss": 1.9813,
      "nll_loss": 1.9264605045318604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17622317373752594,
      "rewards/margins": 0.028948020190000534,
      "rewards/rejected": -0.20517119765281677,
      "step": 214
    },
    {
      "epoch": 0.338849487785658,
      "grad_norm": 0.2940407991409302,
      "learning_rate": 3.943091730196931e-06,
      "log_odds_chosen": 0.24225011467933655,
      "log_odds_ratio": -0.5915079712867737,
      "logits/chosen": 0.5921890139579773,
      "logits/rejected": -0.13432368636131287,
      "logps/chosen": -1.7511825561523438,
      "logps/rejected": -1.9615026712417603,
      "loss": 1.9606,
      "nll_loss": 1.9014678001403809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17511828243732452,
      "rewards/margins": 0.02103200927376747,
      "rewards/rejected": -0.1961502581834793,
      "step": 215
    },
    {
      "epoch": 0.3404255319148936,
      "grad_norm": 0.3325921893119812,
      "learning_rate": 3.941782395901249e-06,
      "log_odds_chosen": 0.30871960520744324,
      "log_odds_ratio": -0.5597559213638306,
      "logits/chosen": 0.4807589054107666,
      "logits/rejected": -0.254393994808197,
      "logps/chosen": -1.8524887561798096,
      "logps/rejected": -2.1190786361694336,
      "loss": 2.0559,
      "nll_loss": 1.9999738931655884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18524888157844543,
      "rewards/margins": 0.026658987626433372,
      "rewards/rejected": -0.21190787851810455,
      "step": 216
    },
    {
      "epoch": 0.34200157604412923,
      "grad_norm": 0.29487061500549316,
      "learning_rate": 3.940458392367032e-06,
      "log_odds_chosen": 0.3391064703464508,
      "log_odds_ratio": -0.5431881546974182,
      "logits/chosen": 0.48744064569473267,
      "logits/rejected": -0.5134267807006836,
      "logps/chosen": -1.8746438026428223,
      "logps/rejected": -2.1684770584106445,
      "loss": 2.0436,
      "nll_loss": 1.989285945892334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1874643713235855,
      "rewards/margins": 0.029383335262537003,
      "rewards/rejected": -0.2168477326631546,
      "step": 217
    },
    {
      "epoch": 0.34357762017336485,
      "grad_norm": 0.29633522033691406,
      "learning_rate": 3.939119729596493e-06,
      "log_odds_chosen": 0.3622989058494568,
      "log_odds_ratio": -0.5331794619560242,
      "logits/chosen": 0.5500465631484985,
      "logits/rejected": -0.39332813024520874,
      "logps/chosen": -1.8827656507492065,
      "logps/rejected": -2.198401927947998,
      "loss": 2.0648,
      "nll_loss": 2.0114388465881348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18827655911445618,
      "rewards/margins": 0.0315636545419693,
      "rewards/rejected": -0.21984019875526428,
      "step": 218
    },
    {
      "epoch": 0.34515366430260047,
      "grad_norm": 0.2902742326259613,
      "learning_rate": 3.93776641770259e-06,
      "log_odds_chosen": 0.2742522060871124,
      "log_odds_ratio": -0.5714613199234009,
      "logits/chosen": 0.5202993154525757,
      "logits/rejected": -0.45154300332069397,
      "logps/chosen": -1.848337173461914,
      "logps/rejected": -2.0857441425323486,
      "loss": 2.0285,
      "nll_loss": 1.9713159799575806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18483375012874603,
      "rewards/margins": 0.023740684613585472,
      "rewards/rejected": -0.20857441425323486,
      "step": 219
    },
    {
      "epoch": 0.3467297084318361,
      "grad_norm": 0.29342028498649597,
      "learning_rate": 3.93639846690895e-06,
      "log_odds_chosen": 0.46571236848831177,
      "log_odds_ratio": -0.4907868206501007,
      "logits/chosen": 0.5409021973609924,
      "logits/rejected": -0.6227868795394897,
      "logps/chosen": -1.8621845245361328,
      "logps/rejected": -2.268977165222168,
      "loss": 2.0331,
      "nll_loss": 1.9839848279953003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1862184703350067,
      "rewards/margins": 0.040679264813661575,
      "rewards/rejected": -0.226897731423378,
      "step": 220
    },
    {
      "epoch": 0.3483057525610717,
      "grad_norm": 0.2885148823261261,
      "learning_rate": 3.9350158875497855e-06,
      "log_odds_chosen": 0.5439132452011108,
      "log_odds_ratio": -0.46580708026885986,
      "logits/chosen": 0.5373093485832214,
      "logits/rejected": -0.4585607051849365,
      "logps/chosen": -1.767594575881958,
      "logps/rejected": -2.2380287647247314,
      "loss": 1.956,
      "nll_loss": 1.909372329711914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17675945162773132,
      "rewards/margins": 0.04704343155026436,
      "rewards/rejected": -0.22380287945270538,
      "step": 221
    },
    {
      "epoch": 0.34988179669030733,
      "grad_norm": 0.2833883762359619,
      "learning_rate": 3.933618690069824e-06,
      "log_odds_chosen": 0.48881855607032776,
      "log_odds_ratio": -0.4842972159385681,
      "logits/chosen": 0.39756086468696594,
      "logits/rejected": -0.41850200295448303,
      "logps/chosen": -1.7153518199920654,
      "logps/rejected": -2.129936933517456,
      "loss": 1.9156,
      "nll_loss": 1.8671796321868896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1715351641178131,
      "rewards/margins": 0.04145851731300354,
      "rewards/rejected": -0.21299368143081665,
      "step": 222
    },
    {
      "epoch": 0.35145784081954295,
      "grad_norm": 0.2946211099624634,
      "learning_rate": 3.932206885024226e-06,
      "log_odds_chosen": 0.2477441281080246,
      "log_odds_ratio": -0.5846911072731018,
      "logits/chosen": 0.466760516166687,
      "logits/rejected": -0.3636370003223419,
      "logps/chosen": -1.8313201665878296,
      "logps/rejected": -2.0461394786834717,
      "loss": 2.005,
      "nll_loss": 1.946488857269287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18313202261924744,
      "rewards/margins": 0.021481947973370552,
      "rewards/rejected": -0.20461396872997284,
      "step": 223
    },
    {
      "epoch": 0.3530338849487786,
      "grad_norm": 0.2853477895259857,
      "learning_rate": 3.930780483078502e-06,
      "log_odds_chosen": 0.34601038694381714,
      "log_odds_ratio": -0.5424416661262512,
      "logits/chosen": 0.43299296498298645,
      "logits/rejected": -0.43704044818878174,
      "logps/chosen": -1.7491978406906128,
      "logps/rejected": -2.0430784225463867,
      "loss": 1.9483,
      "nll_loss": 1.8940699100494385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17491979897022247,
      "rewards/margins": 0.02938804030418396,
      "rewards/rejected": -0.20430783927440643,
      "step": 224
    },
    {
      "epoch": 0.3546099290780142,
      "grad_norm": 0.3010624647140503,
      "learning_rate": 3.92933949500844e-06,
      "log_odds_chosen": 0.3569672405719757,
      "log_odds_ratio": -0.548423171043396,
      "logits/chosen": 0.555142879486084,
      "logits/rejected": -0.394178569316864,
      "logps/chosen": -1.7384213209152222,
      "logps/rejected": -2.036893367767334,
      "loss": 1.9405,
      "nll_loss": 1.8856838941574097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17384213209152222,
      "rewards/margins": 0.029847197234630585,
      "rewards/rejected": -0.2036893218755722,
      "step": 225
    },
    {
      "epoch": 0.3561859732072498,
      "grad_norm": 0.28172364830970764,
      "learning_rate": 3.9278839317000155e-06,
      "log_odds_chosen": 0.4323083162307739,
      "log_odds_ratio": -0.5072457790374756,
      "logits/chosen": 0.5468685030937195,
      "logits/rejected": -0.37213990092277527,
      "logps/chosen": -1.7517664432525635,
      "logps/rejected": -2.123047113418579,
      "loss": 1.9389,
      "nll_loss": 1.888132929801941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1751766800880432,
      "rewards/margins": 0.03712804615497589,
      "rewards/rejected": -0.2123047113418579,
      "step": 226
    },
    {
      "epoch": 0.35776201733648544,
      "grad_norm": 0.28558024764060974,
      "learning_rate": 3.926413804149314e-06,
      "log_odds_chosen": 0.12638477981090546,
      "log_odds_ratio": -0.64253169298172,
      "logits/chosen": 0.5238491296768188,
      "logits/rejected": -0.41080769896507263,
      "logps/chosen": -1.8467971086502075,
      "logps/rejected": -1.955765724182129,
      "loss": 2.03,
      "nll_loss": 1.9657707214355469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18467971682548523,
      "rewards/margins": 0.010896885767579079,
      "rewards/rejected": -0.19557659327983856,
      "step": 227
    },
    {
      "epoch": 0.35933806146572106,
      "grad_norm": 0.2915712594985962,
      "learning_rate": 3.924929123462447e-06,
      "log_odds_chosen": 0.3050719201564789,
      "log_odds_ratio": -0.5605703592300415,
      "logits/chosen": 0.5805931687355042,
      "logits/rejected": -0.4595981538295746,
      "logps/chosen": -1.8690290451049805,
      "logps/rejected": -2.13301157951355,
      "loss": 2.0441,
      "nll_loss": 1.9880114793777466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18690288066864014,
      "rewards/margins": 0.026398256421089172,
      "rewards/rejected": -0.2133011519908905,
      "step": 228
    },
    {
      "epoch": 0.3609141055949567,
      "grad_norm": 0.29803889989852905,
      "learning_rate": 3.923429900855468e-06,
      "log_odds_chosen": 0.19842049479484558,
      "log_odds_ratio": -0.6108622550964355,
      "logits/chosen": 0.5528815984725952,
      "logits/rejected": -0.23056240379810333,
      "logps/chosen": -1.9518077373504639,
      "logps/rejected": -2.122875213623047,
      "loss": 2.1321,
      "nll_loss": 2.0710208415985107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1951807588338852,
      "rewards/margins": 0.01710677333176136,
      "rewards/rejected": -0.2122875452041626,
      "step": 229
    },
    {
      "epoch": 0.3624901497241923,
      "grad_norm": 0.2763267457485199,
      "learning_rate": 3.921916147654287e-06,
      "log_odds_chosen": 0.4757264256477356,
      "log_odds_ratio": -0.49002140760421753,
      "logits/chosen": 0.4894074499607086,
      "logits/rejected": -0.5041885375976562,
      "logps/chosen": -1.774259328842163,
      "logps/rejected": -2.182471513748169,
      "loss": 1.9376,
      "nll_loss": 1.8885548114776611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17742595076560974,
      "rewards/margins": 0.04082119092345238,
      "rewards/rejected": -0.21824713051319122,
      "step": 230
    },
    {
      "epoch": 0.3640661938534279,
      "grad_norm": 0.2874124348163605,
      "learning_rate": 3.920387875294588e-06,
      "log_odds_chosen": 0.30937978625297546,
      "log_odds_ratio": -0.5523663759231567,
      "logits/chosen": 0.5325709581375122,
      "logits/rejected": -0.34745681285858154,
      "logps/chosen": -1.7468411922454834,
      "logps/rejected": -2.0082056522369385,
      "loss": 1.9317,
      "nll_loss": 1.8764588832855225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17468410730361938,
      "rewards/margins": 0.026136459782719612,
      "rewards/rejected": -0.20082058012485504,
      "step": 231
    },
    {
      "epoch": 0.3656422379826635,
      "grad_norm": 0.2740110456943512,
      "learning_rate": 3.918845095321737e-06,
      "log_odds_chosen": 0.2758645713329315,
      "log_odds_ratio": -0.576305627822876,
      "logits/chosen": 0.45111507177352905,
      "logits/rejected": -0.5447308421134949,
      "logps/chosen": -1.7711262702941895,
      "logps/rejected": -1.999793529510498,
      "loss": 1.9468,
      "nll_loss": 1.8891706466674805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1771126240491867,
      "rewards/margins": 0.02286672033369541,
      "rewards/rejected": -0.19997934997081757,
      "step": 232
    },
    {
      "epoch": 0.3672182821118991,
      "grad_norm": 0.30315297842025757,
      "learning_rate": 3.9172878193907004e-06,
      "log_odds_chosen": 0.24979126453399658,
      "log_odds_ratio": -0.5865897536277771,
      "logits/chosen": 0.4408178925514221,
      "logits/rejected": -0.430459588766098,
      "logps/chosen": -1.8613289594650269,
      "logps/rejected": -2.078932762145996,
      "loss": 2.0549,
      "nll_loss": 1.9962174892425537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18613289296627045,
      "rewards/margins": 0.02176038548350334,
      "rewards/rejected": -0.2078932672739029,
      "step": 233
    },
    {
      "epoch": 0.36879432624113473,
      "grad_norm": 0.28406012058258057,
      "learning_rate": 3.915716059265955e-06,
      "log_odds_chosen": 0.3215256929397583,
      "log_odds_ratio": -0.5549464225769043,
      "logits/chosen": 0.502613365650177,
      "logits/rejected": -0.5350136160850525,
      "logps/chosen": -1.8017627000808716,
      "logps/rejected": -2.079495668411255,
      "loss": 1.98,
      "nll_loss": 1.9245139360427856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.180176243185997,
      "rewards/margins": 0.027773309499025345,
      "rewards/rejected": -0.20794956386089325,
      "step": 234
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 0.26154837012290955,
      "learning_rate": 3.9141298268213966e-06,
      "log_odds_chosen": 0.474556028842926,
      "log_odds_ratio": -0.4884706139564514,
      "logits/chosen": 0.5216464996337891,
      "logits/rejected": -0.4089938998222351,
      "logps/chosen": -1.7237547636032104,
      "logps/rejected": -2.131652355194092,
      "loss": 1.9058,
      "nll_loss": 1.8569673299789429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17237547039985657,
      "rewards/margins": 0.04078977555036545,
      "rewards/rejected": -0.21316523849964142,
      "step": 235
    },
    {
      "epoch": 0.37194641449960597,
      "grad_norm": 0.28945106267929077,
      "learning_rate": 3.912529134040255e-06,
      "log_odds_chosen": 0.2693881094455719,
      "log_odds_ratio": -0.5723139643669128,
      "logits/chosen": 0.5206422805786133,
      "logits/rejected": -0.3318684697151184,
      "logps/chosen": -1.8110253810882568,
      "logps/rejected": -2.0432167053222656,
      "loss": 1.9982,
      "nll_loss": 1.9410051107406616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18110254406929016,
      "rewards/margins": 0.023219116032123566,
      "rewards/rejected": -0.20432165265083313,
      "step": 236
    },
    {
      "epoch": 0.3735224586288416,
      "grad_norm": 0.2875734567642212,
      "learning_rate": 3.910913993014998e-06,
      "log_odds_chosen": 0.4889007806777954,
      "log_odds_ratio": -0.4840152859687805,
      "logits/chosen": 0.6125221252441406,
      "logits/rejected": -0.47014880180358887,
      "logps/chosen": -1.7982161045074463,
      "logps/rejected": -2.2228915691375732,
      "loss": 1.9734,
      "nll_loss": 1.9249656200408936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17982161045074463,
      "rewards/margins": 0.042467549443244934,
      "rewards/rejected": -0.22228917479515076,
      "step": 237
    },
    {
      "epoch": 0.3750985027580772,
      "grad_norm": 0.2845603823661804,
      "learning_rate": 3.909284415947246e-06,
      "log_odds_chosen": 0.4321085214614868,
      "log_odds_ratio": -0.508188009262085,
      "logits/chosen": 0.5699516534805298,
      "logits/rejected": -0.634801983833313,
      "logps/chosen": -1.8531954288482666,
      "logps/rejected": -2.2272119522094727,
      "loss": 2.013,
      "nll_loss": 1.9621832370758057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18531955778598785,
      "rewards/margins": 0.03740163892507553,
      "rewards/rejected": -0.2227211892604828,
      "step": 238
    },
    {
      "epoch": 0.37667454688731283,
      "grad_norm": 0.27208924293518066,
      "learning_rate": 3.907640415147674e-06,
      "log_odds_chosen": 0.5407861471176147,
      "log_odds_ratio": -0.4670700430870056,
      "logits/chosen": 0.45632970333099365,
      "logits/rejected": -0.6743687391281128,
      "logps/chosen": -1.7466856241226196,
      "logps/rejected": -2.2121920585632324,
      "loss": 1.9239,
      "nll_loss": 1.8772082328796387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1746685802936554,
      "rewards/margins": 0.046550631523132324,
      "rewards/rejected": -0.22121921181678772,
      "step": 239
    },
    {
      "epoch": 0.37825059101654845,
      "grad_norm": 0.28472790122032166,
      "learning_rate": 3.905982003035924e-06,
      "log_odds_chosen": 0.37944096326828003,
      "log_odds_ratio": -0.5271102786064148,
      "logits/chosen": 0.4610236883163452,
      "logits/rejected": -0.6050642132759094,
      "logps/chosen": -1.8401498794555664,
      "logps/rejected": -2.1687233448028564,
      "loss": 2.004,
      "nll_loss": 1.9512531757354736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18401499092578888,
      "rewards/margins": 0.03285733237862587,
      "rewards/rejected": -0.21687231957912445,
      "step": 240
    },
    {
      "epoch": 0.3798266351457841,
      "grad_norm": 0.28115785121917725,
      "learning_rate": 3.904309192140506e-06,
      "log_odds_chosen": 0.21689362823963165,
      "log_odds_ratio": -0.6001635789871216,
      "logits/chosen": 0.5072639584541321,
      "logits/rejected": -0.26205065846443176,
      "logps/chosen": -1.7706879377365112,
      "logps/rejected": -1.9564616680145264,
      "loss": 1.9545,
      "nll_loss": 1.8944413661956787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1770688146352768,
      "rewards/margins": 0.018577365204691887,
      "rewards/rejected": -0.19564618170261383,
      "step": 241
    },
    {
      "epoch": 0.3814026792750197,
      "grad_norm": 0.2975656986236572,
      "learning_rate": 3.90262199509871e-06,
      "log_odds_chosen": 0.26215988397598267,
      "log_odds_ratio": -0.5752044916152954,
      "logits/chosen": 0.4758797287940979,
      "logits/rejected": -0.33018162846565247,
      "logps/chosen": -1.8034093379974365,
      "logps/rejected": -2.0274529457092285,
      "loss": 1.9887,
      "nll_loss": 1.9311902523040771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1803409308195114,
      "rewards/margins": 0.022404363378882408,
      "rewards/rejected": -0.20274530351161957,
      "step": 242
    },
    {
      "epoch": 0.3829787234042553,
      "grad_norm": 0.2805575728416443,
      "learning_rate": 3.900920424656501e-06,
      "log_odds_chosen": 0.4157733917236328,
      "log_odds_ratio": -0.5154089331626892,
      "logits/chosen": 0.4855855107307434,
      "logits/rejected": -0.49067410826683044,
      "logps/chosen": -1.7054016590118408,
      "logps/rejected": -2.058197021484375,
      "loss": 1.8792,
      "nll_loss": 1.8276593685150146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17054018378257751,
      "rewards/margins": 0.0352795347571373,
      "rewards/rejected": -0.20581969618797302,
      "step": 243
    },
    {
      "epoch": 0.38455476753349094,
      "grad_norm": 0.32197999954223633,
      "learning_rate": 3.899204493668432e-06,
      "log_odds_chosen": 0.3415714502334595,
      "log_odds_ratio": -0.5509850978851318,
      "logits/chosen": 0.5429471731185913,
      "logits/rejected": -0.42629846930503845,
      "logps/chosen": -1.8829054832458496,
      "logps/rejected": -2.185755491256714,
      "loss": 2.0453,
      "nll_loss": 1.9901872873306274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1882905513048172,
      "rewards/margins": 0.030284997075796127,
      "rewards/rejected": -0.21857555210590363,
      "step": 244
    },
    {
      "epoch": 0.38613081166272656,
      "grad_norm": 0.2792482376098633,
      "learning_rate": 3.897474215097542e-06,
      "log_odds_chosen": 0.3295517563819885,
      "log_odds_ratio": -0.5488483309745789,
      "logits/chosen": 0.5338771939277649,
      "logits/rejected": -0.7156805396080017,
      "logps/chosen": -1.8040363788604736,
      "logps/rejected": -2.0901222229003906,
      "loss": 1.99,
      "nll_loss": 1.9351005554199219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18040362000465393,
      "rewards/margins": 0.028608599677681923,
      "rewards/rejected": -0.2090122103691101,
      "step": 245
    },
    {
      "epoch": 0.3877068557919622,
      "grad_norm": 0.2792900502681732,
      "learning_rate": 3.8957296020152596e-06,
      "log_odds_chosen": 0.4721974730491638,
      "log_odds_ratio": -0.49195337295532227,
      "logits/chosen": 0.5625882148742676,
      "logits/rejected": -0.5468290448188782,
      "logps/chosen": -1.7749059200286865,
      "logps/rejected": -2.1837642192840576,
      "loss": 1.953,
      "nll_loss": 1.9038406610488892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17749059200286865,
      "rewards/margins": 0.04088583216071129,
      "rewards/rejected": -0.21837644279003143,
      "step": 246
    },
    {
      "epoch": 0.3892828999211978,
      "grad_norm": 0.29311904311180115,
      "learning_rate": 3.893970667601303e-06,
      "log_odds_chosen": 0.3301757276058197,
      "log_odds_ratio": -0.5501846075057983,
      "logits/chosen": 0.45822829008102417,
      "logits/rejected": -0.7882699966430664,
      "logps/chosen": -1.8771990537643433,
      "logps/rejected": -2.1641597747802734,
      "loss": 2.0396,
      "nll_loss": 1.984574317932129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18771988153457642,
      "rewards/margins": 0.028696071356534958,
      "rewards/rejected": -0.21641597151756287,
      "step": 247
    },
    {
      "epoch": 0.3908589440504334,
      "grad_norm": 0.2894313931465149,
      "learning_rate": 3.892197425143581e-06,
      "log_odds_chosen": 0.5370165705680847,
      "log_odds_ratio": -0.4752396047115326,
      "logits/chosen": 0.5151894092559814,
      "logits/rejected": -0.387676477432251,
      "logps/chosen": -1.799578309059143,
      "logps/rejected": -2.2680466175079346,
      "loss": 1.9568,
      "nll_loss": 1.9092282056808472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17995783686637878,
      "rewards/margins": 0.046846821904182434,
      "rewards/rejected": -0.22680465877056122,
      "step": 248
    },
    {
      "epoch": 0.39243498817966904,
      "grad_norm": 0.3017396926879883,
      "learning_rate": 3.890409888038094e-06,
      "log_odds_chosen": 0.3672383725643158,
      "log_odds_ratio": -0.5369208455085754,
      "logits/chosen": 0.5536250472068787,
      "logits/rejected": -0.3620547950267792,
      "logps/chosen": -1.8479626178741455,
      "logps/rejected": -2.171208143234253,
      "loss": 2.0219,
      "nll_loss": 1.9681875705718994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1847962588071823,
      "rewards/margins": 0.03232455253601074,
      "rewards/rejected": -0.21712081134319305,
      "step": 249
    },
    {
      "epoch": 0.39401103230890466,
      "grad_norm": 0.28757238388061523,
      "learning_rate": 3.888608069788831e-06,
      "log_odds_chosen": 0.28784364461898804,
      "log_odds_ratio": -0.575218141078949,
      "logits/chosen": 0.4429545998573303,
      "logits/rejected": -0.35244786739349365,
      "logps/chosen": -1.7926141023635864,
      "logps/rejected": -2.0435385704040527,
      "loss": 1.9702,
      "nll_loss": 1.912689208984375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17926141619682312,
      "rewards/margins": 0.02509242296218872,
      "rewards/rejected": -0.20435382425785065,
      "step": 250
    },
    {
      "epoch": 0.3955870764381403,
      "grad_norm": 0.26302191615104675,
      "learning_rate": 3.8867919840076685e-06,
      "log_odds_chosen": 0.4418867826461792,
      "log_odds_ratio": -0.5003976225852966,
      "logits/chosen": 0.46342819929122925,
      "logits/rejected": -0.6135963797569275,
      "logps/chosen": -1.7101153135299683,
      "logps/rejected": -2.0860018730163574,
      "loss": 1.8719,
      "nll_loss": 1.8218750953674316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17101150751113892,
      "rewards/margins": 0.0375886932015419,
      "rewards/rejected": -0.2086002230644226,
      "step": 251
    },
    {
      "epoch": 0.3971631205673759,
      "grad_norm": 0.28215491771698,
      "learning_rate": 3.884961644414267e-06,
      "log_odds_chosen": 0.33040478825569153,
      "log_odds_ratio": -0.5436782836914062,
      "logits/chosen": 0.4993625581264496,
      "logits/rejected": -0.1604970395565033,
      "logps/chosen": -1.8361539840698242,
      "logps/rejected": -2.1208314895629883,
      "loss": 1.98,
      "nll_loss": 1.9255832433700562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18361540138721466,
      "rewards/margins": 0.028467737138271332,
      "rewards/rejected": -0.2120831459760666,
      "step": 252
    },
    {
      "epoch": 0.3987391646966115,
      "grad_norm": 0.280631959438324,
      "learning_rate": 3.883117064835967e-06,
      "log_odds_chosen": 0.36852777004241943,
      "log_odds_ratio": -0.5398227572441101,
      "logits/chosen": 0.45098748803138733,
      "logits/rejected": -0.4938051402568817,
      "logps/chosen": -1.7849329710006714,
      "logps/rejected": -2.1051712036132812,
      "loss": 1.9618,
      "nll_loss": 1.9077905416488647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17849330604076385,
      "rewards/margins": 0.0320238396525383,
      "rewards/rejected": -0.21051713824272156,
      "step": 253
    },
    {
      "epoch": 0.40031520882584715,
      "grad_norm": 0.2820970416069031,
      "learning_rate": 3.881258259207688e-06,
      "log_odds_chosen": 0.30618369579315186,
      "log_odds_ratio": -0.5557839870452881,
      "logits/chosen": 0.549262523651123,
      "logits/rejected": -0.5373132228851318,
      "logps/chosen": -1.7485055923461914,
      "logps/rejected": -2.0079431533813477,
      "loss": 1.9015,
      "nll_loss": 1.845925211906433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17485055327415466,
      "rewards/margins": 0.025943776592612267,
      "rewards/rejected": -0.20079432427883148,
      "step": 254
    },
    {
      "epoch": 0.40189125295508277,
      "grad_norm": 0.26481202244758606,
      "learning_rate": 3.8793852415718165e-06,
      "log_odds_chosen": 0.44486790895462036,
      "log_odds_ratio": -0.49916309118270874,
      "logits/chosen": 0.4596560597419739,
      "logits/rejected": -0.7777143716812134,
      "logps/chosen": -1.6682631969451904,
      "logps/rejected": -2.0447134971618652,
      "loss": 1.8254,
      "nll_loss": 1.7754442691802979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16682633757591248,
      "rewards/margins": 0.03764501214027405,
      "rewards/rejected": -0.20447133481502533,
      "step": 255
    },
    {
      "epoch": 0.4034672970843184,
      "grad_norm": 0.27511125802993774,
      "learning_rate": 3.877498026078107e-06,
      "log_odds_chosen": 0.2643274664878845,
      "log_odds_ratio": -0.5784630179405212,
      "logits/chosen": 0.4659520089626312,
      "logits/rejected": -0.6729861497879028,
      "logps/chosen": -1.8191360235214233,
      "logps/rejected": -2.0481255054473877,
      "loss": 1.9784,
      "nll_loss": 1.9205960035324097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1819136142730713,
      "rewards/margins": 0.02289893850684166,
      "rewards/rejected": -0.20481254160404205,
      "step": 256
    },
    {
      "epoch": 0.40504334121355395,
      "grad_norm": 0.28119486570358276,
      "learning_rate": 3.875596626983573e-06,
      "log_odds_chosen": 0.4398841857910156,
      "log_odds_ratio": -0.5096471309661865,
      "logits/chosen": 0.48892465233802795,
      "logits/rejected": -0.37292981147766113,
      "logps/chosen": -1.7355577945709229,
      "logps/rejected": -2.1096043586730957,
      "loss": 1.8938,
      "nll_loss": 1.842874526977539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17355579137802124,
      "rewards/margins": 0.037404656410217285,
      "rewards/rejected": -0.21096044778823853,
      "step": 257
    },
    {
      "epoch": 0.4066193853427896,
      "grad_norm": 0.2830592691898346,
      "learning_rate": 3.873681058652374e-06,
      "log_odds_chosen": 0.21046969294548035,
      "log_odds_ratio": -0.6032708883285522,
      "logits/chosen": 0.47122278809547424,
      "logits/rejected": -0.4699457287788391,
      "logps/chosen": -1.8599016666412354,
      "logps/rejected": -2.043604850769043,
      "loss": 2.02,
      "nll_loss": 1.9596831798553467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1859901398420334,
      "rewards/margins": 0.01837034709751606,
      "rewards/rejected": -0.2043604850769043,
      "step": 258
    },
    {
      "epoch": 0.4081954294720252,
      "grad_norm": 0.2882324457168579,
      "learning_rate": 3.871751335555715e-06,
      "log_odds_chosen": 0.3207879066467285,
      "log_odds_ratio": -0.551344633102417,
      "logits/chosen": 0.4617171585559845,
      "logits/rejected": -0.5876399874687195,
      "logps/chosen": -1.7937874794006348,
      "logps/rejected": -2.069718360900879,
      "loss": 1.9415,
      "nll_loss": 1.8863192796707153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17937874794006348,
      "rewards/margins": 0.027593078091740608,
      "rewards/rejected": -0.20697180926799774,
      "step": 259
    },
    {
      "epoch": 0.4097714736012608,
      "grad_norm": 0.26368269324302673,
      "learning_rate": 3.869807472271731e-06,
      "log_odds_chosen": 0.3248588442802429,
      "log_odds_ratio": -0.5475046634674072,
      "logits/chosen": 0.4813993573188782,
      "logits/rejected": -0.5130388140678406,
      "logps/chosen": -1.7483863830566406,
      "logps/rejected": -2.0266005992889404,
      "loss": 1.9345,
      "nll_loss": 1.8797162771224976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17483864724636078,
      "rewards/margins": 0.02782142162322998,
      "rewards/rejected": -0.20266005396842957,
      "step": 260
    },
    {
      "epoch": 0.41134751773049644,
      "grad_norm": 0.27178314328193665,
      "learning_rate": 3.8678494834853826e-06,
      "log_odds_chosen": 0.3267313838005066,
      "log_odds_ratio": -0.5527905821800232,
      "logits/chosen": 0.463585764169693,
      "logits/rejected": -0.4223080277442932,
      "logps/chosen": -1.7635301351547241,
      "logps/rejected": -2.0397844314575195,
      "loss": 1.9186,
      "nll_loss": 1.863335132598877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17635302245616913,
      "rewards/margins": 0.027625398710370064,
      "rewards/rejected": -0.20397843420505524,
      "step": 261
    },
    {
      "epoch": 0.41292356185973206,
      "grad_norm": 0.29288655519485474,
      "learning_rate": 3.865877383988339e-06,
      "log_odds_chosen": 0.3967463970184326,
      "log_odds_ratio": -0.5207533836364746,
      "logits/chosen": 0.4554799199104309,
      "logits/rejected": -0.4541538953781128,
      "logps/chosen": -1.8009213209152222,
      "logps/rejected": -2.139960765838623,
      "loss": 1.9765,
      "nll_loss": 1.9244519472122192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18009214103221893,
      "rewards/margins": 0.03390393778681755,
      "rewards/rejected": -0.21399609744548798,
      "step": 262
    },
    {
      "epoch": 0.4144996059889677,
      "grad_norm": 0.2784212827682495,
      "learning_rate": 3.863891188678869e-06,
      "log_odds_chosen": 0.14312618970870972,
      "log_odds_ratio": -0.6421398520469666,
      "logits/chosen": 0.4103483259677887,
      "logits/rejected": -0.6072329878807068,
      "logps/chosen": -1.6957128047943115,
      "logps/rejected": -1.8229939937591553,
      "loss": 1.865,
      "nll_loss": 1.8007937669754028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16957129538059235,
      "rewards/margins": 0.012728110887110233,
      "rewards/rejected": -0.1822993904352188,
      "step": 263
    },
    {
      "epoch": 0.4160756501182033,
      "grad_norm": 0.2715873122215271,
      "learning_rate": 3.8618909125617305e-06,
      "log_odds_chosen": 0.5240032076835632,
      "log_odds_ratio": -0.4674418270587921,
      "logits/chosen": 0.4032454192638397,
      "logits/rejected": -0.8130543828010559,
      "logps/chosen": -1.6866666078567505,
      "logps/rejected": -2.1323814392089844,
      "loss": 1.8611,
      "nll_loss": 1.8143792152404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16866666078567505,
      "rewards/margins": 0.044571489095687866,
      "rewards/rejected": -0.21323813498020172,
      "step": 264
    },
    {
      "epoch": 0.4176516942474389,
      "grad_norm": 0.2634105682373047,
      "learning_rate": 3.859876570748054e-06,
      "log_odds_chosen": 0.3052539527416229,
      "log_odds_ratio": -0.555004894733429,
      "logits/chosen": 0.484012246131897,
      "logits/rejected": -0.15861767530441284,
      "logps/chosen": -1.7536225318908691,
      "logps/rejected": -2.012622833251953,
      "loss": 1.9283,
      "nll_loss": 1.8727929592132568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1753622442483902,
      "rewards/margins": 0.025900062173604965,
      "rewards/rejected": -0.20126229524612427,
      "step": 265
    },
    {
      "epoch": 0.41922773837667454,
      "grad_norm": 0.26199325919151306,
      "learning_rate": 3.857848178455231e-06,
      "log_odds_chosen": 0.10959838330745697,
      "log_odds_ratio": -0.6467946767807007,
      "logits/chosen": 0.48704051971435547,
      "logits/rejected": -0.8094627261161804,
      "logps/chosen": -1.7569371461868286,
      "logps/rejected": -1.8549176454544067,
      "loss": 1.9459,
      "nll_loss": 1.88121497631073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17569372057914734,
      "rewards/margins": 0.00979805551469326,
      "rewards/rejected": -0.18549177050590515,
      "step": 266
    },
    {
      "epoch": 0.42080378250591016,
      "grad_norm": 0.29430001974105835,
      "learning_rate": 3.855805751006794e-06,
      "log_odds_chosen": 0.4646510183811188,
      "log_odds_ratio": -0.5037495493888855,
      "logits/chosen": 0.45664528012275696,
      "logits/rejected": -0.7876089215278625,
      "logps/chosen": -1.7912521362304688,
      "logps/rejected": -2.192134380340576,
      "loss": 1.9399,
      "nll_loss": 1.8894766569137573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17912520468235016,
      "rewards/margins": 0.04008820280432701,
      "rewards/rejected": -0.21921342611312866,
      "step": 267
    },
    {
      "epoch": 0.4223798266351458,
      "grad_norm": 0.24849706888198853,
      "learning_rate": 3.853749303832308e-06,
      "log_odds_chosen": 0.3170139491558075,
      "log_odds_ratio": -0.5517860651016235,
      "logits/chosen": 0.43677279353141785,
      "logits/rejected": -0.38730910420417786,
      "logps/chosen": -1.7149559259414673,
      "logps/rejected": -1.982513427734375,
      "loss": 1.8752,
      "nll_loss": 1.8200231790542603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17149560153484344,
      "rewards/margins": 0.026755744591355324,
      "rewards/rejected": -0.19825133681297302,
      "step": 268
    },
    {
      "epoch": 0.4239558707643814,
      "grad_norm": 0.2742369771003723,
      "learning_rate": 3.8516788524672495e-06,
      "log_odds_chosen": 0.1780942976474762,
      "log_odds_ratio": -0.6158064603805542,
      "logits/chosen": 0.4654761552810669,
      "logits/rejected": -0.6833764314651489,
      "logps/chosen": -1.7883644104003906,
      "logps/rejected": -1.942136526107788,
      "loss": 1.9458,
      "nll_loss": 1.8842130899429321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17883646488189697,
      "rewards/margins": 0.015377204865217209,
      "rewards/rejected": -0.1942136585712433,
      "step": 269
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 0.2591206133365631,
      "learning_rate": 3.849594412552889e-06,
      "log_odds_chosen": 0.3254651725292206,
      "log_odds_ratio": -0.5489966869354248,
      "logits/chosen": 0.47052422165870667,
      "logits/rejected": -0.4670095443725586,
      "logps/chosen": -1.6865018606185913,
      "logps/rejected": -1.9543278217315674,
      "loss": 1.8487,
      "nll_loss": 1.7938035726547241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16865020990371704,
      "rewards/margins": 0.026782585307955742,
      "rewards/rejected": -0.19543276727199554,
      "step": 270
    },
    {
      "epoch": 0.42710795902285265,
      "grad_norm": 0.26700544357299805,
      "learning_rate": 3.847495999836175e-06,
      "log_odds_chosen": 0.3362584114074707,
      "log_odds_ratio": -0.5431700944900513,
      "logits/chosen": 0.5047026872634888,
      "logits/rejected": -0.694282054901123,
      "logps/chosen": -1.7739330530166626,
      "logps/rejected": -2.061260461807251,
      "loss": 1.92,
      "nll_loss": 1.8657115697860718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17739331722259521,
      "rewards/margins": 0.028732730075716972,
      "rewards/rejected": -0.20612604916095734,
      "step": 271
    },
    {
      "epoch": 0.42868400315208827,
      "grad_norm": 0.25737646222114563,
      "learning_rate": 3.845383630169613e-06,
      "log_odds_chosen": 0.33631449937820435,
      "log_odds_ratio": -0.5460928678512573,
      "logits/chosen": 0.3562855124473572,
      "logits/rejected": -0.6789432168006897,
      "logps/chosen": -1.6831501722335815,
      "logps/rejected": -1.9663141965866089,
      "loss": 1.8306,
      "nll_loss": 1.7760382890701294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16831502318382263,
      "rewards/margins": 0.028316402807831764,
      "rewards/rejected": -0.19663141667842865,
      "step": 272
    },
    {
      "epoch": 0.4302600472813239,
      "grad_norm": 0.27558234333992004,
      "learning_rate": 3.843257319511147e-06,
      "log_odds_chosen": 0.5057035684585571,
      "log_odds_ratio": -0.480516254901886,
      "logits/chosen": 0.4118584394454956,
      "logits/rejected": -0.8097151517868042,
      "logps/chosen": -1.6787090301513672,
      "logps/rejected": -2.1040146350860596,
      "loss": 1.8281,
      "nll_loss": 1.7800368070602417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1678709089756012,
      "rewards/margins": 0.04253055900335312,
      "rewards/rejected": -0.21040146052837372,
      "step": 273
    },
    {
      "epoch": 0.4318360914105595,
      "grad_norm": 0.26741060614585876,
      "learning_rate": 3.841117083924039e-06,
      "log_odds_chosen": 0.3235999643802643,
      "log_odds_ratio": -0.5523552298545837,
      "logits/chosen": 0.397009938955307,
      "logits/rejected": -0.8522763252258301,
      "logps/chosen": -1.7239158153533936,
      "logps/rejected": -2.0000529289245605,
      "loss": 1.874,
      "nll_loss": 1.8187570571899414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17239157855510712,
      "rewards/margins": 0.027613701298832893,
      "rewards/rejected": -0.20000529289245605,
      "step": 274
    },
    {
      "epoch": 0.43341213553979513,
      "grad_norm": 0.26848408579826355,
      "learning_rate": 3.838962939576746e-06,
      "log_odds_chosen": 0.36745256185531616,
      "log_odds_ratio": -0.5347477793693542,
      "logits/chosen": 0.34434112906455994,
      "logits/rejected": -0.6846289038658142,
      "logps/chosen": -1.6550366878509521,
      "logps/rejected": -1.963791847229004,
      "loss": 1.8302,
      "nll_loss": 1.7767248153686523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16550366580486298,
      "rewards/margins": 0.030875524505972862,
      "rewards/rejected": -0.19637919962406158,
      "step": 275
    },
    {
      "epoch": 0.43498817966903075,
      "grad_norm": 0.27719369530677795,
      "learning_rate": 3.8367949027427985e-06,
      "log_odds_chosen": 0.24599358439445496,
      "log_odds_ratio": -0.581466794013977,
      "logits/chosen": 0.41534924507141113,
      "logits/rejected": -0.5528399348258972,
      "logps/chosen": -1.7868237495422363,
      "logps/rejected": -1.9973784685134888,
      "loss": 1.9663,
      "nll_loss": 1.908132791519165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1786823868751526,
      "rewards/margins": 0.02105545997619629,
      "rewards/rejected": -0.19973784685134888,
      "step": 276
    },
    {
      "epoch": 0.43656422379826637,
      "grad_norm": 0.2863655984401703,
      "learning_rate": 3.834612989800681e-06,
      "log_odds_chosen": 0.6205421686172485,
      "log_odds_ratio": -0.43973320722579956,
      "logits/chosen": 0.4383569359779358,
      "logits/rejected": -1.0248867273330688,
      "logps/chosen": -1.768141269683838,
      "logps/rejected": -2.3065080642700195,
      "loss": 1.9028,
      "nll_loss": 1.8588356971740723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17681412398815155,
      "rewards/margins": 0.05383666604757309,
      "rewards/rejected": -0.23065082728862762,
      "step": 277
    },
    {
      "epoch": 0.438140267927502,
      "grad_norm": 0.2790682315826416,
      "learning_rate": 3.832417217233703e-06,
      "log_odds_chosen": 0.23550176620483398,
      "log_odds_ratio": -0.5852302312850952,
      "logits/chosen": 0.39655160903930664,
      "logits/rejected": -0.5482574105262756,
      "logps/chosen": -1.7667053937911987,
      "logps/rejected": -1.9674649238586426,
      "loss": 1.925,
      "nll_loss": 1.8664298057556152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17667050659656525,
      "rewards/margins": 0.020075976848602295,
      "rewards/rejected": -0.19674648344516754,
      "step": 278
    },
    {
      "epoch": 0.4397163120567376,
      "grad_norm": 0.26800537109375,
      "learning_rate": 3.8302076016298775e-06,
      "log_odds_chosen": 0.39774730801582336,
      "log_odds_ratio": -0.5189967751502991,
      "logits/chosen": 0.33551838994026184,
      "logits/rejected": -0.7844797372817993,
      "logps/chosen": -1.6954327821731567,
      "logps/rejected": -2.031496524810791,
      "loss": 1.8556,
      "nll_loss": 1.8036519289016724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16954326629638672,
      "rewards/margins": 0.03360636159777641,
      "rewards/rejected": -0.20314966142177582,
      "step": 279
    },
    {
      "epoch": 0.44129235618597323,
      "grad_norm": 0.26829519867897034,
      "learning_rate": 3.827984159681796e-06,
      "log_odds_chosen": 0.3000229299068451,
      "log_odds_ratio": -0.5594607591629028,
      "logits/chosen": 0.31995582580566406,
      "logits/rejected": -0.6465460062026978,
      "logps/chosen": -1.7473139762878418,
      "logps/rejected": -2.0010228157043457,
      "loss": 1.8941,
      "nll_loss": 1.838153600692749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17473138868808746,
      "rewards/margins": 0.025370875373482704,
      "rewards/rejected": -0.20010226964950562,
      "step": 280
    },
    {
      "epoch": 0.4428684003152088,
      "grad_norm": 0.2701958417892456,
      "learning_rate": 3.825746908186498e-06,
      "log_odds_chosen": 0.38087302446365356,
      "log_odds_ratio": -0.5301374793052673,
      "logits/chosen": 0.4656312167644501,
      "logits/rejected": -0.8502053618431091,
      "logps/chosen": -1.7905412912368774,
      "logps/rejected": -2.118225336074829,
      "loss": 1.9211,
      "nll_loss": 1.8680615425109863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1790541410446167,
      "rewards/margins": 0.032768405973911285,
      "rewards/rejected": -0.2118225395679474,
      "step": 281
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.28369569778442383,
      "learning_rate": 3.823495864045352e-06,
      "log_odds_chosen": 0.41946643590927124,
      "log_odds_ratio": -0.5092715620994568,
      "logits/chosen": 0.4426696002483368,
      "logits/rejected": -0.2964620292186737,
      "logps/chosen": -1.7443768978118896,
      "logps/rejected": -2.0997912883758545,
      "loss": 1.8958,
      "nll_loss": 1.8448481559753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17443770170211792,
      "rewards/margins": 0.03554143011569977,
      "rewards/rejected": -0.2099791169166565,
      "step": 282
    },
    {
      "epoch": 0.44602048857368004,
      "grad_norm": 0.24753610789775848,
      "learning_rate": 3.8212310442639205e-06,
      "log_odds_chosen": 0.3292975425720215,
      "log_odds_ratio": -0.5465047359466553,
      "logits/chosen": 0.4356715977191925,
      "logits/rejected": -0.4361802339553833,
      "logps/chosen": -1.6899651288986206,
      "logps/rejected": -1.9682831764221191,
      "loss": 1.8516,
      "nll_loss": 1.7969615459442139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16899652779102325,
      "rewards/margins": 0.027831794694066048,
      "rewards/rejected": -0.19682832062244415,
      "step": 283
    },
    {
      "epoch": 0.44759653270291566,
      "grad_norm": 0.249053955078125,
      "learning_rate": 3.8189524659518355e-06,
      "log_odds_chosen": 0.38190656900405884,
      "log_odds_ratio": -0.5236896276473999,
      "logits/chosen": 0.48760396242141724,
      "logits/rejected": -0.6940497756004333,
      "logps/chosen": -1.7510186433792114,
      "logps/rejected": -2.0749480724334717,
      "loss": 1.8959,
      "nll_loss": 1.8434971570968628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1751018613576889,
      "rewards/margins": 0.0323929525911808,
      "rewards/rejected": -0.2074948251247406,
      "step": 284
    },
    {
      "epoch": 0.4491725768321513,
      "grad_norm": 0.26106294989585876,
      "learning_rate": 3.816660146322667e-06,
      "log_odds_chosen": 0.31609034538269043,
      "log_odds_ratio": -0.5512754917144775,
      "logits/chosen": 0.42169806361198425,
      "logits/rejected": -0.4507894217967987,
      "logps/chosen": -1.6568574905395508,
      "logps/rejected": -1.9196453094482422,
      "loss": 1.8096,
      "nll_loss": 1.7544704675674438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1656857579946518,
      "rewards/margins": 0.026278777047991753,
      "rewards/rejected": -0.1919645369052887,
      "step": 285
    },
    {
      "epoch": 0.4507486209613869,
      "grad_norm": 0.24721866846084595,
      "learning_rate": 3.814354102693797e-06,
      "log_odds_chosen": 0.4020289182662964,
      "log_odds_ratio": -0.519049882888794,
      "logits/chosen": 0.4439999759197235,
      "logits/rejected": -0.6853295564651489,
      "logps/chosen": -1.718071699142456,
      "logps/rejected": -2.0572900772094727,
      "loss": 1.8677,
      "nll_loss": 1.8157658576965332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1718071699142456,
      "rewards/margins": 0.03392184525728226,
      "rewards/rejected": -0.20572902262210846,
      "step": 286
    },
    {
      "epoch": 0.4523246650906225,
      "grad_norm": 0.2539658844470978,
      "learning_rate": 3.8120343524862814e-06,
      "log_odds_chosen": 0.2967430651187897,
      "log_odds_ratio": -0.5581727027893066,
      "logits/chosen": 0.40530580282211304,
      "logits/rejected": -0.6203088760375977,
      "logps/chosen": -1.7594120502471924,
      "logps/rejected": -2.0090889930725098,
      "loss": 1.8925,
      "nll_loss": 1.8366661071777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17594121396541595,
      "rewards/margins": 0.024967655539512634,
      "rewards/rejected": -0.2009088695049286,
      "step": 287
    },
    {
      "epoch": 0.45390070921985815,
      "grad_norm": 0.24290117621421814,
      "learning_rate": 3.809700913224726e-06,
      "log_odds_chosen": 0.41434168815612793,
      "log_odds_ratio": -0.5204272866249084,
      "logits/chosen": 0.367045521736145,
      "logits/rejected": -1.0038524866104126,
      "logps/chosen": -1.6730784177780151,
      "logps/rejected": -2.0234670639038086,
      "loss": 1.8168,
      "nll_loss": 1.764800786972046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16730786859989166,
      "rewards/margins": 0.03503885120153427,
      "rewards/rejected": -0.20234668254852295,
      "step": 288
    },
    {
      "epoch": 0.45547675334909377,
      "grad_norm": 0.2513137757778168,
      "learning_rate": 3.8073538025371494e-06,
      "log_odds_chosen": 0.49752530455589294,
      "log_odds_ratio": -0.4798320233821869,
      "logits/chosen": 0.4955059885978699,
      "logits/rejected": -0.8152831792831421,
      "logps/chosen": -1.6548831462860107,
      "logps/rejected": -2.0759389400482178,
      "loss": 1.8087,
      "nll_loss": 1.7607529163360596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16548830270767212,
      "rewards/margins": 0.04210559278726578,
      "rewards/rejected": -0.2075939029455185,
      "step": 289
    },
    {
      "epoch": 0.4570527974783294,
      "grad_norm": 0.2759801745414734,
      "learning_rate": 3.804993038154852e-06,
      "log_odds_chosen": 0.37791135907173157,
      "log_odds_ratio": -0.5382211208343506,
      "logits/chosen": 0.4199434518814087,
      "logits/rejected": -0.7131822109222412,
      "logps/chosen": -1.695268154144287,
      "logps/rejected": -2.019188165664673,
      "loss": 1.8613,
      "nll_loss": 1.8074455261230469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1695268154144287,
      "rewards/margins": 0.03239201009273529,
      "rewards/rejected": -0.201918825507164,
      "step": 290
    },
    {
      "epoch": 0.458628841607565,
      "grad_norm": 0.2639031708240509,
      "learning_rate": 3.8026186379122816e-06,
      "log_odds_chosen": 0.2410442978143692,
      "log_odds_ratio": -0.5826308727264404,
      "logits/chosen": 0.36750051379203796,
      "logits/rejected": -0.7549840807914734,
      "logps/chosen": -1.7680672407150269,
      "logps/rejected": -1.972318410873413,
      "loss": 1.9077,
      "nll_loss": 1.8494325876235962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1768067330121994,
      "rewards/margins": 0.02042512409389019,
      "rewards/rejected": -0.19723184406757355,
      "step": 291
    },
    {
      "epoch": 0.46020488573680063,
      "grad_norm": 0.24375998973846436,
      "learning_rate": 3.8002306197468983e-06,
      "log_odds_chosen": 0.20314940810203552,
      "log_odds_ratio": -0.6030868291854858,
      "logits/chosen": 0.3567318320274353,
      "logits/rejected": -0.7969244122505188,
      "logps/chosen": -1.7306976318359375,
      "logps/rejected": -1.901548981666565,
      "loss": 1.8857,
      "nll_loss": 1.8254362344741821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17306974530220032,
      "rewards/margins": 0.01708514243364334,
      "rewards/rejected": -0.19015489518642426,
      "step": 292
    },
    {
      "epoch": 0.46178092986603625,
      "grad_norm": 0.24072591960430145,
      "learning_rate": 3.7978290016990367e-06,
      "log_odds_chosen": 0.4728235602378845,
      "log_odds_ratio": -0.48956677317619324,
      "logits/chosen": 0.346945583820343,
      "logits/rejected": -0.7346411943435669,
      "logps/chosen": -1.7421035766601562,
      "logps/rejected": -2.148958444595337,
      "loss": 1.8835,
      "nll_loss": 1.8345091342926025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17421036958694458,
      "rewards/margins": 0.040685467422008514,
      "rewards/rejected": -0.2148958295583725,
      "step": 293
    },
    {
      "epoch": 0.46335697399527187,
      "grad_norm": 0.2703053951263428,
      "learning_rate": 3.795413801911776e-06,
      "log_odds_chosen": 0.6493099927902222,
      "log_odds_ratio": -0.43461745977401733,
      "logits/chosen": 0.34954923391342163,
      "logits/rejected": -0.8253506422042847,
      "logps/chosen": -1.6791515350341797,
      "logps/rejected": -2.2405524253845215,
      "loss": 1.8362,
      "nll_loss": 1.7926918268203735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16791515052318573,
      "rewards/margins": 0.05614008009433746,
      "rewards/rejected": -0.2240552306175232,
      "step": 294
    },
    {
      "epoch": 0.4649330181245075,
      "grad_norm": 0.2824196219444275,
      "learning_rate": 3.7929850386307965e-06,
      "log_odds_chosen": 0.31321728229522705,
      "log_odds_ratio": -0.5547051429748535,
      "logits/chosen": 0.4384803771972656,
      "logits/rejected": -0.619215726852417,
      "logps/chosen": -1.8919639587402344,
      "logps/rejected": -2.1647329330444336,
      "loss": 2.0217,
      "nll_loss": 1.9662539958953857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1891964077949524,
      "rewards/margins": 0.027276910841464996,
      "rewards/rejected": -0.21647332608699799,
      "step": 295
    },
    {
      "epoch": 0.4665090622537431,
      "grad_norm": 0.25732651352882385,
      "learning_rate": 3.790542730204245e-06,
      "log_odds_chosen": 0.3414647579193115,
      "log_odds_ratio": -0.5392407178878784,
      "logits/chosen": 0.3852652311325073,
      "logits/rejected": -0.8155044913291931,
      "logps/chosen": -1.8048094511032104,
      "logps/rejected": -2.098794460296631,
      "loss": 1.9317,
      "nll_loss": 1.877764105796814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1804809719324112,
      "rewards/margins": 0.02939850091934204,
      "rewards/rejected": -0.20987947285175323,
      "step": 296
    },
    {
      "epoch": 0.46808510638297873,
      "grad_norm": 0.26373788714408875,
      "learning_rate": 3.7880868950825935e-06,
      "log_odds_chosen": 0.4434778094291687,
      "log_odds_ratio": -0.4981243908405304,
      "logits/chosen": 0.3620328903198242,
      "logits/rejected": -0.8764305114746094,
      "logps/chosen": -1.7774946689605713,
      "logps/rejected": -2.1578266620635986,
      "loss": 1.8923,
      "nll_loss": 1.842448353767395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17774948477745056,
      "rewards/margins": 0.03803320974111557,
      "rewards/rejected": -0.21578267216682434,
      "step": 297
    },
    {
      "epoch": 0.46966115051221435,
      "grad_norm": 0.26903581619262695,
      "learning_rate": 3.7856175518185058e-06,
      "log_odds_chosen": 0.38129091262817383,
      "log_odds_ratio": -0.5281710028648376,
      "logits/chosen": 0.20757347345352173,
      "logits/rejected": -0.9698464870452881,
      "logps/chosen": -1.742034912109375,
      "logps/rejected": -2.0659961700439453,
      "loss": 1.888,
      "nll_loss": 1.8351514339447021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17420348525047302,
      "rewards/margins": 0.032396137714385986,
      "rewards/rejected": -0.206599622964859,
      "step": 298
    },
    {
      "epoch": 0.47123719464145,
      "grad_norm": 0.2571781575679779,
      "learning_rate": 3.7831347190666883e-06,
      "log_odds_chosen": 0.4503501057624817,
      "log_odds_ratio": -0.49767959117889404,
      "logits/chosen": 0.29682573676109314,
      "logits/rejected": -0.8833941221237183,
      "logps/chosen": -1.7364448308944702,
      "logps/rejected": -2.1207523345947266,
      "loss": 1.8836,
      "nll_loss": 1.8338348865509033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17364446818828583,
      "rewards/margins": 0.03843075782060623,
      "rewards/rejected": -0.21207523345947266,
      "step": 299
    },
    {
      "epoch": 0.4728132387706856,
      "grad_norm": 0.2652571499347687,
      "learning_rate": 3.780638415583759e-06,
      "log_odds_chosen": 0.2772998809814453,
      "log_odds_ratio": -0.5754559636116028,
      "logits/chosen": 0.41935643553733826,
      "logits/rejected": -0.6864033341407776,
      "logps/chosen": -1.7198681831359863,
      "logps/rejected": -1.9582655429840088,
      "loss": 1.8772,
      "nll_loss": 1.819682240486145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17198681831359863,
      "rewards/margins": 0.02383972890675068,
      "rewards/rejected": -0.19582654535770416,
      "step": 300
    },
    {
      "epoch": 0.4743892828999212,
      "grad_norm": 0.26339584589004517,
      "learning_rate": 3.7781286602280967e-06,
      "log_odds_chosen": 0.1517128050327301,
      "log_odds_ratio": -0.6284233331680298,
      "logits/chosen": 0.3332933187484741,
      "logits/rejected": -0.9037990570068359,
      "logps/chosen": -1.7000682353973389,
      "logps/rejected": -1.8223541975021362,
      "loss": 1.8654,
      "nll_loss": 1.802597165107727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17000681161880493,
      "rewards/margins": 0.012228596955537796,
      "rewards/rejected": -0.18223541975021362,
      "step": 301
    },
    {
      "epoch": 0.47596532702915684,
      "grad_norm": 0.24838051199913025,
      "learning_rate": 3.7756054719597044e-06,
      "log_odds_chosen": 0.23710456490516663,
      "log_odds_ratio": -0.617565393447876,
      "logits/chosen": 0.23182198405265808,
      "logits/rejected": -0.9376928806304932,
      "logps/chosen": -1.7347861528396606,
      "logps/rejected": -1.9488537311553955,
      "loss": 1.8729,
      "nll_loss": 1.8111293315887451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1734786182641983,
      "rewards/margins": 0.021406756713986397,
      "rewards/rejected": -0.19488537311553955,
      "step": 302
    },
    {
      "epoch": 0.47754137115839246,
      "grad_norm": 0.3430563509464264,
      "learning_rate": 3.773068869840066e-06,
      "log_odds_chosen": 0.21702063083648682,
      "log_odds_ratio": -0.5988757014274597,
      "logits/chosen": 0.4508088231086731,
      "logits/rejected": -0.9070717096328735,
      "logps/chosen": -1.7604243755340576,
      "logps/rejected": -1.9423820972442627,
      "loss": 1.8963,
      "nll_loss": 1.8363980054855347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17604243755340576,
      "rewards/margins": 0.018195772543549538,
      "rewards/rejected": -0.19423823058605194,
      "step": 303
    },
    {
      "epoch": 0.4791174152876281,
      "grad_norm": 0.23839880526065826,
      "learning_rate": 3.770518873031997e-06,
      "log_odds_chosen": 0.4319533109664917,
      "log_odds_ratio": -0.5028316378593445,
      "logits/chosen": 0.21307101845741272,
      "logits/rejected": -0.6253250241279602,
      "logps/chosen": -1.619141936302185,
      "logps/rejected": -1.9784443378448486,
      "loss": 1.7785,
      "nll_loss": 1.7282602787017822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16191419959068298,
      "rewards/margins": 0.03593023866415024,
      "rewards/rejected": -0.197844460606575,
      "step": 304
    },
    {
      "epoch": 0.4806934594168637,
      "grad_norm": 0.2817673683166504,
      "learning_rate": 3.7679555007995065e-06,
      "log_odds_chosen": 0.4497647285461426,
      "log_odds_ratio": -0.5144132375717163,
      "logits/chosen": 0.3390055298805237,
      "logits/rejected": -0.7776145935058594,
      "logps/chosen": -1.7621970176696777,
      "logps/rejected": -2.1498844623565674,
      "loss": 1.908,
      "nll_loss": 1.856536865234375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17621970176696777,
      "rewards/margins": 0.03876873850822449,
      "rewards/rejected": -0.21498845517635345,
      "step": 305
    },
    {
      "epoch": 0.48226950354609927,
      "grad_norm": 0.26427942514419556,
      "learning_rate": 3.7653787725076464e-06,
      "log_odds_chosen": 0.2555605173110962,
      "log_odds_ratio": -0.5800157785415649,
      "logits/chosen": 0.3456537425518036,
      "logits/rejected": -0.738465428352356,
      "logps/chosen": -1.7156627178192139,
      "logps/rejected": -1.9342702627182007,
      "loss": 1.8576,
      "nll_loss": 1.7995985746383667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17156627774238586,
      "rewards/margins": 0.02186075784265995,
      "rewards/rejected": -0.19342704117298126,
      "step": 306
    },
    {
      "epoch": 0.4838455476753349,
      "grad_norm": 0.2824785113334656,
      "learning_rate": 3.7627887076223685e-06,
      "log_odds_chosen": 0.3448987305164337,
      "log_odds_ratio": -0.5420399904251099,
      "logits/chosen": 0.3761395215988159,
      "logits/rejected": -0.5656124353408813,
      "logps/chosen": -1.7869853973388672,
      "logps/rejected": -2.0822932720184326,
      "loss": 1.9128,
      "nll_loss": 1.8585689067840576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17869853973388672,
      "rewards/margins": 0.02953079342842102,
      "rewards/rejected": -0.20822934806346893,
      "step": 307
    },
    {
      "epoch": 0.4854215918045705,
      "grad_norm": 0.2356492429971695,
      "learning_rate": 3.7601853257103765e-06,
      "log_odds_chosen": 0.17670860886573792,
      "log_odds_ratio": -0.6152773499488831,
      "logits/chosen": 0.26213836669921875,
      "logits/rejected": -0.832030177116394,
      "logps/chosen": -1.6793239116668701,
      "logps/rejected": -1.829579472541809,
      "loss": 1.81,
      "nll_loss": 1.7484831809997559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.167932391166687,
      "rewards/margins": 0.015025541186332703,
      "rewards/rejected": -0.18295791745185852,
      "step": 308
    },
    {
      "epoch": 0.48699763593380613,
      "grad_norm": 0.24143275618553162,
      "learning_rate": 3.7575686464389767e-06,
      "log_odds_chosen": 0.32946810126304626,
      "log_odds_ratio": -0.5521824359893799,
      "logits/chosen": 0.3027976453304291,
      "logits/rejected": -0.8635303378105164,
      "logps/chosen": -1.655510663986206,
      "logps/rejected": -1.9322257041931152,
      "loss": 1.8093,
      "nll_loss": 1.754052996635437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1655510514974594,
      "rewards/margins": 0.027671512216329575,
      "rewards/rejected": -0.19322258234024048,
      "step": 309
    },
    {
      "epoch": 0.48857368006304175,
      "grad_norm": 0.24138718843460083,
      "learning_rate": 3.7549386895759315e-06,
      "log_odds_chosen": 0.364417165517807,
      "log_odds_ratio": -0.5298891663551331,
      "logits/chosen": 0.26288849115371704,
      "logits/rejected": -0.7961046099662781,
      "logps/chosen": -1.605391025543213,
      "logps/rejected": -1.9066240787506104,
      "loss": 1.7459,
      "nll_loss": 1.692944049835205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16053910553455353,
      "rewards/margins": 0.030123300850391388,
      "rewards/rejected": -0.19066239893436432,
      "step": 310
    },
    {
      "epoch": 0.49014972419227737,
      "grad_norm": 0.4169044494628906,
      "learning_rate": 3.7522954749893086e-06,
      "log_odds_chosen": 0.609459638595581,
      "log_odds_ratio": -0.4481661021709442,
      "logits/chosen": 0.1874232292175293,
      "logits/rejected": -0.9124564528465271,
      "logps/chosen": -1.6414852142333984,
      "logps/rejected": -2.157813549041748,
      "loss": 1.7921,
      "nll_loss": 1.7472877502441406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1641485095024109,
      "rewards/margins": 0.051632851362228394,
      "rewards/rejected": -0.21578137576580048,
      "step": 311
    },
    {
      "epoch": 0.491725768321513,
      "grad_norm": 0.2586691379547119,
      "learning_rate": 3.749639022647332e-06,
      "log_odds_chosen": 0.3325248658657074,
      "log_odds_ratio": -0.5451020002365112,
      "logits/chosen": 0.24192464351654053,
      "logits/rejected": -0.8050363063812256,
      "logps/chosen": -1.7278075218200684,
      "logps/rejected": -2.010347366333008,
      "loss": 1.8693,
      "nll_loss": 1.814815878868103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17278075218200684,
      "rewards/margins": 0.02825397439301014,
      "rewards/rejected": -0.20103472471237183,
      "step": 312
    },
    {
      "epoch": 0.4933018124507486,
      "grad_norm": 0.22876015305519104,
      "learning_rate": 3.7469693526182304e-06,
      "log_odds_chosen": 0.4910639524459839,
      "log_odds_ratio": -0.4973025321960449,
      "logits/chosen": 0.18789926171302795,
      "logits/rejected": -1.2326761484146118,
      "logps/chosen": -1.6729106903076172,
      "logps/rejected": -2.0879974365234375,
      "loss": 1.7994,
      "nll_loss": 1.749713659286499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.167291060090065,
      "rewards/margins": 0.041508689522743225,
      "rewards/rejected": -0.20879974961280823,
      "step": 313
    },
    {
      "epoch": 0.49487785657998423,
      "grad_norm": 0.23891504108905792,
      "learning_rate": 3.744286485070085e-06,
      "log_odds_chosen": 0.46477484703063965,
      "log_odds_ratio": -0.49088478088378906,
      "logits/chosen": 0.2151736468076706,
      "logits/rejected": -0.9330959916114807,
      "logps/chosen": -1.6210802793502808,
      "logps/rejected": -2.0096402168273926,
      "loss": 1.754,
      "nll_loss": 1.7048873901367188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16210801899433136,
      "rewards/margins": 0.038856007158756256,
      "rewards/rejected": -0.20096401870250702,
      "step": 314
    },
    {
      "epoch": 0.49645390070921985,
      "grad_norm": 0.2731998562812805,
      "learning_rate": 3.7415904402706795e-06,
      "log_odds_chosen": 0.3603336811065674,
      "log_odds_ratio": -0.5388191342353821,
      "logits/chosen": 0.2712251543998718,
      "logits/rejected": -0.8804117441177368,
      "logps/chosen": -1.8475456237792969,
      "logps/rejected": -2.1596999168395996,
      "loss": 1.9762,
      "nll_loss": 1.9222995042800903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18475456535816193,
      "rewards/margins": 0.031215444207191467,
      "rewards/rejected": -0.2159700095653534,
      "step": 315
    },
    {
      "epoch": 0.4980299448384555,
      "grad_norm": 0.24663853645324707,
      "learning_rate": 3.7388812385873435e-06,
      "log_odds_chosen": 0.321816086769104,
      "log_odds_ratio": -0.5476392507553101,
      "logits/chosen": 0.21599149703979492,
      "logits/rejected": -0.9269928336143494,
      "logps/chosen": -1.6557197570800781,
      "logps/rejected": -1.9236208200454712,
      "loss": 1.7899,
      "nll_loss": 1.7351197004318237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16557197272777557,
      "rewards/margins": 0.026790114119648933,
      "rewards/rejected": -0.19236209988594055,
      "step": 316
    },
    {
      "epoch": 0.4996059889676911,
      "grad_norm": 0.25920945405960083,
      "learning_rate": 3.7361589004868033e-06,
      "log_odds_chosen": 0.549557626247406,
      "log_odds_ratio": -0.46774470806121826,
      "logits/chosen": 0.33838802576065063,
      "logits/rejected": -0.9401962757110596,
      "logps/chosen": -1.7451956272125244,
      "logps/rejected": -2.222694158554077,
      "loss": 1.8613,
      "nll_loss": 1.8144901990890503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17451956868171692,
      "rewards/margins": 0.047749847173690796,
      "rewards/rejected": -0.22226941585540771,
      "step": 317
    },
    {
      "epoch": 0.5011820330969267,
      "grad_norm": 0.2366023063659668,
      "learning_rate": 3.733423446535022e-06,
      "log_odds_chosen": 0.5207498669624329,
      "log_odds_ratio": -0.4805837869644165,
      "logits/chosen": 0.30363941192626953,
      "logits/rejected": -1.0254158973693848,
      "logps/chosen": -1.738085150718689,
      "logps/rejected": -2.1922197341918945,
      "loss": 1.8581,
      "nll_loss": 1.8100310564041138,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17380854487419128,
      "rewards/margins": 0.04541344195604324,
      "rewards/rejected": -0.21922197937965393,
      "step": 318
    },
    {
      "epoch": 0.5027580772261623,
      "grad_norm": 0.23470795154571533,
      "learning_rate": 3.7306748973970476e-06,
      "log_odds_chosen": 0.34376969933509827,
      "log_odds_ratio": -0.5430240035057068,
      "logits/chosen": 0.20498515665531158,
      "logits/rejected": -1.1637543439865112,
      "logps/chosen": -1.5948607921600342,
      "logps/rejected": -1.8776003122329712,
      "loss": 1.7581,
      "nll_loss": 1.7037681341171265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1594860851764679,
      "rewards/margins": 0.02827395498752594,
      "rewards/rejected": -0.18776002526283264,
      "step": 319
    },
    {
      "epoch": 0.5043341213553979,
      "grad_norm": 0.23327548801898956,
      "learning_rate": 3.7279132738368564e-06,
      "log_odds_chosen": 0.17803671956062317,
      "log_odds_ratio": -0.6121765375137329,
      "logits/chosen": 0.2815796136856079,
      "logits/rejected": -0.9793424606323242,
      "logps/chosen": -1.7961714267730713,
      "logps/rejected": -1.9467929601669312,
      "loss": 1.9245,
      "nll_loss": 1.863244652748108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17961712181568146,
      "rewards/margins": 0.015062162652611732,
      "rewards/rejected": -0.19467930495738983,
      "step": 320
    },
    {
      "epoch": 0.5059101654846335,
      "grad_norm": 0.24164979159832,
      "learning_rate": 3.725138596717195e-06,
      "log_odds_chosen": 0.5048070549964905,
      "log_odds_ratio": -0.4779382348060608,
      "logits/chosen": 0.20476730167865753,
      "logits/rejected": -0.9783819913864136,
      "logps/chosen": -1.6493675708770752,
      "logps/rejected": -2.0773749351501465,
      "loss": 1.7734,
      "nll_loss": 1.7255733013153076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16493676602840424,
      "rewards/margins": 0.04280070960521698,
      "rewards/rejected": -0.2077374905347824,
      "step": 321
    },
    {
      "epoch": 0.5074862096138691,
      "grad_norm": 0.23102499544620514,
      "learning_rate": 3.7223508869994244e-06,
      "log_odds_chosen": 0.42121315002441406,
      "log_odds_ratio": -0.5144806504249573,
      "logits/chosen": 0.23960447311401367,
      "logits/rejected": -1.09214448928833,
      "logps/chosen": -1.702633261680603,
      "logps/rejected": -2.0583620071411133,
      "loss": 1.833,
      "nll_loss": 1.7815495729446411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17026333510875702,
      "rewards/margins": 0.035572879016399384,
      "rewards/rejected": -0.2058362066745758,
      "step": 322
    },
    {
      "epoch": 0.5090622537431048,
      "grad_norm": 0.2813505530357361,
      "learning_rate": 3.7195501657433594e-06,
      "log_odds_chosen": 0.33818790316581726,
      "log_odds_ratio": -0.5427064895629883,
      "logits/chosen": 0.1689353883266449,
      "logits/rejected": -1.0303304195404053,
      "logps/chosen": -1.7009472846984863,
      "logps/rejected": -1.984174132347107,
      "loss": 1.8396,
      "nll_loss": 1.7853001356124878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17009472846984863,
      "rewards/margins": 0.028322694823145866,
      "rewards/rejected": -0.19841741025447845,
      "step": 323
    },
    {
      "epoch": 0.5106382978723404,
      "grad_norm": 0.22637781500816345,
      "learning_rate": 3.716736454107111e-06,
      "log_odds_chosen": 0.5035085082054138,
      "log_odds_ratio": -0.4778319299221039,
      "logits/chosen": 0.1796778440475464,
      "logits/rejected": -0.8279569149017334,
      "logps/chosen": -1.5848971605300903,
      "logps/rejected": -2.0021541118621826,
      "loss": 1.7142,
      "nll_loss": 1.6663726568222046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15848970413208008,
      "rewards/margins": 0.04172568395733833,
      "rewards/rejected": -0.2002153992652893,
      "step": 324
    },
    {
      "epoch": 0.512214342001576,
      "grad_norm": 0.2479918897151947,
      "learning_rate": 3.7139097733469277e-06,
      "log_odds_chosen": 0.5620511770248413,
      "log_odds_ratio": -0.4595773220062256,
      "logits/chosen": 0.20734019577503204,
      "logits/rejected": -0.9722301363945007,
      "logps/chosen": -1.644300937652588,
      "logps/rejected": -2.121246099472046,
      "loss": 1.7752,
      "nll_loss": 1.729271411895752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16443009674549103,
      "rewards/margins": 0.047694504261016846,
      "rewards/rejected": -0.21212461590766907,
      "step": 325
    },
    {
      "epoch": 0.5137903861308116,
      "grad_norm": 0.25066426396369934,
      "learning_rate": 3.711070144817032e-06,
      "log_odds_chosen": 0.3923027515411377,
      "log_odds_ratio": -0.5236109495162964,
      "logits/chosen": 0.18746960163116455,
      "logits/rejected": -0.8730261921882629,
      "logps/chosen": -1.737861156463623,
      "logps/rejected": -2.0755648612976074,
      "loss": 1.8758,
      "nll_loss": 1.8234347105026245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17378610372543335,
      "rewards/margins": 0.03377038985490799,
      "rewards/rejected": -0.20755648612976074,
      "step": 326
    },
    {
      "epoch": 0.5153664302600472,
      "grad_norm": 0.25296109914779663,
      "learning_rate": 3.708217589969461e-06,
      "log_odds_chosen": 0.49952542781829834,
      "log_odds_ratio": -0.4828339219093323,
      "logits/chosen": 0.13944938778877258,
      "logits/rejected": -1.0598782300949097,
      "logps/chosen": -1.7016721963882446,
      "logps/rejected": -2.1262779235839844,
      "loss": 1.8419,
      "nll_loss": 1.793623447418213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1701672375202179,
      "rewards/margins": 0.04246056079864502,
      "rewards/rejected": -0.21262778341770172,
      "step": 327
    },
    {
      "epoch": 0.5169424743892829,
      "grad_norm": 0.223394513130188,
      "learning_rate": 3.705352130353904e-06,
      "log_odds_chosen": 0.4204602837562561,
      "log_odds_ratio": -0.519094705581665,
      "logits/chosen": 0.20659029483795166,
      "logits/rejected": -1.0534716844558716,
      "logps/chosen": -1.6077661514282227,
      "logps/rejected": -1.9581828117370605,
      "loss": 1.7557,
      "nll_loss": 1.7038142681121826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16077663004398346,
      "rewards/margins": 0.03504165634512901,
      "rewards/rejected": -0.19581827521324158,
      "step": 328
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 0.2492428869009018,
      "learning_rate": 3.7024737876175404e-06,
      "log_odds_chosen": 0.39084863662719727,
      "log_odds_ratio": -0.5200874209403992,
      "logits/chosen": 0.17618009448051453,
      "logits/rejected": -0.8343906998634338,
      "logps/chosen": -1.6858640909194946,
      "logps/rejected": -2.01192307472229,
      "loss": 1.8105,
      "nll_loss": 1.7585185766220093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16858640313148499,
      "rewards/margins": 0.03260589390993118,
      "rewards/rejected": -0.20119231939315796,
      "step": 329
    },
    {
      "epoch": 0.5200945626477541,
      "grad_norm": 0.24973885715007782,
      "learning_rate": 3.699582583504874e-06,
      "log_odds_chosen": 0.31327661871910095,
      "log_odds_ratio": -0.5593623518943787,
      "logits/chosen": 0.20105372369289398,
      "logits/rejected": -0.9870561361312866,
      "logps/chosen": -1.8093860149383545,
      "logps/rejected": -2.079317569732666,
      "loss": 1.9242,
      "nll_loss": 1.8682433366775513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18093861639499664,
      "rewards/margins": 0.02699316293001175,
      "rewards/rejected": -0.2079317718744278,
      "step": 330
    },
    {
      "epoch": 0.5216706067769897,
      "grad_norm": 0.24848176538944244,
      "learning_rate": 3.696678539857571e-06,
      "log_odds_chosen": 0.4067533314228058,
      "log_odds_ratio": -0.5259693264961243,
      "logits/chosen": 0.15454429388046265,
      "logits/rejected": -1.0274417400360107,
      "logps/chosen": -1.783982515335083,
      "logps/rejected": -2.1373672485351562,
      "loss": 1.9076,
      "nll_loss": 1.8550231456756592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1783982515335083,
      "rewards/margins": 0.03533848002552986,
      "rewards/rejected": -0.21373675763607025,
      "step": 331
    },
    {
      "epoch": 0.5232466509062254,
      "grad_norm": 0.24895605444908142,
      "learning_rate": 3.6937616786142956e-06,
      "log_odds_chosen": 0.4905010759830475,
      "log_odds_ratio": -0.4821808338165283,
      "logits/chosen": 0.19289091229438782,
      "logits/rejected": -0.7941063642501831,
      "logps/chosen": -1.6132020950317383,
      "logps/rejected": -2.0248632431030273,
      "loss": 1.7617,
      "nll_loss": 1.7134675979614258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16132019460201263,
      "rewards/margins": 0.04116611182689667,
      "rewards/rejected": -0.2024863064289093,
      "step": 332
    },
    {
      "epoch": 0.524822695035461,
      "grad_norm": 0.2128501832485199,
      "learning_rate": 3.6908320218105393e-06,
      "log_odds_chosen": 0.28758326172828674,
      "log_odds_ratio": -0.5683539509773254,
      "logits/chosen": 0.1749098002910614,
      "logits/rejected": -0.9845055341720581,
      "logps/chosen": -1.6016740798950195,
      "logps/rejected": -1.8421180248260498,
      "loss": 1.7326,
      "nll_loss": 1.6757293939590454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1601674109697342,
      "rewards/margins": 0.02404441311955452,
      "rewards/rejected": -0.1842118203639984,
      "step": 333
    },
    {
      "epoch": 0.5263987391646966,
      "grad_norm": 0.21799606084823608,
      "learning_rate": 3.6878895915784607e-06,
      "log_odds_chosen": 0.4461507797241211,
      "log_odds_ratio": -0.4998866617679596,
      "logits/chosen": 0.20331019163131714,
      "logits/rejected": -0.9468520283699036,
      "logps/chosen": -1.565372109413147,
      "logps/rejected": -1.9329590797424316,
      "loss": 1.694,
      "nll_loss": 1.644019603729248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15653720498085022,
      "rewards/margins": 0.036758698523044586,
      "rewards/rejected": -0.1932959109544754,
      "step": 334
    },
    {
      "epoch": 0.5279747832939322,
      "grad_norm": 0.2156113237142563,
      "learning_rate": 3.6849344101467147e-06,
      "log_odds_chosen": 0.38997355103492737,
      "log_odds_ratio": -0.5264524221420288,
      "logits/chosen": 0.19628331065177917,
      "logits/rejected": -1.0862376689910889,
      "logps/chosen": -1.646598219871521,
      "logps/rejected": -1.9763703346252441,
      "loss": 1.7834,
      "nll_loss": 1.730759859085083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16465981304645538,
      "rewards/margins": 0.03297720104455948,
      "rewards/rejected": -0.19763700664043427,
      "step": 335
    },
    {
      "epoch": 0.5295508274231678,
      "grad_norm": 0.23609226942062378,
      "learning_rate": 3.6819664998402857e-06,
      "log_odds_chosen": 0.3619554936885834,
      "log_odds_ratio": -0.5322998762130737,
      "logits/chosen": 0.17890314757823944,
      "logits/rejected": -0.9327296018600464,
      "logps/chosen": -1.6979272365570068,
      "logps/rejected": -2.0048635005950928,
      "loss": 1.8282,
      "nll_loss": 1.7749249935150146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1697927564382553,
      "rewards/margins": 0.030693601816892624,
      "rewards/rejected": -0.20048634707927704,
      "step": 336
    },
    {
      "epoch": 0.5311268715524035,
      "grad_norm": 0.24500414729118347,
      "learning_rate": 3.6789858830803186e-06,
      "log_odds_chosen": 0.3548503518104553,
      "log_odds_ratio": -0.5371453762054443,
      "logits/chosen": 0.19275718927383423,
      "logits/rejected": -0.961004912853241,
      "logps/chosen": -1.751190423965454,
      "logps/rejected": -2.051436424255371,
      "loss": 1.8707,
      "nll_loss": 1.8169537782669067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17511902749538422,
      "rewards/margins": 0.03002461977303028,
      "rewards/rejected": -0.20514366030693054,
      "step": 337
    },
    {
      "epoch": 0.5327029156816391,
      "grad_norm": 0.2349151223897934,
      "learning_rate": 3.6759925823839486e-06,
      "log_odds_chosen": 0.2844861149787903,
      "log_odds_ratio": -0.5660067200660706,
      "logits/chosen": 0.14869461953639984,
      "logits/rejected": -0.836445689201355,
      "logps/chosen": -1.6251897811889648,
      "logps/rejected": -1.8610213994979858,
      "loss": 1.7789,
      "nll_loss": 1.722307562828064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1625189632177353,
      "rewards/margins": 0.023583177477121353,
      "rewards/rejected": -0.18610213696956635,
      "step": 338
    },
    {
      "epoch": 0.5342789598108747,
      "grad_norm": 0.21934184432029724,
      "learning_rate": 3.672986620364134e-06,
      "log_odds_chosen": 0.4246414601802826,
      "log_odds_ratio": -0.5082186460494995,
      "logits/chosen": 0.21185454726219177,
      "logits/rejected": -0.8254536986351013,
      "logps/chosen": -1.7098535299301147,
      "logps/rejected": -2.0699710845947266,
      "loss": 1.8338,
      "nll_loss": 1.7829303741455078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1709853559732437,
      "rewards/margins": 0.036011770367622375,
      "rewards/rejected": -0.2069971263408661,
      "step": 339
    },
    {
      "epoch": 0.5358550039401103,
      "grad_norm": 0.22029125690460205,
      "learning_rate": 3.669968019729481e-06,
      "log_odds_chosen": 0.4750698208808899,
      "log_odds_ratio": -0.4959007501602173,
      "logits/chosen": 0.14572562277317047,
      "logits/rejected": -1.0968286991119385,
      "logps/chosen": -1.5743448734283447,
      "logps/rejected": -1.9716068506240845,
      "loss": 1.7095,
      "nll_loss": 1.6599397659301758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15743449330329895,
      "rewards/margins": 0.039726193994283676,
      "rewards/rejected": -0.19716067612171173,
      "step": 340
    },
    {
      "epoch": 0.5374310480693459,
      "grad_norm": 0.2289605438709259,
      "learning_rate": 3.666936803284076e-06,
      "log_odds_chosen": 0.4306865632534027,
      "log_odds_ratio": -0.5031254887580872,
      "logits/chosen": 0.1732909083366394,
      "logits/rejected": -0.9109163880348206,
      "logps/chosen": -1.721662998199463,
      "logps/rejected": -2.0888686180114746,
      "loss": 1.8312,
      "nll_loss": 1.780855417251587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17216630280017853,
      "rewards/margins": 0.03672058507800102,
      "rewards/rejected": -0.20888687670230865,
      "step": 341
    },
    {
      "epoch": 0.5390070921985816,
      "grad_norm": 0.23886913061141968,
      "learning_rate": 3.663892993927312e-06,
      "log_odds_chosen": 0.5287318825721741,
      "log_odds_ratio": -0.46768540143966675,
      "logits/chosen": 0.10965054482221603,
      "logits/rejected": -1.0303406715393066,
      "logps/chosen": -1.671059012413025,
      "logps/rejected": -2.1188931465148926,
      "loss": 1.8029,
      "nll_loss": 1.7561312913894653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16710591316223145,
      "rewards/margins": 0.044783420860767365,
      "rewards/rejected": -0.2118893265724182,
      "step": 342
    },
    {
      "epoch": 0.5405831363278172,
      "grad_norm": 0.2293994128704071,
      "learning_rate": 3.6608366146537136e-06,
      "log_odds_chosen": 0.552444577217102,
      "log_odds_ratio": -0.47390323877334595,
      "logits/chosen": 0.09345658123493195,
      "logits/rejected": -0.875726580619812,
      "logps/chosen": -1.5994176864624023,
      "logps/rejected": -2.0705385208129883,
      "loss": 1.7346,
      "nll_loss": 1.6872093677520752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15994176268577576,
      "rewards/margins": 0.047112107276916504,
      "rewards/rejected": -0.20705386996269226,
      "step": 343
    },
    {
      "epoch": 0.5421591804570528,
      "grad_norm": 0.2236412763595581,
      "learning_rate": 3.6577676885527674e-06,
      "log_odds_chosen": 0.46793222427368164,
      "log_odds_ratio": -0.4905901551246643,
      "logits/chosen": 0.10005174577236176,
      "logits/rejected": -1.1285041570663452,
      "logps/chosen": -1.649950623512268,
      "logps/rejected": -2.043297290802002,
      "loss": 1.7655,
      "nll_loss": 1.716444492340088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16499507427215576,
      "rewards/margins": 0.03933466970920563,
      "rewards/rejected": -0.2043297290802002,
      "step": 344
    },
    {
      "epoch": 0.5437352245862884,
      "grad_norm": 0.21764251589775085,
      "learning_rate": 3.654686238808744e-06,
      "log_odds_chosen": 0.4737105667591095,
      "log_odds_ratio": -0.4936067461967468,
      "logits/chosen": 0.0544014573097229,
      "logits/rejected": -1.042222023010254,
      "logps/chosen": -1.746705174446106,
      "logps/rejected": -2.1518256664276123,
      "loss": 1.863,
      "nll_loss": 1.8135954141616821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1746705323457718,
      "rewards/margins": 0.04051202908158302,
      "rewards/rejected": -0.21518255770206451,
      "step": 345
    },
    {
      "epoch": 0.545311268715524,
      "grad_norm": 0.1986258327960968,
      "learning_rate": 3.6515922887005245e-06,
      "log_odds_chosen": 0.6257790923118591,
      "log_odds_ratio": -0.4334947466850281,
      "logits/chosen": -0.02682601287961006,
      "logits/rejected": -1.096382737159729,
      "logps/chosen": -1.5822004079818726,
      "logps/rejected": -2.1046130657196045,
      "loss": 1.7103,
      "nll_loss": 1.6669962406158447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1582200527191162,
      "rewards/margins": 0.052241258323192596,
      "rewards/rejected": -0.2104613035917282,
      "step": 346
    },
    {
      "epoch": 0.5468873128447597,
      "grad_norm": 0.20813308656215668,
      "learning_rate": 3.6484858616014236e-06,
      "log_odds_chosen": 0.256540983915329,
      "log_odds_ratio": -0.5760852098464966,
      "logits/chosen": 0.12902772426605225,
      "logits/rejected": -1.1435496807098389,
      "logps/chosen": -1.707273244857788,
      "logps/rejected": -1.9220017194747925,
      "loss": 1.8215,
      "nll_loss": 1.763900876045227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17072734236717224,
      "rewards/margins": 0.02147284336388111,
      "rewards/rejected": -0.19220015406608582,
      "step": 347
    },
    {
      "epoch": 0.5484633569739953,
      "grad_norm": 0.3381063640117645,
      "learning_rate": 3.6453669809790154e-06,
      "log_odds_chosen": 0.2898363769054413,
      "log_odds_ratio": -0.5690465569496155,
      "logits/chosen": 0.1767110675573349,
      "logits/rejected": -0.8112865090370178,
      "logps/chosen": -1.6394524574279785,
      "logps/rejected": -1.8821648359298706,
      "loss": 1.765,
      "nll_loss": 1.7080632448196411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1639452427625656,
      "rewards/margins": 0.02427123486995697,
      "rewards/rejected": -0.18821649253368378,
      "step": 348
    },
    {
      "epoch": 0.5500394011032309,
      "grad_norm": 0.21555578708648682,
      "learning_rate": 3.642235670394952e-06,
      "log_odds_chosen": 0.3761124312877655,
      "log_odds_ratio": -0.528813362121582,
      "logits/chosen": 0.13704705238342285,
      "logits/rejected": -0.8061936497688293,
      "logps/chosen": -1.6551097631454468,
      "logps/rejected": -1.9656535387039185,
      "loss": 1.7871,
      "nll_loss": 1.7342618703842163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16551096737384796,
      "rewards/margins": 0.03105437569320202,
      "rewards/rejected": -0.19656534492969513,
      "step": 349
    },
    {
      "epoch": 0.5516154452324665,
      "grad_norm": 0.20423917472362518,
      "learning_rate": 3.63909195350479e-06,
      "log_odds_chosen": 0.3855099081993103,
      "log_odds_ratio": -0.5282479524612427,
      "logits/chosen": 0.2054780125617981,
      "logits/rejected": -0.9389649629592896,
      "logps/chosen": -1.7321782112121582,
      "logps/rejected": -2.0566914081573486,
      "loss": 1.8494,
      "nll_loss": 1.7965946197509766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17321781814098358,
      "rewards/margins": 0.032451316714286804,
      "rewards/rejected": -0.20566913485527039,
      "step": 350
    },
    {
      "epoch": 0.5531914893617021,
      "grad_norm": 0.22335051000118256,
      "learning_rate": 3.635935854057809e-06,
      "log_odds_chosen": 0.3637022376060486,
      "log_odds_ratio": -0.5370145440101624,
      "logits/chosen": 0.021777212619781494,
      "logits/rejected": -0.6934305429458618,
      "logps/chosen": -1.7031822204589844,
      "logps/rejected": -2.01242733001709,
      "loss": 1.8245,
      "nll_loss": 1.770749568939209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17031821608543396,
      "rewards/margins": 0.03092453069984913,
      "rewards/rejected": -0.20124275982379913,
      "step": 351
    },
    {
      "epoch": 0.5547675334909378,
      "grad_norm": 0.21424627304077148,
      "learning_rate": 3.6327673958968327e-06,
      "log_odds_chosen": 0.5423768758773804,
      "log_odds_ratio": -0.4632525146007538,
      "logits/chosen": 0.051188066601753235,
      "logits/rejected": -0.9619206190109253,
      "logps/chosen": -1.5644475221633911,
      "logps/rejected": -2.0145986080169678,
      "loss": 1.7063,
      "nll_loss": 1.6599864959716797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1564447581768036,
      "rewards/margins": 0.045015111565589905,
      "rewards/rejected": -0.2014598697423935,
      "step": 352
    },
    {
      "epoch": 0.5563435776201734,
      "grad_norm": 0.2026013284921646,
      "learning_rate": 3.6295866029580483e-06,
      "log_odds_chosen": 0.4790729284286499,
      "log_odds_ratio": -0.4863028824329376,
      "logits/chosen": 0.06050941348075867,
      "logits/rejected": -1.0654728412628174,
      "logps/chosen": -1.605035424232483,
      "logps/rejected": -2.004734516143799,
      "loss": 1.7168,
      "nll_loss": 1.6681344509124756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.160503551363945,
      "rewards/margins": 0.039969902485609055,
      "rewards/rejected": -0.20047345757484436,
      "step": 353
    },
    {
      "epoch": 0.557919621749409,
      "grad_norm": 0.20405229926109314,
      "learning_rate": 3.626393499270829e-06,
      "log_odds_chosen": 0.4320348799228668,
      "log_odds_ratio": -0.5096400380134583,
      "logits/chosen": 0.07103009521961212,
      "logits/rejected": -1.0413506031036377,
      "logps/chosen": -1.5444334745407104,
      "logps/rejected": -1.9000470638275146,
      "loss": 1.6895,
      "nll_loss": 1.6385679244995117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15444333851337433,
      "rewards/margins": 0.03556136414408684,
      "rewards/rejected": -0.19000470638275146,
      "step": 354
    },
    {
      "epoch": 0.5594956658786446,
      "grad_norm": 0.2170814573764801,
      "learning_rate": 3.6231881089575466e-06,
      "log_odds_chosen": 0.5878276228904724,
      "log_odds_ratio": -0.4456479847431183,
      "logits/chosen": 0.027034137398004532,
      "logits/rejected": -1.0812389850616455,
      "logps/chosen": -1.6104373931884766,
      "logps/rejected": -2.1062474250793457,
      "loss": 1.7061,
      "nll_loss": 1.6615161895751953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16104374825954437,
      "rewards/margins": 0.04958099126815796,
      "rewards/rejected": -0.21062473952770233,
      "step": 355
    },
    {
      "epoch": 0.5610717100078803,
      "grad_norm": 0.2183144986629486,
      "learning_rate": 3.6199704562333945e-06,
      "log_odds_chosen": 0.40854716300964355,
      "log_odds_ratio": -0.5191488265991211,
      "logits/chosen": 0.029446076601743698,
      "logits/rejected": -1.0460023880004883,
      "logps/chosen": -1.5693233013153076,
      "logps/rejected": -1.903488278388977,
      "loss": 1.6957,
      "nll_loss": 1.643739938735962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15693233907222748,
      "rewards/margins": 0.0334165021777153,
      "rewards/rejected": -0.19034883379936218,
      "step": 356
    },
    {
      "epoch": 0.5626477541371159,
      "grad_norm": 0.2159062922000885,
      "learning_rate": 3.6167405654062024e-06,
      "log_odds_chosen": 0.4419677257537842,
      "log_odds_ratio": -0.5147860050201416,
      "logits/chosen": 0.08764218538999557,
      "logits/rejected": -0.9715548157691956,
      "logps/chosen": -1.6319985389709473,
      "logps/rejected": -2.0011050701141357,
      "loss": 1.7458,
      "nll_loss": 1.694358229637146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16319985687732697,
      "rewards/margins": 0.03691064938902855,
      "rewards/rejected": -0.2001105099916458,
      "step": 357
    },
    {
      "epoch": 0.5642237982663515,
      "grad_norm": 0.19851958751678467,
      "learning_rate": 3.6134984608762515e-06,
      "log_odds_chosen": 0.4827573299407959,
      "log_odds_ratio": -0.4849916100502014,
      "logits/chosen": 0.073295459151268,
      "logits/rejected": -1.0616111755371094,
      "logps/chosen": -1.5443633794784546,
      "logps/rejected": -1.9410005807876587,
      "loss": 1.6782,
      "nll_loss": 1.6297074556350708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15443634986877441,
      "rewards/margins": 0.03966371342539787,
      "rewards/rejected": -0.19410006701946259,
      "step": 358
    },
    {
      "epoch": 0.5657998423955871,
      "grad_norm": 0.23549191653728485,
      "learning_rate": 3.6102441671360945e-06,
      "log_odds_chosen": 0.4817807078361511,
      "log_odds_ratio": -0.49621692299842834,
      "logits/chosen": 0.05556647107005119,
      "logits/rejected": -0.9252523183822632,
      "logps/chosen": -1.7002850770950317,
      "logps/rejected": -2.1108858585357666,
      "loss": 1.8078,
      "nll_loss": 1.7582156658172607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17002852261066437,
      "rewards/margins": 0.04106004908680916,
      "rewards/rejected": -0.21108856797218323,
      "step": 359
    },
    {
      "epoch": 0.5673758865248227,
      "grad_norm": 0.2164311408996582,
      "learning_rate": 3.6069777087703654e-06,
      "log_odds_chosen": 0.47711795568466187,
      "log_odds_ratio": -0.4905511140823364,
      "logits/chosen": 0.06711583584547043,
      "logits/rejected": -0.9263973832130432,
      "logps/chosen": -1.6224663257598877,
      "logps/rejected": -2.021437168121338,
      "loss": 1.7555,
      "nll_loss": 1.7064476013183594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16224664449691772,
      "rewards/margins": 0.039897091686725616,
      "rewards/rejected": -0.20214374363422394,
      "step": 360
    },
    {
      "epoch": 0.5689519306540584,
      "grad_norm": 0.21710002422332764,
      "learning_rate": 3.6036991104555973e-06,
      "log_odds_chosen": 0.407501757144928,
      "log_odds_ratio": -0.5239275693893433,
      "logits/chosen": 0.07704886794090271,
      "logits/rejected": -0.9694811105728149,
      "logps/chosen": -1.7658026218414307,
      "logps/rejected": -2.117172956466675,
      "loss": 1.8712,
      "nll_loss": 1.8187607526779175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1765802651643753,
      "rewards/margins": 0.03513704985380173,
      "rewards/rejected": -0.21171732246875763,
      "step": 361
    },
    {
      "epoch": 0.570527974783294,
      "grad_norm": 0.20836122334003448,
      "learning_rate": 3.600408396960034e-06,
      "log_odds_chosen": 0.47454649209976196,
      "log_odds_ratio": -0.49302300810813904,
      "logits/chosen": 0.09529343247413635,
      "logits/rejected": -0.6717502474784851,
      "logps/chosen": -1.5604360103607178,
      "logps/rejected": -1.9463016986846924,
      "loss": 1.6685,
      "nll_loss": 1.6191602945327759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15604360401630402,
      "rewards/margins": 0.038586560636758804,
      "rewards/rejected": -0.19463016092777252,
      "step": 362
    },
    {
      "epoch": 0.5721040189125296,
      "grad_norm": 0.22342000901699066,
      "learning_rate": 3.5971055931434447e-06,
      "log_odds_chosen": 0.42324817180633545,
      "log_odds_ratio": -0.5131744742393494,
      "logits/chosen": 0.11431422084569931,
      "logits/rejected": -0.8911008834838867,
      "logps/chosen": -1.6157538890838623,
      "logps/rejected": -1.9717456102371216,
      "loss": 1.7379,
      "nll_loss": 1.6865862607955933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16157537698745728,
      "rewards/margins": 0.035599179565906525,
      "rewards/rejected": -0.1971745789051056,
      "step": 363
    },
    {
      "epoch": 0.5736800630417652,
      "grad_norm": 0.20487506687641144,
      "learning_rate": 3.5937907239569343e-06,
      "log_odds_chosen": 0.46312034130096436,
      "log_odds_ratio": -0.5019612908363342,
      "logits/chosen": 0.022356808185577393,
      "logits/rejected": -0.989889919757843,
      "logps/chosen": -1.667398452758789,
      "logps/rejected": -2.0568132400512695,
      "loss": 1.7903,
      "nll_loss": 1.7401193380355835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16673985123634338,
      "rewards/margins": 0.038941483944654465,
      "rewards/rejected": -0.20568133890628815,
      "step": 364
    },
    {
      "epoch": 0.5752561071710008,
      "grad_norm": 0.23341180384159088,
      "learning_rate": 3.5904638144427572e-06,
      "log_odds_chosen": 0.2531343400478363,
      "log_odds_ratio": -0.5829412937164307,
      "logits/chosen": 0.07688252627849579,
      "logits/rejected": -0.7990673780441284,
      "logps/chosen": -1.73116135597229,
      "logps/rejected": -1.9438457489013672,
      "loss": 1.8433,
      "nll_loss": 1.7850035429000854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17311613261699677,
      "rewards/margins": 0.021268434822559357,
      "rewards/rejected": -0.19438455998897552,
      "step": 365
    },
    {
      "epoch": 0.5768321513002365,
      "grad_norm": 0.2151210755109787,
      "learning_rate": 3.5871248897341246e-06,
      "log_odds_chosen": 0.4656566381454468,
      "log_odds_ratio": -0.493457555770874,
      "logits/chosen": 0.033955443650484085,
      "logits/rejected": -0.7494514584541321,
      "logps/chosen": -1.5631694793701172,
      "logps/rejected": -1.954725742340088,
      "loss": 1.7017,
      "nll_loss": 1.6523396968841553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15631696581840515,
      "rewards/margins": 0.03915562108159065,
      "rewards/rejected": -0.1954725980758667,
      "step": 366
    },
    {
      "epoch": 0.578408195429472,
      "grad_norm": 0.19448134303092957,
      "learning_rate": 3.5837739750550182e-06,
      "log_odds_chosen": 0.4710557460784912,
      "log_odds_ratio": -0.4958794116973877,
      "logits/chosen": -0.001226938795298338,
      "logits/rejected": -1.270828127861023,
      "logps/chosen": -1.6350599527359009,
      "logps/rejected": -2.034780740737915,
      "loss": 1.7501,
      "nll_loss": 1.7005350589752197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16350601613521576,
      "rewards/margins": 0.03997207060456276,
      "rewards/rejected": -0.20347808301448822,
      "step": 367
    },
    {
      "epoch": 0.5799842395587076,
      "grad_norm": 0.19283808767795563,
      "learning_rate": 3.5804110957199977e-06,
      "log_odds_chosen": 0.48457401990890503,
      "log_odds_ratio": -0.48585668206214905,
      "logits/chosen": 0.1169048473238945,
      "logits/rejected": -0.8879725337028503,
      "logps/chosen": -1.61647367477417,
      "logps/rejected": -2.023709297180176,
      "loss": 1.7294,
      "nll_loss": 1.6807751655578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16164736449718475,
      "rewards/margins": 0.04072355851531029,
      "rewards/rejected": -0.20237095654010773,
      "step": 368
    },
    {
      "epoch": 0.5815602836879432,
      "grad_norm": 0.19826547801494598,
      "learning_rate": 3.577036277134011e-06,
      "log_odds_chosen": 0.5813598036766052,
      "log_odds_ratio": -0.4480295777320862,
      "logits/chosen": 0.03684981167316437,
      "logits/rejected": -1.0105152130126953,
      "logps/chosen": -1.6038864850997925,
      "logps/rejected": -2.0928397178649902,
      "loss": 1.7299,
      "nll_loss": 1.6851170063018799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16038864850997925,
      "rewards/margins": 0.04889533668756485,
      "rewards/rejected": -0.2092839926481247,
      "step": 369
    },
    {
      "epoch": 0.5831363278171788,
      "grad_norm": 0.21686360239982605,
      "learning_rate": 3.5736495447922e-06,
      "log_odds_chosen": 0.3633495569229126,
      "log_odds_ratio": -0.5332463383674622,
      "logits/chosen": 0.037289127707481384,
      "logits/rejected": -1.074998140335083,
      "logps/chosen": -1.7311910390853882,
      "logps/rejected": -2.037515878677368,
      "loss": 1.847,
      "nll_loss": 1.7936969995498657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17311908304691315,
      "rewards/margins": 0.03063248284161091,
      "rewards/rejected": -0.2037515640258789,
      "step": 370
    },
    {
      "epoch": 0.5847123719464145,
      "grad_norm": 0.18624477088451385,
      "learning_rate": 3.5702509242797096e-06,
      "log_odds_chosen": 0.6746554970741272,
      "log_odds_ratio": -0.4293067753314972,
      "logits/chosen": 0.031168414279818535,
      "logits/rejected": -1.0391783714294434,
      "logps/chosen": -1.5503772497177124,
      "logps/rejected": -2.1166985034942627,
      "loss": 1.6782,
      "nll_loss": 1.6352722644805908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15503773093223572,
      "rewards/margins": 0.05663213133811951,
      "rewards/rejected": -0.21166983246803284,
      "step": 371
    },
    {
      "epoch": 0.5862884160756501,
      "grad_norm": 0.19968333840370178,
      "learning_rate": 3.566840441271495e-06,
      "log_odds_chosen": 0.630666196346283,
      "log_odds_ratio": -0.44081687927246094,
      "logits/chosen": 0.06830377876758575,
      "logits/rejected": -1.0593217611312866,
      "logps/chosen": -1.5753437280654907,
      "logps/rejected": -2.1008994579315186,
      "loss": 1.6789,
      "nll_loss": 1.634864091873169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1575343906879425,
      "rewards/margins": 0.05255556106567383,
      "rewards/rejected": -0.21008995175361633,
      "step": 372
    },
    {
      "epoch": 0.5878644602048857,
      "grad_norm": 0.20025794208049774,
      "learning_rate": 3.5634181215321265e-06,
      "log_odds_chosen": 0.663584291934967,
      "log_odds_ratio": -0.4243214428424835,
      "logits/chosen": 0.07222753763198853,
      "logits/rejected": -0.9715753793716431,
      "logps/chosen": -1.6212844848632812,
      "logps/rejected": -2.1870157718658447,
      "loss": 1.7468,
      "nll_loss": 1.7043309211730957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16212844848632812,
      "rewards/margins": 0.05657312646508217,
      "rewards/rejected": -0.2187015861272812,
      "step": 373
    },
    {
      "epoch": 0.5894405043341213,
      "grad_norm": 0.19907920062541962,
      "learning_rate": 3.5599839909155947e-06,
      "log_odds_chosen": 0.4552556872367859,
      "log_odds_ratio": -0.5016735196113586,
      "logits/chosen": 0.0466487854719162,
      "logits/rejected": -1.0045429468154907,
      "logps/chosen": -1.584489107131958,
      "logps/rejected": -1.9633792638778687,
      "loss": 1.7012,
      "nll_loss": 1.6509945392608643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15844891965389252,
      "rewards/margins": 0.03788900747895241,
      "rewards/rejected": -0.19633793830871582,
      "step": 374
    },
    {
      "epoch": 0.5910165484633569,
      "grad_norm": 0.19784437119960785,
      "learning_rate": 3.556538075365116e-06,
      "log_odds_chosen": 0.5365445017814636,
      "log_odds_ratio": -0.4729233384132385,
      "logits/chosen": 0.10310062021017075,
      "logits/rejected": -1.0228643417358398,
      "logps/chosen": -1.5829472541809082,
      "logps/rejected": -2.030022144317627,
      "loss": 1.6987,
      "nll_loss": 1.651432752609253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15829472243785858,
      "rewards/margins": 0.04470750689506531,
      "rewards/rejected": -0.2030022293329239,
      "step": 375
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 0.2056477963924408,
      "learning_rate": 3.5530804009129367e-06,
      "log_odds_chosen": 0.5238441228866577,
      "log_odds_ratio": -0.47168248891830444,
      "logits/chosen": 0.05509530007839203,
      "logits/rejected": -0.8633387684822083,
      "logps/chosen": -1.6203933954238892,
      "logps/rejected": -2.0622262954711914,
      "loss": 1.7216,
      "nll_loss": 1.6743903160095215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16203933954238892,
      "rewards/margins": 0.04418327659368515,
      "rewards/rejected": -0.20622262358665466,
      "step": 376
    },
    {
      "epoch": 0.5941686367218282,
      "grad_norm": 0.20296871662139893,
      "learning_rate": 3.5496109936801368e-06,
      "log_odds_chosen": 0.47061851620674133,
      "log_odds_ratio": -0.4923834204673767,
      "logits/chosen": -0.022969774901866913,
      "logits/rejected": -1.096977710723877,
      "logps/chosen": -1.6428031921386719,
      "logps/rejected": -2.04349946975708,
      "loss": 1.7551,
      "nll_loss": 1.7058473825454712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16428032517433167,
      "rewards/margins": 0.04006962105631828,
      "rewards/rejected": -0.20434993505477905,
      "step": 377
    },
    {
      "epoch": 0.5957446808510638,
      "grad_norm": 0.21032318472862244,
      "learning_rate": 3.546129879876429e-06,
      "log_odds_chosen": 0.3223990797996521,
      "log_odds_ratio": -0.5593502521514893,
      "logits/chosen": 0.11473214626312256,
      "logits/rejected": -0.7919275760650635,
      "logps/chosen": -1.7105305194854736,
      "logps/rejected": -1.983834981918335,
      "loss": 1.8197,
      "nll_loss": 1.763751745223999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17105305194854736,
      "rewards/margins": 0.027330443263053894,
      "rewards/rejected": -0.19838349521160126,
      "step": 378
    },
    {
      "epoch": 0.5973207249802994,
      "grad_norm": 0.20299871265888214,
      "learning_rate": 3.5426370857999662e-06,
      "log_odds_chosen": 0.32067781686782837,
      "log_odds_ratio": -0.5509010553359985,
      "logits/chosen": 0.04862087219953537,
      "logits/rejected": -0.9239585399627686,
      "logps/chosen": -1.6904906034469604,
      "logps/rejected": -1.9611345529556274,
      "loss": 1.8102,
      "nll_loss": 1.7551449537277222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16904906928539276,
      "rewards/margins": 0.027064397931098938,
      "rewards/rejected": -0.1961134672164917,
      "step": 379
    },
    {
      "epoch": 0.598896769109535,
      "grad_norm": 0.20678678154945374,
      "learning_rate": 3.53913263783714e-06,
      "log_odds_chosen": 0.37589550018310547,
      "log_odds_ratio": -0.5370999574661255,
      "logits/chosen": 0.007449129596352577,
      "logits/rejected": -1.0229346752166748,
      "logps/chosen": -1.633695363998413,
      "logps/rejected": -1.9534492492675781,
      "loss": 1.7419,
      "nll_loss": 1.6882388591766357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1633695513010025,
      "rewards/margins": 0.0319753997027874,
      "rewards/rejected": -0.195344939827919,
      "step": 380
    },
    {
      "epoch": 0.6004728132387707,
      "grad_norm": 0.19402244687080383,
      "learning_rate": 3.53561656246238e-06,
      "log_odds_chosen": 0.6378045678138733,
      "log_odds_ratio": -0.42818355560302734,
      "logits/chosen": -0.08074290305376053,
      "logits/rejected": -1.2157189846038818,
      "logps/chosen": -1.6589109897613525,
      "logps/rejected": -2.201770782470703,
      "loss": 1.7652,
      "nll_loss": 1.7223517894744873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1658911108970642,
      "rewards/margins": 0.05428597331047058,
      "rewards/rejected": -0.2201770842075348,
      "step": 381
    },
    {
      "epoch": 0.6020488573680063,
      "grad_norm": 0.19140584766864777,
      "learning_rate": 3.532088886237956e-06,
      "log_odds_chosen": 0.5052012801170349,
      "log_odds_ratio": -0.4754614233970642,
      "logits/chosen": -0.016120534390211105,
      "logits/rejected": -1.1966180801391602,
      "logps/chosen": -1.6766316890716553,
      "logps/rejected": -2.1046011447906494,
      "loss": 1.7709,
      "nll_loss": 1.7233370542526245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16766318678855896,
      "rewards/margins": 0.042796939611434937,
      "rewards/rejected": -0.2104601114988327,
      "step": 382
    },
    {
      "epoch": 0.6036249014972419,
      "grad_norm": 0.18412470817565918,
      "learning_rate": 3.528549635813778e-06,
      "log_odds_chosen": 0.5283522605895996,
      "log_odds_ratio": -0.4716714024543762,
      "logits/chosen": -0.04352571442723274,
      "logits/rejected": -1.0171538591384888,
      "logps/chosen": -1.5115892887115479,
      "logps/rejected": -1.9484082460403442,
      "loss": 1.6296,
      "nll_loss": 1.5823882818222046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15115892887115479,
      "rewards/margins": 0.043681904673576355,
      "rewards/rejected": -0.19484084844589233,
      "step": 383
    },
    {
      "epoch": 0.6052009456264775,
      "grad_norm": 0.2047756463289261,
      "learning_rate": 3.524998837927192e-06,
      "log_odds_chosen": 0.6116353869438171,
      "log_odds_ratio": -0.43900421261787415,
      "logits/chosen": 0.008262881077826023,
      "logits/rejected": -0.9327568411827087,
      "logps/chosen": -1.645035982131958,
      "logps/rejected": -2.1674892902374268,
      "loss": 1.7375,
      "nll_loss": 1.693565845489502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16450358927249908,
      "rewards/margins": 0.05224531143903732,
      "rewards/rejected": -0.2167489230632782,
      "step": 384
    },
    {
      "epoch": 0.6067769897557131,
      "grad_norm": 0.18797335028648376,
      "learning_rate": 3.5214365194027797e-06,
      "log_odds_chosen": 0.5875496864318848,
      "log_odds_ratio": -0.4458829462528229,
      "logits/chosen": -0.019994519650936127,
      "logits/rejected": -1.0720854997634888,
      "logps/chosen": -1.5490386486053467,
      "logps/rejected": -2.0399489402770996,
      "loss": 1.656,
      "nll_loss": 1.6113654375076294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15490387380123138,
      "rewards/margins": 0.04909101501107216,
      "rewards/rejected": -0.20399489998817444,
      "step": 385
    },
    {
      "epoch": 0.6083530338849488,
      "grad_norm": 0.21445293724536896,
      "learning_rate": 3.517862707152157e-06,
      "log_odds_chosen": 0.45695003867149353,
      "log_odds_ratio": -0.5020439624786377,
      "logits/chosen": 0.08733657747507095,
      "logits/rejected": -0.7828851938247681,
      "logps/chosen": -1.7037779092788696,
      "logps/rejected": -2.0940959453582764,
      "loss": 1.7985,
      "nll_loss": 1.7482857704162598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17037779092788696,
      "rewards/margins": 0.03903179615736008,
      "rewards/rejected": -0.20940957963466644,
      "step": 386
    },
    {
      "epoch": 0.6099290780141844,
      "grad_norm": 0.2248964160680771,
      "learning_rate": 3.5142774281737674e-06,
      "log_odds_chosen": 0.6219518184661865,
      "log_odds_ratio": -0.43292707204818726,
      "logits/chosen": -0.0042352513410151005,
      "logits/rejected": -0.9333048462867737,
      "logps/chosen": -1.695968747138977,
      "logps/rejected": -2.2276930809020996,
      "loss": 1.7847,
      "nll_loss": 1.741443395614624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1695968508720398,
      "rewards/margins": 0.05317245051264763,
      "rewards/rejected": -0.22276932001113892,
      "step": 387
    },
    {
      "epoch": 0.61150512214342,
      "grad_norm": 0.18177717924118042,
      "learning_rate": 3.5106807095526817e-06,
      "log_odds_chosen": 0.6358584761619568,
      "log_odds_ratio": -0.43654751777648926,
      "logits/chosen": -0.039142899215221405,
      "logits/rejected": -1.1646403074264526,
      "logps/chosen": -1.6416778564453125,
      "logps/rejected": -2.176908493041992,
      "loss": 1.7464,
      "nll_loss": 1.7027642726898193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16416779160499573,
      "rewards/margins": 0.05352305620908737,
      "rewards/rejected": -0.2176908403635025,
      "step": 388
    },
    {
      "epoch": 0.6130811662726556,
      "grad_norm": 0.19595922529697418,
      "learning_rate": 3.5070725784603905e-06,
      "log_odds_chosen": 0.49472156167030334,
      "log_odds_ratio": -0.48498183488845825,
      "logits/chosen": -0.10919392853975296,
      "logits/rejected": -0.9267551898956299,
      "logps/chosen": -1.5107898712158203,
      "logps/rejected": -1.9186828136444092,
      "loss": 1.6134,
      "nll_loss": 1.5649324655532837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1510789841413498,
      "rewards/margins": 0.04078929126262665,
      "rewards/rejected": -0.19186827540397644,
      "step": 389
    },
    {
      "epoch": 0.6146572104018913,
      "grad_norm": 0.21639399230480194,
      "learning_rate": 3.503453062154602e-06,
      "log_odds_chosen": 0.42183130979537964,
      "log_odds_ratio": -0.5190197825431824,
      "logits/chosen": -0.023847615346312523,
      "logits/rejected": -0.8754997253417969,
      "logps/chosen": -1.6942306756973267,
      "logps/rejected": -2.06078839302063,
      "loss": 1.8063,
      "nll_loss": 1.754384160041809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16942308843135834,
      "rewards/margins": 0.036655765026807785,
      "rewards/rejected": -0.20607884228229523,
      "step": 390
    },
    {
      "epoch": 0.6162332545311269,
      "grad_norm": 0.1975736767053604,
      "learning_rate": 3.499822187979032e-06,
      "log_odds_chosen": 0.40866145491600037,
      "log_odds_ratio": -0.5196468830108643,
      "logits/chosen": 0.0417475588619709,
      "logits/rejected": -1.139929175376892,
      "logps/chosen": -1.6336414813995361,
      "logps/rejected": -1.9767978191375732,
      "loss": 1.7518,
      "nll_loss": 1.6998686790466309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16336414217948914,
      "rewards/margins": 0.03431565314531326,
      "rewards/rejected": -0.1976797729730606,
      "step": 391
    },
    {
      "epoch": 0.6178092986603625,
      "grad_norm": 0.19061535596847534,
      "learning_rate": 3.496179983363202e-06,
      "log_odds_chosen": 0.37159737944602966,
      "log_odds_ratio": -0.5294693112373352,
      "logits/chosen": 0.022184930741786957,
      "logits/rejected": -0.9958785176277161,
      "logps/chosen": -1.6446539163589478,
      "logps/rejected": -1.9568779468536377,
      "loss": 1.7428,
      "nll_loss": 1.6898627281188965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16446539759635925,
      "rewards/margins": 0.031222393736243248,
      "rewards/rejected": -0.19568778574466705,
      "step": 392
    },
    {
      "epoch": 0.6193853427895981,
      "grad_norm": 0.2155338078737259,
      "learning_rate": 3.4925264758222268e-06,
      "log_odds_chosen": 0.5748679637908936,
      "log_odds_ratio": -0.45070305466651917,
      "logits/chosen": -0.02329150401055813,
      "logits/rejected": -0.8651071786880493,
      "logps/chosen": -1.5768085718154907,
      "logps/rejected": -2.059037208557129,
      "loss": 1.6901,
      "nll_loss": 1.645035982131958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15768086910247803,
      "rewards/margins": 0.0482228547334671,
      "rewards/rejected": -0.20590370893478394,
      "step": 393
    },
    {
      "epoch": 0.6209613869188337,
      "grad_norm": 0.19670893251895905,
      "learning_rate": 3.488861692956611e-06,
      "log_odds_chosen": 0.5193027257919312,
      "log_odds_ratio": -0.4729624092578888,
      "logits/chosen": -0.062314629554748535,
      "logits/rejected": -0.9838546514511108,
      "logps/chosen": -1.6177668571472168,
      "logps/rejected": -2.0553812980651855,
      "loss": 1.7207,
      "nll_loss": 1.6734217405319214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16177670657634735,
      "rewards/margins": 0.04376142844557762,
      "rewards/rejected": -0.20553813874721527,
      "step": 394
    },
    {
      "epoch": 0.6225374310480694,
      "grad_norm": 0.20605877041816711,
      "learning_rate": 3.4851856624520394e-06,
      "log_odds_chosen": 0.5313916802406311,
      "log_odds_ratio": -0.46628880500793457,
      "logits/chosen": 0.00429939292371273,
      "logits/rejected": -1.0193074941635132,
      "logps/chosen": -1.6535357236862183,
      "logps/rejected": -2.1009347438812256,
      "loss": 1.7514,
      "nll_loss": 1.7047302722930908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16535356640815735,
      "rewards/margins": 0.044739894568920135,
      "rewards/rejected": -0.21009346842765808,
      "step": 395
    },
    {
      "epoch": 0.624113475177305,
      "grad_norm": 0.2116907835006714,
      "learning_rate": 3.4814984120791664e-06,
      "log_odds_chosen": 0.4503478407859802,
      "log_odds_ratio": -0.4949477016925812,
      "logits/chosen": 0.009901851415634155,
      "logits/rejected": -0.8826244473457336,
      "logps/chosen": -1.6289476156234741,
      "logps/rejected": -2.004061698913574,
      "loss": 1.7392,
      "nll_loss": 1.6896657943725586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16289478540420532,
      "rewards/margins": 0.03751137852668762,
      "rewards/rejected": -0.20040614902973175,
      "step": 396
    },
    {
      "epoch": 0.6256895193065406,
      "grad_norm": 0.46380433440208435,
      "learning_rate": 3.477799969693407e-06,
      "log_odds_chosen": 0.4436909854412079,
      "log_odds_ratio": -0.5040198564529419,
      "logits/chosen": 0.04017059504985809,
      "logits/rejected": -0.9966317415237427,
      "logps/chosen": -1.6149952411651611,
      "logps/rejected": -1.980851173400879,
      "loss": 1.7189,
      "nll_loss": 1.6685235500335693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1614995300769806,
      "rewards/margins": 0.03658558428287506,
      "rewards/rejected": -0.19808511435985565,
      "step": 397
    },
    {
      "epoch": 0.6272655634357762,
      "grad_norm": 0.21046514809131622,
      "learning_rate": 3.474090363234728e-06,
      "log_odds_chosen": 0.7306087017059326,
      "log_odds_ratio": -0.4084674119949341,
      "logits/chosen": -0.1205843836069107,
      "logits/rejected": -1.0663551092147827,
      "logps/chosen": -1.558687448501587,
      "logps/rejected": -2.1748738288879395,
      "loss": 1.6557,
      "nll_loss": 1.6148791313171387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1558687388896942,
      "rewards/margins": 0.061618633568286896,
      "rewards/rejected": -0.2174873799085617,
      "step": 398
    },
    {
      "epoch": 0.6288416075650118,
      "grad_norm": 0.2060404270887375,
      "learning_rate": 3.4703696207274325e-06,
      "log_odds_chosen": 0.46504050493240356,
      "log_odds_ratio": -0.4943721890449524,
      "logits/chosen": -0.008130216039717197,
      "logits/rejected": -0.9074642658233643,
      "logps/chosen": -1.6367112398147583,
      "logps/rejected": -2.0245320796966553,
      "loss": 1.7255,
      "nll_loss": 1.6760835647583008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1636711210012436,
      "rewards/margins": 0.038782089948654175,
      "rewards/rejected": -0.20245321094989777,
      "step": 399
    },
    {
      "epoch": 0.6304176516942475,
      "grad_norm": 0.1868315488100052,
      "learning_rate": 3.4666377702799545e-06,
      "log_odds_chosen": 0.49212151765823364,
      "log_odds_ratio": -0.482626736164093,
      "logits/chosen": -0.12099798768758774,
      "logits/rejected": -1.2031362056732178,
      "logps/chosen": -1.5395071506500244,
      "logps/rejected": -1.9444780349731445,
      "loss": 1.6547,
      "nll_loss": 1.6063977479934692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15395070612430573,
      "rewards/margins": 0.040497105568647385,
      "rewards/rejected": -0.1944478154182434,
      "step": 400
    },
    {
      "epoch": 0.6319936958234831,
      "grad_norm": 0.20555970072746277,
      "learning_rate": 3.4628948400846417e-06,
      "log_odds_chosen": 0.6244911551475525,
      "log_odds_ratio": -0.44323962926864624,
      "logits/chosen": -0.041763000190258026,
      "logits/rejected": -1.1066770553588867,
      "logps/chosen": -1.6332682371139526,
      "logps/rejected": -2.165579319000244,
      "loss": 1.7357,
      "nll_loss": 1.6914199590682983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16332682967185974,
      "rewards/margins": 0.05323110893368721,
      "rewards/rejected": -0.21655793488025665,
      "step": 401
    },
    {
      "epoch": 0.6335697399527187,
      "grad_norm": 0.21198388934135437,
      "learning_rate": 3.4591408584175426e-06,
      "log_odds_chosen": 0.5584430694580078,
      "log_odds_ratio": -0.4683310091495514,
      "logits/chosen": -0.05661184713244438,
      "logits/rejected": -1.0141117572784424,
      "logps/chosen": -1.6664652824401855,
      "logps/rejected": -2.146893262863159,
      "loss": 1.7703,
      "nll_loss": 1.7234219312667847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16664651036262512,
      "rewards/margins": 0.04804282262921333,
      "rewards/rejected": -0.21468935906887054,
      "step": 402
    },
    {
      "epoch": 0.6351457840819543,
      "grad_norm": 0.20266425609588623,
      "learning_rate": 3.4553758536381974e-06,
      "log_odds_chosen": 0.4527875483036041,
      "log_odds_ratio": -0.5067857503890991,
      "logits/chosen": -0.035650648176670074,
      "logits/rejected": -1.0273325443267822,
      "logps/chosen": -1.6237739324569702,
      "logps/rejected": -2.0050816535949707,
      "loss": 1.7259,
      "nll_loss": 1.675266146659851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16237738728523254,
      "rewards/margins": 0.03813077509403229,
      "rewards/rejected": -0.20050817728042603,
      "step": 403
    },
    {
      "epoch": 0.6367218282111899,
      "grad_norm": 0.21295951306819916,
      "learning_rate": 3.451599854189418e-06,
      "log_odds_chosen": 0.4978407025337219,
      "log_odds_ratio": -0.4797380268573761,
      "logits/chosen": -0.005185263231396675,
      "logits/rejected": -0.7095089554786682,
      "logps/chosen": -1.624720573425293,
      "logps/rejected": -2.042205572128296,
      "loss": 1.7394,
      "nll_loss": 1.6913968324661255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16247206926345825,
      "rewards/margins": 0.04174850881099701,
      "rewards/rejected": -0.20422056317329407,
      "step": 404
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 0.20290222764015198,
      "learning_rate": 3.4478128885970765e-06,
      "log_odds_chosen": 0.6090503334999084,
      "log_odds_ratio": -0.4394116997718811,
      "logits/chosen": -0.048805318772792816,
      "logits/rejected": -0.9924325942993164,
      "logps/chosen": -1.6847188472747803,
      "logps/rejected": -2.205502510070801,
      "loss": 1.7831,
      "nll_loss": 1.7391594648361206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16847188770771027,
      "rewards/margins": 0.05207835137844086,
      "rewards/rejected": -0.22055023908615112,
      "step": 405
    },
    {
      "epoch": 0.6398739164696612,
      "grad_norm": 0.19735999405384064,
      "learning_rate": 3.44401498546989e-06,
      "log_odds_chosen": 0.40471282601356506,
      "log_odds_ratio": -0.5204766392707825,
      "logits/chosen": -0.00239025941118598,
      "logits/rejected": -1.1112685203552246,
      "logps/chosen": -1.63077974319458,
      "logps/rejected": -1.9698301553726196,
      "loss": 1.7163,
      "nll_loss": 1.664223551750183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16307799518108368,
      "rewards/margins": 0.033905029296875,
      "rewards/rejected": -0.19698300957679749,
      "step": 406
    },
    {
      "epoch": 0.6414499605988968,
      "grad_norm": 0.20032718777656555,
      "learning_rate": 3.4402061734992005e-06,
      "log_odds_chosen": 0.4921063780784607,
      "log_odds_ratio": -0.48325401544570923,
      "logits/chosen": -0.03169507533311844,
      "logits/rejected": -1.0495617389678955,
      "logps/chosen": -1.5727291107177734,
      "logps/rejected": -1.982052206993103,
      "loss": 1.6824,
      "nll_loss": 1.634063482284546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15727292001247406,
      "rewards/margins": 0.040932293981313705,
      "rewards/rejected": -0.19820521771907806,
      "step": 407
    },
    {
      "epoch": 0.6430260047281324,
      "grad_norm": 0.21005932986736298,
      "learning_rate": 3.4363864814587656e-06,
      "log_odds_chosen": 0.45053696632385254,
      "log_odds_ratio": -0.49728235602378845,
      "logits/chosen": -0.14125539362430573,
      "logits/rejected": -0.6607908010482788,
      "logps/chosen": -1.55253005027771,
      "logps/rejected": -1.9263464212417603,
      "loss": 1.6646,
      "nll_loss": 1.6149158477783203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15525302290916443,
      "rewards/margins": 0.037381626665592194,
      "rewards/rejected": -0.19263465702533722,
      "step": 408
    },
    {
      "epoch": 0.644602048857368,
      "grad_norm": 0.21455970406532288,
      "learning_rate": 3.4325559382045343e-06,
      "log_odds_chosen": 0.33162471652030945,
      "log_odds_ratio": -0.5455266833305359,
      "logits/chosen": -0.10501392185688019,
      "logits/rejected": -1.1451892852783203,
      "logps/chosen": -1.567715048789978,
      "logps/rejected": -1.837162971496582,
      "loss": 1.6723,
      "nll_loss": 1.6177119016647339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1567714810371399,
      "rewards/margins": 0.026944806799292564,
      "rewards/rejected": -0.1837162971496582,
      "step": 409
    },
    {
      "epoch": 0.6461780929866037,
      "grad_norm": 0.20775310695171356,
      "learning_rate": 3.4287145726744295e-06,
      "log_odds_chosen": 0.5500257015228271,
      "log_odds_ratio": -0.46216368675231934,
      "logits/chosen": -0.13800697028636932,
      "logits/rejected": -0.9645453691482544,
      "logps/chosen": -1.550150990486145,
      "logps/rejected": -2.005794048309326,
      "loss": 1.6661,
      "nll_loss": 1.6199290752410889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15501509606838226,
      "rewards/margins": 0.04556430131196976,
      "rewards/rejected": -0.20057940483093262,
      "step": 410
    },
    {
      "epoch": 0.6477541371158393,
      "grad_norm": 0.2134082019329071,
      "learning_rate": 3.4248624138881335e-06,
      "log_odds_chosen": 0.40455663204193115,
      "log_odds_ratio": -0.5388070344924927,
      "logits/chosen": -0.026050057262182236,
      "logits/rejected": -0.811333417892456,
      "logps/chosen": -1.6882728338241577,
      "logps/rejected": -2.0449092388153076,
      "loss": 1.7892,
      "nll_loss": 1.7353262901306152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16882728040218353,
      "rewards/margins": 0.03566364943981171,
      "rewards/rejected": -0.20449092984199524,
      "step": 411
    },
    {
      "epoch": 0.6493301812450749,
      "grad_norm": 0.2133842557668686,
      "learning_rate": 3.4209994909468672e-06,
      "log_odds_chosen": 0.6076403260231018,
      "log_odds_ratio": -0.4480054974555969,
      "logits/chosen": -0.15131589770317078,
      "logits/rejected": -0.7187516093254089,
      "logps/chosen": -1.5922455787658691,
      "logps/rejected": -2.1052651405334473,
      "loss": 1.6941,
      "nll_loss": 1.649259090423584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15922458469867706,
      "rewards/margins": 0.05130196362733841,
      "rewards/rejected": -0.21052654087543488,
      "step": 412
    },
    {
      "epoch": 0.6509062253743105,
      "grad_norm": 0.20478829741477966,
      "learning_rate": 3.4171258330331667e-06,
      "log_odds_chosen": 0.39909109473228455,
      "log_odds_ratio": -0.5157694816589355,
      "logits/chosen": 0.006179895251989365,
      "logits/rejected": -0.8033927083015442,
      "logps/chosen": -1.6710724830627441,
      "logps/rejected": -2.0045435428619385,
      "loss": 1.7696,
      "nll_loss": 1.718004584312439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1671072542667389,
      "rewards/margins": 0.03334709256887436,
      "rewards/rejected": -0.20045433938503265,
      "step": 413
    },
    {
      "epoch": 0.6524822695035462,
      "grad_norm": 0.20214222371578217,
      "learning_rate": 3.4132414694106684e-06,
      "log_odds_chosen": 0.6251203417778015,
      "log_odds_ratio": -0.43651172518730164,
      "logits/chosen": -0.0376095250248909,
      "logits/rejected": -1.10747492313385,
      "logps/chosen": -1.5772483348846436,
      "logps/rejected": -2.1024982929229736,
      "loss": 1.6741,
      "nll_loss": 1.6304349899291992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15772482752799988,
      "rewards/margins": 0.05252499133348465,
      "rewards/rejected": -0.21024984121322632,
      "step": 414
    },
    {
      "epoch": 0.6540583136327817,
      "grad_norm": 0.2120778113603592,
      "learning_rate": 3.409346429423884e-06,
      "log_odds_chosen": 0.39810532331466675,
      "log_odds_ratio": -0.5165068507194519,
      "logits/chosen": 0.11685831099748611,
      "logits/rejected": -0.9671615362167358,
      "logps/chosen": -1.6512268781661987,
      "logps/rejected": -1.9838975667953491,
      "loss": 1.7451,
      "nll_loss": 1.693439245223999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16512268781661987,
      "rewards/margins": 0.0332670658826828,
      "rewards/rejected": -0.19838975369930267,
      "step": 415
    },
    {
      "epoch": 0.6556343577620173,
      "grad_norm": 0.2049696147441864,
      "learning_rate": 3.40544074249798e-06,
      "log_odds_chosen": 0.6329183578491211,
      "log_odds_ratio": -0.43744218349456787,
      "logits/chosen": -0.06616753339767456,
      "logits/rejected": -1.1013216972351074,
      "logps/chosen": -1.6389892101287842,
      "logps/rejected": -2.1797854900360107,
      "loss": 1.7372,
      "nll_loss": 1.6934669017791748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16389892995357513,
      "rewards/margins": 0.05407961830496788,
      "rewards/rejected": -0.2179785519838333,
      "step": 416
    },
    {
      "epoch": 0.6572104018912529,
      "grad_norm": 0.22548918426036835,
      "learning_rate": 3.401524438138556e-06,
      "log_odds_chosen": 0.47032052278518677,
      "log_odds_ratio": -0.49687254428863525,
      "logits/chosen": 6.176391616463661e-05,
      "logits/rejected": -0.8082195520401001,
      "logps/chosen": -1.652639627456665,
      "logps/rejected": -2.0523738861083984,
      "loss": 1.7294,
      "nll_loss": 1.6797451972961426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16526396572589874,
      "rewards/margins": 0.039973437786102295,
      "rewards/rejected": -0.20523738861083984,
      "step": 417
    },
    {
      "epoch": 0.6587864460204885,
      "grad_norm": 0.20852544903755188,
      "learning_rate": 3.39759754593142e-06,
      "log_odds_chosen": 0.49835333228111267,
      "log_odds_ratio": -0.48722827434539795,
      "logits/chosen": -0.06864430010318756,
      "logits/rejected": -0.9766976833343506,
      "logps/chosen": -1.7005680799484253,
      "logps/rejected": -2.128114700317383,
      "loss": 1.7887,
      "nll_loss": 1.7399288415908813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1700568050146103,
      "rewards/margins": 0.04275466501712799,
      "rewards/rejected": -0.21281147003173828,
      "step": 418
    },
    {
      "epoch": 0.6603624901497241,
      "grad_norm": 0.21272076666355133,
      "learning_rate": 3.3936600955423683e-06,
      "log_odds_chosen": 0.6597129702568054,
      "log_odds_ratio": -0.42969611287117004,
      "logits/chosen": 0.03045894019305706,
      "logits/rejected": -0.764129638671875,
      "logps/chosen": -1.6032241582870483,
      "logps/rejected": -2.1603918075561523,
      "loss": 1.7045,
      "nll_loss": 1.6615636348724365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1603224277496338,
      "rewards/margins": 0.05571678280830383,
      "rewards/rejected": -0.21603919565677643,
      "step": 419
    },
    {
      "epoch": 0.6619385342789598,
      "grad_norm": 0.20755982398986816,
      "learning_rate": 3.3897121167169573e-06,
      "log_odds_chosen": 0.34852004051208496,
      "log_odds_ratio": -0.5454325079917908,
      "logits/chosen": -0.11937910318374634,
      "logits/rejected": -0.982914388179779,
      "logps/chosen": -1.5703413486480713,
      "logps/rejected": -1.858556866645813,
      "loss": 1.6827,
      "nll_loss": 1.62813138961792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15703415870666504,
      "rewards/margins": 0.028821531683206558,
      "rewards/rejected": -0.1858556866645813,
      "step": 420
    },
    {
      "epoch": 0.6635145784081954,
      "grad_norm": 0.20710468292236328,
      "learning_rate": 3.38575363928028e-06,
      "log_odds_chosen": 0.5697583556175232,
      "log_odds_ratio": -0.45452526211738586,
      "logits/chosen": -0.02085725963115692,
      "logits/rejected": -0.8392971754074097,
      "logps/chosen": -1.5740097761154175,
      "logps/rejected": -2.0498874187469482,
      "loss": 1.6588,
      "nll_loss": 1.6133930683135986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1574009507894516,
      "rewards/margins": 0.04758777469396591,
      "rewards/rejected": -0.2049887329339981,
      "step": 421
    },
    {
      "epoch": 0.665090622537431,
      "grad_norm": 0.2102571576833725,
      "learning_rate": 3.3817846931367452e-06,
      "log_odds_chosen": 0.41366758942604065,
      "log_odds_ratio": -0.5172683000564575,
      "logits/chosen": -0.10860089957714081,
      "logits/rejected": -0.8454552888870239,
      "logps/chosen": -1.565171718597412,
      "logps/rejected": -1.9058090448379517,
      "loss": 1.6679,
      "nll_loss": 1.6161460876464844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1565171778202057,
      "rewards/margins": 0.03406372666358948,
      "rewards/rejected": -0.19058090448379517,
      "step": 422
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 0.191580131649971,
      "learning_rate": 3.377805308269844e-06,
      "log_odds_chosen": 0.6439597606658936,
      "log_odds_ratio": -0.43700361251831055,
      "logits/chosen": -0.01071137934923172,
      "logits/rejected": -1.181455373764038,
      "logps/chosen": -1.608168601989746,
      "logps/rejected": -2.153388738632202,
      "loss": 1.6986,
      "nll_loss": 1.654897928237915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16081684827804565,
      "rewards/margins": 0.054522015154361725,
      "rewards/rejected": -0.21533887088298798,
      "step": 423
    },
    {
      "epoch": 0.6682427107959023,
      "grad_norm": 0.19327178597450256,
      "learning_rate": 3.3738155147419275e-06,
      "log_odds_chosen": 0.575452446937561,
      "log_odds_ratio": -0.45816850662231445,
      "logits/chosen": -0.13200272619724274,
      "logits/rejected": -0.9068926572799683,
      "logps/chosen": -1.6064414978027344,
      "logps/rejected": -2.092689275741577,
      "loss": 1.6817,
      "nll_loss": 1.6358880996704102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16064415872097015,
      "rewards/margins": 0.048624761402606964,
      "rewards/rejected": -0.20926889777183533,
      "step": 424
    },
    {
      "epoch": 0.6698187549251379,
      "grad_norm": 0.20792369544506073,
      "learning_rate": 3.3698153426939824e-06,
      "log_odds_chosen": 0.674706220626831,
      "log_odds_ratio": -0.4227129817008972,
      "logits/chosen": -0.11872132122516632,
      "logits/rejected": -0.758519172668457,
      "logps/chosen": -1.5436768531799316,
      "logps/rejected": -2.105792284011841,
      "loss": 1.6326,
      "nll_loss": 1.5903273820877075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15436768531799316,
      "rewards/margins": 0.05621154233813286,
      "rewards/rejected": -0.21057923138141632,
      "step": 425
    },
    {
      "epoch": 0.6713947990543735,
      "grad_norm": 0.2181132733821869,
      "learning_rate": 3.3658048223453954e-06,
      "log_odds_chosen": 0.6492812633514404,
      "log_odds_ratio": -0.43314385414123535,
      "logits/chosen": -0.07235290110111237,
      "logits/rejected": -0.9915630221366882,
      "logps/chosen": -1.6831142902374268,
      "logps/rejected": -2.2399418354034424,
      "loss": 1.7433,
      "nll_loss": 1.6999669075012207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16831141710281372,
      "rewards/margins": 0.05568274110555649,
      "rewards/rejected": -0.223994180560112,
      "step": 426
    },
    {
      "epoch": 0.6729708431836091,
      "grad_norm": 0.20445337891578674,
      "learning_rate": 3.3617839839937337e-06,
      "log_odds_chosen": 0.7020919919013977,
      "log_odds_ratio": -0.41846412420272827,
      "logits/chosen": -0.014502273872494698,
      "logits/rejected": -1.0373268127441406,
      "logps/chosen": -1.6425418853759766,
      "logps/rejected": -2.2469215393066406,
      "loss": 1.7212,
      "nll_loss": 1.6793993711471558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16425418853759766,
      "rewards/margins": 0.06043797731399536,
      "rewards/rejected": -0.22469216585159302,
      "step": 427
    },
    {
      "epoch": 0.6745468873128447,
      "grad_norm": 0.2129041850566864,
      "learning_rate": 3.3577528580145107e-06,
      "log_odds_chosen": 0.2998935580253601,
      "log_odds_ratio": -0.5577186346054077,
      "logits/chosen": -0.047993727028369904,
      "logits/rejected": -0.9543691277503967,
      "logps/chosen": -1.6607805490493774,
      "logps/rejected": -1.912225604057312,
      "loss": 1.7552,
      "nll_loss": 1.699418306350708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16607806086540222,
      "rewards/margins": 0.025144489482045174,
      "rewards/rejected": -0.19122257828712463,
      "step": 428
    },
    {
      "epoch": 0.6761229314420804,
      "grad_norm": 0.20701919496059418,
      "learning_rate": 3.353711474860956e-06,
      "log_odds_chosen": 0.513329029083252,
      "log_odds_ratio": -0.4874694049358368,
      "logits/chosen": -0.08273675292730331,
      "logits/rejected": -0.9507322907447815,
      "logps/chosen": -1.6140192747116089,
      "logps/rejected": -2.052968978881836,
      "loss": 1.6898,
      "nll_loss": 1.6410080194473267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16140194237232208,
      "rewards/margins": 0.043894968926906586,
      "rewards/rejected": -0.20529691874980927,
      "step": 429
    },
    {
      "epoch": 0.677698975571316,
      "grad_norm": 0.19609764218330383,
      "learning_rate": 3.3496598650637916e-06,
      "log_odds_chosen": 0.526725172996521,
      "log_odds_ratio": -0.47201910614967346,
      "logits/chosen": -0.09990064054727554,
      "logits/rejected": -0.8700954914093018,
      "logps/chosen": -1.5076314210891724,
      "logps/rejected": -1.9389755725860596,
      "loss": 1.6043,
      "nll_loss": 1.55709707736969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.150763139128685,
      "rewards/margins": 0.0431344136595726,
      "rewards/rejected": -0.1938975602388382,
      "step": 430
    },
    {
      "epoch": 0.6792750197005516,
      "grad_norm": 0.1991858035326004,
      "learning_rate": 3.3455980592309923e-06,
      "log_odds_chosen": 0.5936864614486694,
      "log_odds_ratio": -0.4512265920639038,
      "logits/chosen": -0.1363341212272644,
      "logits/rejected": -0.8848008513450623,
      "logps/chosen": -1.489920735359192,
      "logps/rejected": -1.9833248853683472,
      "loss": 1.5942,
      "nll_loss": 1.5491033792495728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14899206161499023,
      "rewards/margins": 0.04934043064713478,
      "rewards/rejected": -0.1983325034379959,
      "step": 431
    },
    {
      "epoch": 0.6808510638297872,
      "grad_norm": 0.20983241498470306,
      "learning_rate": 3.341526088047562e-06,
      "log_odds_chosen": 0.5074108839035034,
      "log_odds_ratio": -0.48232734203338623,
      "logits/chosen": -0.06561825424432755,
      "logits/rejected": -0.9465508460998535,
      "logps/chosen": -1.6827867031097412,
      "logps/rejected": -2.1165406703948975,
      "loss": 1.7561,
      "nll_loss": 1.7078619003295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16827866435050964,
      "rewards/margins": 0.043375398963689804,
      "rewards/rejected": -0.21165406703948975,
      "step": 432
    },
    {
      "epoch": 0.6824271079590228,
      "grad_norm": 0.22068631649017334,
      "learning_rate": 3.3374439822752972e-06,
      "log_odds_chosen": 0.3745832145214081,
      "log_odds_ratio": -0.5286702513694763,
      "logits/chosen": -0.008446307852864265,
      "logits/rejected": -0.816412627696991,
      "logps/chosen": -1.6505870819091797,
      "logps/rejected": -1.96774423122406,
      "loss": 1.7536,
      "nll_loss": 1.7007298469543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1650587022304535,
      "rewards/margins": 0.03171573579311371,
      "rewards/rejected": -0.196774423122406,
      "step": 433
    },
    {
      "epoch": 0.6840031520882585,
      "grad_norm": 0.21903684735298157,
      "learning_rate": 3.333351772752559e-06,
      "log_odds_chosen": 0.5724027752876282,
      "log_odds_ratio": -0.4620168209075928,
      "logits/chosen": -0.1362486630678177,
      "logits/rejected": -0.9352101683616638,
      "logps/chosen": -1.7139655351638794,
      "logps/rejected": -2.2006916999816895,
      "loss": 1.7887,
      "nll_loss": 1.7425075769424438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17139655351638794,
      "rewards/margins": 0.04867260903120041,
      "rewards/rejected": -0.22006915509700775,
      "step": 434
    },
    {
      "epoch": 0.6855791962174941,
      "grad_norm": 0.2128954529762268,
      "learning_rate": 3.3292494903940338e-06,
      "log_odds_chosen": 0.6801972985267639,
      "log_odds_ratio": -0.4192766547203064,
      "logits/chosen": -0.032803162932395935,
      "logits/rejected": -0.91535884141922,
      "logps/chosen": -1.6139404773712158,
      "logps/rejected": -2.1860525608062744,
      "loss": 1.7009,
      "nll_loss": 1.6589348316192627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16139404475688934,
      "rewards/margins": 0.057211197912693024,
      "rewards/rejected": -0.21860525012016296,
      "step": 435
    },
    {
      "epoch": 0.6871552403467297,
      "grad_norm": 0.2002972662448883,
      "learning_rate": 3.3251371661905063e-06,
      "log_odds_chosen": 0.5376681685447693,
      "log_odds_ratio": -0.4723852574825287,
      "logits/chosen": -0.08951784670352936,
      "logits/rejected": -0.8478531837463379,
      "logps/chosen": -1.4331471920013428,
      "logps/rejected": -1.8777246475219727,
      "loss": 1.5344,
      "nll_loss": 1.487158179283142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14331473410129547,
      "rewards/margins": 0.04445774853229523,
      "rewards/rejected": -0.1877724677324295,
      "step": 436
    },
    {
      "epoch": 0.6887312844759653,
      "grad_norm": 0.20821936428546906,
      "learning_rate": 3.321014831208622e-06,
      "log_odds_chosen": 0.5342722535133362,
      "log_odds_ratio": -0.46678629517555237,
      "logits/chosen": -0.08730736374855042,
      "logits/rejected": -0.8291983604431152,
      "logps/chosen": -1.500610589981079,
      "logps/rejected": -1.9357343912124634,
      "loss": 1.6101,
      "nll_loss": 1.5633900165557861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15006107091903687,
      "rewards/margins": 0.04351237416267395,
      "rewards/rejected": -0.19357344508171082,
      "step": 437
    },
    {
      "epoch": 0.6903073286052009,
      "grad_norm": 0.20071037113666534,
      "learning_rate": 3.316882516590652e-06,
      "log_odds_chosen": 0.5401603579521179,
      "log_odds_ratio": -0.4615623354911804,
      "logits/chosen": -0.07496978342533112,
      "logits/rejected": -0.99763023853302,
      "logps/chosen": -1.5472073554992676,
      "logps/rejected": -1.99533212184906,
      "loss": 1.6436,
      "nll_loss": 1.5974241495132446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1547207534313202,
      "rewards/margins": 0.044812463223934174,
      "rewards/rejected": -0.19953319430351257,
      "step": 438
    },
    {
      "epoch": 0.6918833727344366,
      "grad_norm": 0.22594793140888214,
      "learning_rate": 3.31274025355426e-06,
      "log_odds_chosen": 0.5037475228309631,
      "log_odds_ratio": -0.4795573353767395,
      "logits/chosen": -0.10893750190734863,
      "logits/rejected": -0.843508780002594,
      "logps/chosen": -1.6426455974578857,
      "logps/rejected": -2.06754207611084,
      "loss": 1.7082,
      "nll_loss": 1.6602920293807983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16426457464694977,
      "rewards/margins": 0.042489632964134216,
      "rewards/rejected": -0.20675420761108398,
      "step": 439
    },
    {
      "epoch": 0.6934594168636722,
      "grad_norm": 0.21982692182064056,
      "learning_rate": 3.308588073392265e-06,
      "log_odds_chosen": 0.5123088359832764,
      "log_odds_ratio": -0.48421573638916016,
      "logits/chosen": -0.10025706887245178,
      "logits/rejected": -0.8698948621749878,
      "logps/chosen": -1.6772215366363525,
      "logps/rejected": -2.1163253784179688,
      "loss": 1.7454,
      "nll_loss": 1.6969974040985107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16772215068340302,
      "rewards/margins": 0.043910376727581024,
      "rewards/rejected": -0.21163253486156464,
      "step": 440
    },
    {
      "epoch": 0.6950354609929078,
      "grad_norm": 0.20160053670406342,
      "learning_rate": 3.3044260074724035e-06,
      "log_odds_chosen": 0.5939540863037109,
      "log_odds_ratio": -0.4523179233074188,
      "logits/chosen": -0.11239210516214371,
      "logits/rejected": -1.0533446073532104,
      "logps/chosen": -1.577032208442688,
      "logps/rejected": -2.0725245475769043,
      "loss": 1.667,
      "nll_loss": 1.6217741966247559,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1577032506465912,
      "rewards/margins": 0.04954922944307327,
      "rewards/rejected": -0.20725245773792267,
      "step": 441
    },
    {
      "epoch": 0.6966115051221434,
      "grad_norm": 0.20258580148220062,
      "learning_rate": 3.300254087237097e-06,
      "log_odds_chosen": 0.49938884377479553,
      "log_odds_ratio": -0.4795615077018738,
      "logits/chosen": -0.07772234082221985,
      "logits/rejected": -0.969111442565918,
      "logps/chosen": -1.4605151414871216,
      "logps/rejected": -1.867677927017212,
      "loss": 1.5803,
      "nll_loss": 1.5323054790496826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460515260696411,
      "rewards/margins": 0.04071628302335739,
      "rewards/rejected": -0.1867678016424179,
      "step": 442
    },
    {
      "epoch": 0.698187549251379,
      "grad_norm": 0.20618461072444916,
      "learning_rate": 3.2960723442032105e-06,
      "log_odds_chosen": 0.6804747581481934,
      "log_odds_ratio": -0.4170026481151581,
      "logits/chosen": -0.14151881635189056,
      "logits/rejected": -1.196169376373291,
      "logps/chosen": -1.669822335243225,
      "logps/rejected": -2.254673957824707,
      "loss": 1.7346,
      "nll_loss": 1.6929285526275635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1669822335243225,
      "rewards/margins": 0.05848517641425133,
      "rewards/rejected": -0.22546738386154175,
      "step": 443
    },
    {
      "epoch": 0.6997635933806147,
      "grad_norm": 0.198680579662323,
      "learning_rate": 3.291880809961814e-06,
      "log_odds_chosen": 0.6177921295166016,
      "log_odds_ratio": -0.44483864307403564,
      "logits/chosen": -0.06093186140060425,
      "logits/rejected": -0.8600667715072632,
      "logps/chosen": -1.6328524351119995,
      "logps/rejected": -2.154387950897217,
      "loss": 1.7133,
      "nll_loss": 1.6687994003295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16328522562980652,
      "rewards/margins": 0.052153587341308594,
      "rewards/rejected": -0.2154388129711151,
      "step": 444
    },
    {
      "epoch": 0.7013396375098503,
      "grad_norm": 0.21375496685504913,
      "learning_rate": 3.2876795161779473e-06,
      "log_odds_chosen": 0.6847823858261108,
      "log_odds_ratio": -0.4106296896934509,
      "logits/chosen": -0.07752035558223724,
      "logits/rejected": -1.2114906311035156,
      "logps/chosen": -1.6328837871551514,
      "logps/rejected": -2.21699857711792,
      "loss": 1.6964,
      "nll_loss": 1.655341625213623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16328836977481842,
      "rewards/margins": 0.058411478996276855,
      "rewards/rejected": -0.22169984877109528,
      "step": 445
    },
    {
      "epoch": 0.7029156816390859,
      "grad_norm": 0.21601800620555878,
      "learning_rate": 3.2834684945903776e-06,
      "log_odds_chosen": 0.5214787125587463,
      "log_odds_ratio": -0.47087210416793823,
      "logits/chosen": -0.14094819128513336,
      "logits/rejected": -0.9257452487945557,
      "logps/chosen": -1.5604510307312012,
      "logps/rejected": -1.9898643493652344,
      "loss": 1.6529,
      "nll_loss": 1.6058588027954102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1560451090335846,
      "rewards/margins": 0.042941320687532425,
      "rewards/rejected": -0.19898641109466553,
      "step": 446
    },
    {
      "epoch": 0.7044917257683215,
      "grad_norm": 0.21135088801383972,
      "learning_rate": 3.2792477770113624e-06,
      "log_odds_chosen": 0.4751344323158264,
      "log_odds_ratio": -0.489793986082077,
      "logits/chosen": -0.21940332651138306,
      "logits/rejected": -1.019394874572754,
      "logps/chosen": -1.6616731882095337,
      "logps/rejected": -2.0628890991210938,
      "loss": 1.7293,
      "nll_loss": 1.680342435836792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16616733372211456,
      "rewards/margins": 0.04012158513069153,
      "rewards/rejected": -0.2062889188528061,
      "step": 447
    },
    {
      "epoch": 0.7060677698975572,
      "grad_norm": 0.2001531720161438,
      "learning_rate": 3.275017395326407e-06,
      "log_odds_chosen": 0.41810929775238037,
      "log_odds_ratio": -0.513521671295166,
      "logits/chosen": -0.03719010949134827,
      "logits/rejected": -0.803317666053772,
      "logps/chosen": -1.5478110313415527,
      "logps/rejected": -1.887550950050354,
      "loss": 1.644,
      "nll_loss": 1.5926766395568848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15478110313415527,
      "rewards/margins": 0.033973999321460724,
      "rewards/rejected": -0.1887550950050354,
      "step": 448
    },
    {
      "epoch": 0.7076438140267928,
      "grad_norm": 0.2054382860660553,
      "learning_rate": 3.2707773814940244e-06,
      "log_odds_chosen": 0.5953386425971985,
      "log_odds_ratio": -0.44647330045700073,
      "logits/chosen": -0.07013065367937088,
      "logits/rejected": -1.0036925077438354,
      "logps/chosen": -1.5742981433868408,
      "logps/rejected": -2.065192222595215,
      "loss": 1.6672,
      "nll_loss": 1.6225340366363525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15742981433868408,
      "rewards/margins": 0.04908942058682442,
      "rewards/rejected": -0.2065192461013794,
      "step": 449
    },
    {
      "epoch": 0.7092198581560284,
      "grad_norm": 0.19840051233768463,
      "learning_rate": 3.2665277675454935e-06,
      "log_odds_chosen": 0.648217499256134,
      "log_odds_ratio": -0.4395694434642792,
      "logits/chosen": -0.13534650206565857,
      "logits/rejected": -1.220763087272644,
      "logps/chosen": -1.650367259979248,
      "logps/rejected": -2.209742546081543,
      "loss": 1.7142,
      "nll_loss": 1.6702194213867188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16503672301769257,
      "rewards/margins": 0.0559375137090683,
      "rewards/rejected": -0.22097423672676086,
      "step": 450
    },
    {
      "epoch": 0.710795902285264,
      "grad_norm": 0.1972638964653015,
      "learning_rate": 3.262268585584619e-06,
      "log_odds_chosen": 0.5737844705581665,
      "log_odds_ratio": -0.4500337839126587,
      "logits/chosen": -0.041236281394958496,
      "logits/rejected": -0.917519211769104,
      "logps/chosen": -1.5991514921188354,
      "logps/rejected": -2.0825254917144775,
      "loss": 1.6783,
      "nll_loss": 1.6332542896270752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15991514921188354,
      "rewards/margins": 0.048337388783693314,
      "rewards/rejected": -0.20825253427028656,
      "step": 451
    },
    {
      "epoch": 0.7123719464144996,
      "grad_norm": 0.21535180509090424,
      "learning_rate": 3.2579998677874853e-06,
      "log_odds_chosen": 0.6206746101379395,
      "log_odds_ratio": -0.44026538729667664,
      "logits/chosen": -0.12528139352798462,
      "logits/rejected": -0.9106650352478027,
      "logps/chosen": -1.6482163667678833,
      "logps/rejected": -2.1782073974609375,
      "loss": 1.7116,
      "nll_loss": 1.6675549745559692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16482165455818176,
      "rewards/margins": 0.05299907922744751,
      "rewards/rejected": -0.21782071888446808,
      "step": 452
    },
    {
      "epoch": 0.7139479905437353,
      "grad_norm": 0.1972205489873886,
      "learning_rate": 3.2537216464022155e-06,
      "log_odds_chosen": 0.7232244610786438,
      "log_odds_ratio": -0.4066394865512848,
      "logits/chosen": -0.11965236067771912,
      "logits/rejected": -0.8536319136619568,
      "logps/chosen": -1.64108145236969,
      "logps/rejected": -2.2579519748687744,
      "loss": 1.7132,
      "nll_loss": 1.6725622415542603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16410814225673676,
      "rewards/margins": 0.06168707460165024,
      "rewards/rejected": -0.2257952094078064,
      "step": 453
    },
    {
      "epoch": 0.7155240346729709,
      "grad_norm": 0.19788628816604614,
      "learning_rate": 3.2494339537487314e-06,
      "log_odds_chosen": 0.48940759897232056,
      "log_odds_ratio": -0.47984084486961365,
      "logits/chosen": -0.13651666045188904,
      "logits/rejected": -1.131873607635498,
      "logps/chosen": -1.6382532119750977,
      "logps/rejected": -2.0482289791107178,
      "loss": 1.7249,
      "nll_loss": 1.6769294738769531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16382533311843872,
      "rewards/margins": 0.04099758341908455,
      "rewards/rejected": -0.20482289791107178,
      "step": 454
    },
    {
      "epoch": 0.7171000788022065,
      "grad_norm": 0.1971239149570465,
      "learning_rate": 3.2451368222185006e-06,
      "log_odds_chosen": 0.3923978805541992,
      "log_odds_ratio": -0.5212135314941406,
      "logits/chosen": -0.12293338775634766,
      "logits/rejected": -0.8603593111038208,
      "logps/chosen": -1.5797085762023926,
      "logps/rejected": -1.901029348373413,
      "loss": 1.6756,
      "nll_loss": 1.6234517097473145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1579708606004715,
      "rewards/margins": 0.032132066786289215,
      "rewards/rejected": -0.1901029348373413,
      "step": 455
    },
    {
      "epoch": 0.7186761229314421,
      "grad_norm": 0.18168888986110687,
      "learning_rate": 3.2408302842743007e-06,
      "log_odds_chosen": 0.624774694442749,
      "log_odds_ratio": -0.4350818395614624,
      "logits/chosen": -0.08135272562503815,
      "logits/rejected": -0.9583423733711243,
      "logps/chosen": -1.5263901948928833,
      "logps/rejected": -2.04479718208313,
      "loss": 1.6239,
      "nll_loss": 1.580367922782898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526390165090561,
      "rewards/margins": 0.05184072256088257,
      "rewards/rejected": -0.20447973906993866,
      "step": 456
    },
    {
      "epoch": 0.7202521670606777,
      "grad_norm": 0.19839978218078613,
      "learning_rate": 3.2365143724499684e-06,
      "log_odds_chosen": 0.5523067712783813,
      "log_odds_ratio": -0.45656055212020874,
      "logits/chosen": -0.09882719814777374,
      "logits/rejected": -0.9976221323013306,
      "logps/chosen": -1.5835844278335571,
      "logps/rejected": -2.0447030067443848,
      "loss": 1.6806,
      "nll_loss": 1.6349064111709595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15835845470428467,
      "rewards/margins": 0.04611185938119888,
      "rewards/rejected": -0.20447030663490295,
      "step": 457
    },
    {
      "epoch": 0.7218282111899134,
      "grad_norm": 0.2024889588356018,
      "learning_rate": 3.2321891193501564e-06,
      "log_odds_chosen": 0.595514714717865,
      "log_odds_ratio": -0.4551790952682495,
      "logits/chosen": -0.06068085506558418,
      "logits/rejected": -0.5954574942588806,
      "logps/chosen": -1.5871148109436035,
      "logps/rejected": -2.089128017425537,
      "loss": 1.6641,
      "nll_loss": 1.6186047792434692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1587114930152893,
      "rewards/margins": 0.05020132660865784,
      "rewards/rejected": -0.20891280472278595,
      "step": 458
    },
    {
      "epoch": 0.723404255319149,
      "grad_norm": 0.19239795207977295,
      "learning_rate": 3.2278545576500858e-06,
      "log_odds_chosen": 0.7714004516601562,
      "log_odds_ratio": -0.38966119289398193,
      "logits/chosen": 0.014270449988543987,
      "logits/rejected": -0.7276147603988647,
      "logps/chosen": -1.4306761026382446,
      "logps/rejected": -2.06968355178833,
      "loss": 1.5279,
      "nll_loss": 1.4889153242111206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430675983428955,
      "rewards/margins": 0.06390075385570526,
      "rewards/rejected": -0.20696833729743958,
      "step": 459
    },
    {
      "epoch": 0.7249802994483846,
      "grad_norm": 0.20809946954250336,
      "learning_rate": 3.223510720095299e-06,
      "log_odds_chosen": 0.7304480671882629,
      "log_odds_ratio": -0.40124380588531494,
      "logits/chosen": -0.1674218326807022,
      "logits/rejected": -1.0309466123580933,
      "logps/chosen": -1.6155967712402344,
      "logps/rejected": -2.2373650074005127,
      "loss": 1.6819,
      "nll_loss": 1.6417958736419678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16155968606472015,
      "rewards/margins": 0.06217682361602783,
      "rewards/rejected": -0.2237364947795868,
      "step": 460
    },
    {
      "epoch": 0.7265563435776202,
      "grad_norm": 0.19244162738323212,
      "learning_rate": 3.2191576395014158e-06,
      "log_odds_chosen": 0.6560048460960388,
      "log_odds_ratio": -0.42095711827278137,
      "logits/chosen": -0.10353583842515945,
      "logits/rejected": -0.9026417136192322,
      "logps/chosen": -1.5507631301879883,
      "logps/rejected": -2.0990896224975586,
      "loss": 1.6312,
      "nll_loss": 1.5891027450561523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15507632493972778,
      "rewards/margins": 0.054832637310028076,
      "rewards/rejected": -0.20990897715091705,
      "step": 461
    },
    {
      "epoch": 0.7281323877068558,
      "grad_norm": 0.196467325091362,
      "learning_rate": 3.2147953487538794e-06,
      "log_odds_chosen": 0.6500884294509888,
      "log_odds_ratio": -0.42691317200660706,
      "logits/chosen": -0.04400169104337692,
      "logits/rejected": -0.973447322845459,
      "logps/chosen": -1.519189476966858,
      "logps/rejected": -2.0596096515655518,
      "loss": 1.5998,
      "nll_loss": 1.5571023225784302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1519189476966858,
      "rewards/margins": 0.054042041301727295,
      "rewards/rejected": -0.20596098899841309,
      "step": 462
    },
    {
      "epoch": 0.7297084318360915,
      "grad_norm": 0.20894914865493774,
      "learning_rate": 3.2104238808077133e-06,
      "log_odds_chosen": 0.5251021981239319,
      "log_odds_ratio": -0.474967896938324,
      "logits/chosen": -0.07077591866254807,
      "logits/rejected": -0.7537460327148438,
      "logps/chosen": -1.5554322004318237,
      "logps/rejected": -1.992119550704956,
      "loss": 1.6398,
      "nll_loss": 1.5922644138336182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1555432230234146,
      "rewards/margins": 0.04366873577237129,
      "rewards/rejected": -0.1992119550704956,
      "step": 463
    },
    {
      "epoch": 0.731284475965327,
      "grad_norm": 0.19019024074077606,
      "learning_rate": 3.2060432686872704e-06,
      "log_odds_chosen": 0.8081085681915283,
      "log_odds_ratio": -0.3813464343547821,
      "logits/chosen": -0.13858087360858917,
      "logits/rejected": -0.7866955995559692,
      "logps/chosen": -1.4605531692504883,
      "logps/rejected": -2.129701852798462,
      "loss": 1.5568,
      "nll_loss": 1.518630027770996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14605532586574554,
      "rewards/margins": 0.0669148787856102,
      "rewards/rejected": -0.21297019720077515,
      "step": 464
    },
    {
      "epoch": 0.7328605200945626,
      "grad_norm": 0.19666843116283417,
      "learning_rate": 3.201653545485982e-06,
      "log_odds_chosen": 0.6164845824241638,
      "log_odds_ratio": -0.44083070755004883,
      "logits/chosen": -0.0490589365363121,
      "logits/rejected": -0.8681581616401672,
      "logps/chosen": -1.6057840585708618,
      "logps/rejected": -2.1277639865875244,
      "loss": 1.6728,
      "nll_loss": 1.6286746263504028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1605783998966217,
      "rewards/margins": 0.05219800025224686,
      "rewards/rejected": -0.21277640759944916,
      "step": 465
    },
    {
      "epoch": 0.7344365642237982,
      "grad_norm": 0.2226024568080902,
      "learning_rate": 3.197254744366111e-06,
      "log_odds_chosen": 0.6797471642494202,
      "log_odds_ratio": -0.418096125125885,
      "logits/chosen": -0.03286769241094589,
      "logits/rejected": -0.8219181299209595,
      "logps/chosen": -1.515041708946228,
      "logps/rejected": -2.079200267791748,
      "loss": 1.6003,
      "nll_loss": 1.5584660768508911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15150417387485504,
      "rewards/margins": 0.0564158670604229,
      "rewards/rejected": -0.20792002975940704,
      "step": 466
    },
    {
      "epoch": 0.7360126083530338,
      "grad_norm": 0.2114436775445938,
      "learning_rate": 3.192846898558498e-06,
      "log_odds_chosen": 0.5047996044158936,
      "log_odds_ratio": -0.47521814703941345,
      "logits/chosen": -0.0795680359005928,
      "logits/rejected": -0.892188310623169,
      "logps/chosen": -1.6534972190856934,
      "logps/rejected": -2.0805842876434326,
      "loss": 1.7331,
      "nll_loss": 1.6855862140655518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16534970700740814,
      "rewards/margins": 0.042708709836006165,
      "rewards/rejected": -0.2080584317445755,
      "step": 467
    },
    {
      "epoch": 0.7375886524822695,
      "grad_norm": 0.20920921862125397,
      "learning_rate": 3.188430041362313e-06,
      "log_odds_chosen": 0.4469729959964752,
      "log_odds_ratio": -0.5012507438659668,
      "logits/chosen": -0.03374030441045761,
      "logits/rejected": -0.9043279886245728,
      "logps/chosen": -1.645135521888733,
      "logps/rejected": -2.022472381591797,
      "loss": 1.7118,
      "nll_loss": 1.6616994142532349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16451355814933777,
      "rewards/margins": 0.03773367404937744,
      "rewards/rejected": -0.2022472321987152,
      "step": 468
    },
    {
      "epoch": 0.7391646966115051,
      "grad_norm": 0.2058577537536621,
      "learning_rate": 3.184004206144803e-06,
      "log_odds_chosen": 0.6731261014938354,
      "log_odds_ratio": -0.420916885137558,
      "logits/chosen": -0.14038869738578796,
      "logits/rejected": -0.8868641257286072,
      "logps/chosen": -1.5474592447280884,
      "logps/rejected": -2.1062960624694824,
      "loss": 1.6267,
      "nll_loss": 1.5845807790756226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1547459214925766,
      "rewards/margins": 0.05588366463780403,
      "rewards/rejected": -0.21062961220741272,
      "step": 469
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 0.19909264147281647,
      "learning_rate": 3.1795694263410386e-06,
      "log_odds_chosen": 0.7366645932197571,
      "log_odds_ratio": -0.4054761230945587,
      "logits/chosen": -0.11322111636400223,
      "logits/rejected": -0.9165863990783691,
      "logps/chosen": -1.52811861038208,
      "logps/rejected": -2.152003049850464,
      "loss": 1.6016,
      "nll_loss": 1.5610570907592773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15281186997890472,
      "rewards/margins": 0.062388449907302856,
      "rewards/rejected": -0.21520031988620758,
      "step": 470
    },
    {
      "epoch": 0.7423167848699763,
      "grad_norm": 0.1952643245458603,
      "learning_rate": 3.1751257354536634e-06,
      "log_odds_chosen": 0.520018458366394,
      "log_odds_ratio": -0.48046696186065674,
      "logits/chosen": -0.07400179654359818,
      "logits/rejected": -1.0409823656082153,
      "logps/chosen": -1.5231245756149292,
      "logps/rejected": -1.958061695098877,
      "loss": 1.6132,
      "nll_loss": 1.5651495456695557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15231245756149292,
      "rewards/margins": 0.043493710458278656,
      "rewards/rejected": -0.19580616056919098,
      "step": 471
    },
    {
      "epoch": 0.7438928289992119,
      "grad_norm": 0.20432332158088684,
      "learning_rate": 3.1706731670526394e-06,
      "log_odds_chosen": 0.5711886882781982,
      "log_odds_ratio": -0.4503237009048462,
      "logits/chosen": -0.1537346988916397,
      "logits/rejected": -0.8866163492202759,
      "logps/chosen": -1.5807658433914185,
      "logps/rejected": -2.0559804439544678,
      "loss": 1.6442,
      "nll_loss": 1.5991592407226562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15807659924030304,
      "rewards/margins": 0.04752147197723389,
      "rewards/rejected": -0.20559807121753693,
      "step": 472
    },
    {
      "epoch": 0.7454688731284476,
      "grad_norm": 0.19950729608535767,
      "learning_rate": 3.166211754774994e-06,
      "log_odds_chosen": 0.6052391529083252,
      "log_odds_ratio": -0.4445112943649292,
      "logits/chosen": -0.1134694516658783,
      "logits/rejected": -1.0386989116668701,
      "logps/chosen": -1.629151701927185,
      "logps/rejected": -2.1410460472106934,
      "loss": 1.7275,
      "nll_loss": 1.683068871498108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1629151701927185,
      "rewards/margins": 0.05118943005800247,
      "rewards/rejected": -0.21410460770130157,
      "step": 473
    },
    {
      "epoch": 0.7470449172576832,
      "grad_norm": 0.20564451813697815,
      "learning_rate": 3.1617415323245665e-06,
      "log_odds_chosen": 0.5903938412666321,
      "log_odds_ratio": -0.45206212997436523,
      "logits/chosen": -0.14191381633281708,
      "logits/rejected": -0.9877166748046875,
      "logps/chosen": -1.5389567613601685,
      "logps/rejected": -2.0210845470428467,
      "loss": 1.6103,
      "nll_loss": 1.5650646686553955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15389567613601685,
      "rewards/margins": 0.04821278899908066,
      "rewards/rejected": -0.2021084576845169,
      "step": 474
    },
    {
      "epoch": 0.7486209613869188,
      "grad_norm": 0.19367830455303192,
      "learning_rate": 3.157262533471752e-06,
      "log_odds_chosen": 0.6683068871498108,
      "log_odds_ratio": -0.41824913024902344,
      "logits/chosen": -0.07336349785327911,
      "logits/rejected": -1.0358256101608276,
      "logps/chosen": -1.5820767879486084,
      "logps/rejected": -2.1471550464630127,
      "loss": 1.6612,
      "nll_loss": 1.6193286180496216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15820768475532532,
      "rewards/margins": 0.05650782957673073,
      "rewards/rejected": -0.21471551060676575,
      "step": 475
    },
    {
      "epoch": 0.7501970055161544,
      "grad_norm": 0.20863281190395355,
      "learning_rate": 3.1527747920532468e-06,
      "log_odds_chosen": 0.5243900418281555,
      "log_odds_ratio": -0.4661727547645569,
      "logits/chosen": -0.08033692091703415,
      "logits/rejected": -0.7630578279495239,
      "logps/chosen": -1.5552059412002563,
      "logps/rejected": -1.9899144172668457,
      "loss": 1.6392,
      "nll_loss": 1.5925623178482056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15552058815956116,
      "rewards/margins": 0.04347085952758789,
      "rewards/rejected": -0.19899143278598785,
      "step": 476
    },
    {
      "epoch": 0.75177304964539,
      "grad_norm": 0.19890214502811432,
      "learning_rate": 3.148278341971795e-06,
      "log_odds_chosen": 0.6338871717453003,
      "log_odds_ratio": -0.4345587193965912,
      "logits/chosen": -0.13997706770896912,
      "logits/rejected": -0.8114413619041443,
      "logps/chosen": -1.5798550844192505,
      "logps/rejected": -2.1142609119415283,
      "loss": 1.6548,
      "nll_loss": 1.6113535165786743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15798552334308624,
      "rewards/margins": 0.053440578281879425,
      "rewards/rejected": -0.21142609417438507,
      "step": 477
    },
    {
      "epoch": 0.7533490937746257,
      "grad_norm": 0.2100202441215515,
      "learning_rate": 3.143773217195929e-06,
      "log_odds_chosen": 0.694851279258728,
      "log_odds_ratio": -0.4138457775115967,
      "logits/chosen": -0.13817080855369568,
      "logits/rejected": -0.9909247159957886,
      "logps/chosen": -1.6483888626098633,
      "logps/rejected": -2.2417492866516113,
      "loss": 1.7141,
      "nll_loss": 1.6727272272109985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16483889520168304,
      "rewards/margins": 0.059336043894290924,
      "rewards/rejected": -0.22417494654655457,
      "step": 478
    },
    {
      "epoch": 0.7549251379038613,
      "grad_norm": 0.19809815287590027,
      "learning_rate": 3.139259451759714e-06,
      "log_odds_chosen": 0.5344829559326172,
      "log_odds_ratio": -0.4648602604866028,
      "logits/chosen": -0.15662164986133575,
      "logits/rejected": -0.9446511268615723,
      "logps/chosen": -1.6435620784759521,
      "logps/rejected": -2.094280242919922,
      "loss": 1.7243,
      "nll_loss": 1.6778401136398315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16435623168945312,
      "rewards/margins": 0.045071791857481,
      "rewards/rejected": -0.20942802727222443,
      "step": 479
    },
    {
      "epoch": 0.7565011820330969,
      "grad_norm": 0.1964648813009262,
      "learning_rate": 3.134737079762493e-06,
      "log_odds_chosen": 0.49474141001701355,
      "log_odds_ratio": -0.48102235794067383,
      "logits/chosen": -0.09573915600776672,
      "logits/rejected": -0.9279856085777283,
      "logps/chosen": -1.5789200067520142,
      "logps/rejected": -1.9883246421813965,
      "loss": 1.6553,
      "nll_loss": 1.6071677207946777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15789198875427246,
      "rewards/margins": 0.04094048589468002,
      "rewards/rejected": -0.1988324671983719,
      "step": 480
    },
    {
      "epoch": 0.7580772261623325,
      "grad_norm": 0.20796369016170502,
      "learning_rate": 3.130206135368626e-06,
      "log_odds_chosen": 0.5486950278282166,
      "log_odds_ratio": -0.46876269578933716,
      "logits/chosen": -0.14321769773960114,
      "logits/rejected": -0.7052893042564392,
      "logps/chosen": -1.4953614473342896,
      "logps/rejected": -1.9454658031463623,
      "loss": 1.5968,
      "nll_loss": 1.5499558448791504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495361477136612,
      "rewards/margins": 0.04501045495271683,
      "rewards/rejected": -0.19454661011695862,
      "step": 481
    },
    {
      "epoch": 0.7596532702915682,
      "grad_norm": 0.20492172241210938,
      "learning_rate": 3.1256666528072327e-06,
      "log_odds_chosen": 0.7209620475769043,
      "log_odds_ratio": -0.4015035629272461,
      "logits/chosen": -0.13152581453323364,
      "logits/rejected": -0.7092732787132263,
      "logps/chosen": -1.5489566326141357,
      "logps/rejected": -2.1531386375427246,
      "loss": 1.6228,
      "nll_loss": 1.5826079845428467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15489566326141357,
      "rewards/margins": 0.06041823327541351,
      "rewards/rejected": -0.2153138965368271,
      "step": 482
    },
    {
      "epoch": 0.7612293144208038,
      "grad_norm": 0.1936446577310562,
      "learning_rate": 3.121118666371937e-06,
      "log_odds_chosen": 0.5240681171417236,
      "log_odds_ratio": -0.47471654415130615,
      "logits/chosen": -0.07757510244846344,
      "logits/rejected": -1.1584354639053345,
      "logps/chosen": -1.6076598167419434,
      "logps/rejected": -2.050039291381836,
      "loss": 1.674,
      "nll_loss": 1.6264898777008057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16076599061489105,
      "rewards/margins": 0.044237952679395676,
      "rewards/rejected": -0.20500393211841583,
      "step": 483
    },
    {
      "epoch": 0.7628053585500394,
      "grad_norm": 0.2122434675693512,
      "learning_rate": 3.1165622104206034e-06,
      "log_odds_chosen": 0.7360461354255676,
      "log_odds_ratio": -0.40052106976509094,
      "logits/chosen": -0.27119144797325134,
      "logits/rejected": -0.8301371932029724,
      "logps/chosen": -1.541393518447876,
      "logps/rejected": -2.163299322128296,
      "loss": 1.618,
      "nll_loss": 1.5779178142547607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15413935482501984,
      "rewards/margins": 0.06219058483839035,
      "rewards/rejected": -0.2163299322128296,
      "step": 484
    },
    {
      "epoch": 0.764381402679275,
      "grad_norm": 0.19363641738891602,
      "learning_rate": 3.1119973193750816e-06,
      "log_odds_chosen": 0.620286226272583,
      "log_odds_ratio": -0.43595632910728455,
      "logits/chosen": -0.16278237104415894,
      "logits/rejected": -0.8815320730209351,
      "logps/chosen": -1.5484811067581177,
      "logps/rejected": -2.066248893737793,
      "loss": 1.6171,
      "nll_loss": 1.5735543966293335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.154848113656044,
      "rewards/margins": 0.05177678167819977,
      "rewards/rejected": -0.20662488043308258,
      "step": 485
    },
    {
      "epoch": 0.7659574468085106,
      "grad_norm": 0.20480599999427795,
      "learning_rate": 3.1074240277209408e-06,
      "log_odds_chosen": 0.5229519605636597,
      "log_odds_ratio": -0.4746035635471344,
      "logits/chosen": -0.13251370191574097,
      "logits/rejected": -0.8053449988365173,
      "logps/chosen": -1.5717124938964844,
      "logps/rejected": -2.003612756729126,
      "loss": 1.6288,
      "nll_loss": 1.581296443939209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15717123448848724,
      "rewards/margins": 0.043190039694309235,
      "rewards/rejected": -0.20036128163337708,
      "step": 486
    },
    {
      "epoch": 0.7675334909377463,
      "grad_norm": 0.22217483818531036,
      "learning_rate": 3.102842370007217e-06,
      "log_odds_chosen": 0.6027494668960571,
      "log_odds_ratio": -0.44312798976898193,
      "logits/chosen": -0.05970533564686775,
      "logits/rejected": -0.7939674258232117,
      "logps/chosen": -1.5866206884384155,
      "logps/rejected": -2.086024045944214,
      "loss": 1.6619,
      "nll_loss": 1.6176303625106812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1586620807647705,
      "rewards/margins": 0.0499403215944767,
      "rewards/rejected": -0.20860238373279572,
      "step": 487
    },
    {
      "epoch": 0.7691095350669819,
      "grad_norm": 0.2019231766462326,
      "learning_rate": 3.0982523808461454e-06,
      "log_odds_chosen": 0.5037473440170288,
      "log_odds_ratio": -0.47748374938964844,
      "logits/chosen": -0.07278791815042496,
      "logits/rejected": -0.8580007553100586,
      "logps/chosen": -1.6201282739639282,
      "logps/rejected": -2.038290023803711,
      "loss": 1.6866,
      "nll_loss": 1.6388630867004395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16201284527778625,
      "rewards/margins": 0.04181616008281708,
      "rewards/rejected": -0.20382899045944214,
      "step": 488
    },
    {
      "epoch": 0.7706855791962175,
      "grad_norm": 0.19960224628448486,
      "learning_rate": 3.0936540949129006e-06,
      "log_odds_chosen": 0.5877503752708435,
      "log_odds_ratio": -0.4546903073787689,
      "logits/chosen": -0.1274491399526596,
      "logits/rejected": -0.8250627517700195,
      "logps/chosen": -1.5944395065307617,
      "logps/rejected": -2.0901992321014404,
      "loss": 1.6698,
      "nll_loss": 1.6242817640304565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1594439595937729,
      "rewards/margins": 0.04957596957683563,
      "rewards/rejected": -0.20901991426944733,
      "step": 489
    },
    {
      "epoch": 0.7722616233254531,
      "grad_norm": 0.1848008632659912,
      "learning_rate": 3.0890475469453378e-06,
      "log_odds_chosen": 0.6991172432899475,
      "log_odds_ratio": -0.4125969707965851,
      "logits/chosen": -0.1196417286992073,
      "logits/rejected": -0.9863728284835815,
      "logps/chosen": -1.4715135097503662,
      "logps/rejected": -2.0471150875091553,
      "loss": 1.5488,
      "nll_loss": 1.507541537284851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14715135097503662,
      "rewards/margins": 0.05756012722849846,
      "rewards/rejected": -0.20471149682998657,
      "step": 490
    },
    {
      "epoch": 0.7738376674546887,
      "grad_norm": 0.19433538615703583,
      "learning_rate": 3.0844327717437263e-06,
      "log_odds_chosen": 0.6452741622924805,
      "log_odds_ratio": -0.4334539771080017,
      "logits/chosen": -0.1350182145833969,
      "logits/rejected": -0.9193748235702515,
      "logps/chosen": -1.4977014064788818,
      "logps/rejected": -2.036151885986328,
      "loss": 1.5787,
      "nll_loss": 1.5353686809539795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14977017045021057,
      "rewards/margins": 0.053845033049583435,
      "rewards/rejected": -0.203615203499794,
      "step": 491
    },
    {
      "epoch": 0.7754137115839244,
      "grad_norm": 0.19196495413780212,
      "learning_rate": 3.0798098041704892e-06,
      "log_odds_chosen": 0.44325220584869385,
      "log_odds_ratio": -0.5018943548202515,
      "logits/chosen": -0.08340089023113251,
      "logits/rejected": -0.9438948631286621,
      "logps/chosen": -1.4235002994537354,
      "logps/rejected": -1.7811520099639893,
      "loss": 1.5264,
      "nll_loss": 1.476258635520935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14235003292560577,
      "rewards/margins": 0.035765163600444794,
      "rewards/rejected": -0.17811518907546997,
      "step": 492
    },
    {
      "epoch": 0.77698975571316,
      "grad_norm": 0.1994369626045227,
      "learning_rate": 3.0751786791499368e-06,
      "log_odds_chosen": 0.5825287103652954,
      "log_odds_ratio": -0.45188358426094055,
      "logits/chosen": -0.10140110552310944,
      "logits/rejected": -0.8894228339195251,
      "logps/chosen": -1.6182955503463745,
      "logps/rejected": -2.1115620136260986,
      "loss": 1.6949,
      "nll_loss": 1.649696707725525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16182956099510193,
      "rewards/margins": 0.04932662099599838,
      "rewards/rejected": -0.2111561894416809,
      "step": 493
    },
    {
      "epoch": 0.7785657998423956,
      "grad_norm": 0.19131864607334137,
      "learning_rate": 3.070539431668008e-06,
      "log_odds_chosen": 0.5550589561462402,
      "log_odds_ratio": -0.4599633514881134,
      "logits/chosen": -0.09749419242143631,
      "logits/rejected": -0.8432252407073975,
      "logps/chosen": -1.6216144561767578,
      "logps/rejected": -2.092540979385376,
      "loss": 1.6915,
      "nll_loss": 1.645534634590149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1621614396572113,
      "rewards/margins": 0.04709266871213913,
      "rewards/rejected": -0.20925410091876984,
      "step": 494
    },
    {
      "epoch": 0.7801418439716312,
      "grad_norm": 0.20070280134677887,
      "learning_rate": 3.0658920967720018e-06,
      "log_odds_chosen": 0.8226516842842102,
      "log_odds_ratio": -0.3818792700767517,
      "logits/chosen": -0.25017157196998596,
      "logits/rejected": -0.8173326253890991,
      "logps/chosen": -1.558310866355896,
      "logps/rejected": -2.2607622146606445,
      "loss": 1.6328,
      "nll_loss": 1.5946537256240845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15583109855651855,
      "rewards/margins": 0.07024513185024261,
      "rewards/rejected": -0.22607621550559998,
      "step": 495
    },
    {
      "epoch": 0.7817178881008668,
      "grad_norm": 0.193608358502388,
      "learning_rate": 3.0612367095703116e-06,
      "log_odds_chosen": 0.6440973877906799,
      "log_odds_ratio": -0.42651355266571045,
      "logits/chosen": -0.10096383839845657,
      "logits/rejected": -1.0842796564102173,
      "logps/chosen": -1.5973050594329834,
      "logps/rejected": -2.1420528888702393,
      "loss": 1.6628,
      "nll_loss": 1.620169997215271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15973050892353058,
      "rewards/margins": 0.05447479337453842,
      "rewards/rejected": -0.2142052799463272,
      "step": 496
    },
    {
      "epoch": 0.7832939322301025,
      "grad_norm": 0.20013193786144257,
      "learning_rate": 3.056573305232167e-06,
      "log_odds_chosen": 0.7367360591888428,
      "log_odds_ratio": -0.39854782819747925,
      "logits/chosen": -0.12801437079906464,
      "logits/rejected": -0.9907239079475403,
      "logps/chosen": -1.5458834171295166,
      "logps/rejected": -2.1604068279266357,
      "loss": 1.6209,
      "nll_loss": 1.5810470581054688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15458835661411285,
      "rewards/margins": 0.061452336609363556,
      "rewards/rejected": -0.2160406857728958,
      "step": 497
    },
    {
      "epoch": 0.7848699763593381,
      "grad_norm": 0.19444431364536285,
      "learning_rate": 3.051901918987359e-06,
      "log_odds_chosen": 0.6834968328475952,
      "log_odds_ratio": -0.41310781240463257,
      "logits/chosen": -0.2773919105529785,
      "logits/rejected": -1.03933584690094,
      "logps/chosen": -1.4846152067184448,
      "logps/rejected": -2.0495989322662354,
      "loss": 1.5632,
      "nll_loss": 1.5218619108200073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14846152067184448,
      "rewards/margins": 0.05649835988879204,
      "rewards/rejected": -0.20495988428592682,
      "step": 498
    },
    {
      "epoch": 0.7864460204885737,
      "grad_norm": 0.22802606225013733,
      "learning_rate": 3.047222586125979e-06,
      "log_odds_chosen": 0.6992647647857666,
      "log_odds_ratio": -0.40760669112205505,
      "logits/chosen": -0.07471290230751038,
      "logits/rejected": -0.5809466242790222,
      "logps/chosen": -1.5056655406951904,
      "logps/rejected": -2.086801767349243,
      "loss": 1.5939,
      "nll_loss": 1.5531069040298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15056656301021576,
      "rewards/margins": 0.05811362713575363,
      "rewards/rejected": -0.2086801826953888,
      "step": 499
    },
    {
      "epoch": 0.7880220646178093,
      "grad_norm": 0.19122202694416046,
      "learning_rate": 3.042535341998152e-06,
      "log_odds_chosen": 0.4867512285709381,
      "log_odds_ratio": -0.485049307346344,
      "logits/chosen": 0.002201020484790206,
      "logits/rejected": -0.9413242936134338,
      "logps/chosen": -1.6487398147583008,
      "logps/rejected": -2.0607011318206787,
      "loss": 1.7045,
      "nll_loss": 1.6559950113296509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16487398743629456,
      "rewards/margins": 0.04119614511728287,
      "rewards/rejected": -0.20607014000415802,
      "step": 500
    },
    {
      "epoch": 0.789598108747045,
      "grad_norm": 0.20656250417232513,
      "learning_rate": 3.037840222013769e-06,
      "log_odds_chosen": 0.5959718227386475,
      "log_odds_ratio": -0.44918760657310486,
      "logits/chosen": -0.08412320166826248,
      "logits/rejected": -0.7087098956108093,
      "logps/chosen": -1.596429705619812,
      "logps/rejected": -2.098769426345825,
      "loss": 1.6604,
      "nll_loss": 1.615524172782898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15964297950267792,
      "rewards/margins": 0.05023399740457535,
      "rewards/rejected": -0.20987696945667267,
      "step": 501
    },
    {
      "epoch": 0.7911741528762806,
      "grad_norm": 0.1939014047384262,
      "learning_rate": 3.033137261642219e-06,
      "log_odds_chosen": 0.7794547080993652,
      "log_odds_ratio": -0.39191603660583496,
      "logits/chosen": -0.11343001574277878,
      "logits/rejected": -0.8432320356369019,
      "logps/chosen": -1.5371760129928589,
      "logps/rejected": -2.196560859680176,
      "loss": 1.6012,
      "nll_loss": 1.5620113611221313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15371759235858917,
      "rewards/margins": 0.06593848019838333,
      "rewards/rejected": -0.2196560651063919,
      "step": 502
    },
    {
      "epoch": 0.7927501970055162,
      "grad_norm": 0.19120925664901733,
      "learning_rate": 3.02842649641212e-06,
      "log_odds_chosen": 0.6927048563957214,
      "log_odds_ratio": -0.41640791296958923,
      "logits/chosen": -0.024987956508994102,
      "logits/rejected": -0.7243817448616028,
      "logps/chosen": -1.5825046300888062,
      "logps/rejected": -2.170022487640381,
      "loss": 1.6543,
      "nll_loss": 1.6126254796981812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15825046598911285,
      "rewards/margins": 0.05875179171562195,
      "rewards/rejected": -0.2170022577047348,
      "step": 503
    },
    {
      "epoch": 0.7943262411347518,
      "grad_norm": 0.18899185955524445,
      "learning_rate": 3.0237079619110554e-06,
      "log_odds_chosen": 0.7431460022926331,
      "log_odds_ratio": -0.3946759104728699,
      "logits/chosen": -0.1665392965078354,
      "logits/rejected": -1.025610327720642,
      "logps/chosen": -1.5416197776794434,
      "logps/rejected": -2.1606340408325195,
      "loss": 1.6072,
      "nll_loss": 1.567701816558838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1541619896888733,
      "rewards/margins": 0.06190142780542374,
      "rewards/rejected": -0.21606341004371643,
      "step": 504
    },
    {
      "epoch": 0.7959022852639874,
      "grad_norm": 0.1975812166929245,
      "learning_rate": 3.0189816937852976e-06,
      "log_odds_chosen": 0.7237305045127869,
      "log_odds_ratio": -0.4039166271686554,
      "logits/chosen": -0.21835149824619293,
      "logits/rejected": -0.9860122203826904,
      "logps/chosen": -1.5278733968734741,
      "logps/rejected": -2.131815195083618,
      "loss": 1.5843,
      "nll_loss": 1.543890118598938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15278732776641846,
      "rewards/margins": 0.06039419770240784,
      "rewards/rejected": -0.2131815254688263,
      "step": 505
    },
    {
      "epoch": 0.797478329393223,
      "grad_norm": 0.19384750723838806,
      "learning_rate": 3.014247727739546e-06,
      "log_odds_chosen": 0.8732264041900635,
      "log_odds_ratio": -0.3583661615848541,
      "logits/chosen": -0.15148352086544037,
      "logits/rejected": -0.9323737621307373,
      "logps/chosen": -1.5279947519302368,
      "logps/rejected": -2.26690411567688,
      "loss": 1.5927,
      "nll_loss": 1.5568546056747437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15279948711395264,
      "rewards/margins": 0.07389093935489655,
      "rewards/rejected": -0.226690411567688,
      "step": 506
    },
    {
      "epoch": 0.7990543735224587,
      "grad_norm": 0.19644856452941895,
      "learning_rate": 3.009506099536653e-06,
      "log_odds_chosen": 0.6030058860778809,
      "log_odds_ratio": -0.43984463810920715,
      "logits/chosen": -0.10360610485076904,
      "logits/rejected": -0.9814126491546631,
      "logps/chosen": -1.5841281414031982,
      "logps/rejected": -2.0906927585601807,
      "loss": 1.6337,
      "nll_loss": 1.5896910429000854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15841279923915863,
      "rewards/margins": 0.05065646022558212,
      "rewards/rejected": -0.20906926691532135,
      "step": 507
    },
    {
      "epoch": 0.8006304176516943,
      "grad_norm": 0.18747441470623016,
      "learning_rate": 3.0047568449973544e-06,
      "log_odds_chosen": 0.8637346029281616,
      "log_odds_ratio": -0.36640870571136475,
      "logits/chosen": -0.20195801556110382,
      "logits/rejected": -0.9524874091148376,
      "logps/chosen": -1.4288989305496216,
      "logps/rejected": -2.1447508335113525,
      "loss": 1.499,
      "nll_loss": 1.4623304605484009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14288990199565887,
      "rewards/margins": 0.07158517837524414,
      "rewards/rejected": -0.21447508037090302,
      "step": 508
    },
    {
      "epoch": 0.8022064617809299,
      "grad_norm": 0.2015579342842102,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.4969418942928314,
      "log_odds_ratio": -0.48475930094718933,
      "logits/chosen": -0.1462898850440979,
      "logits/rejected": -0.8126364350318909,
      "logps/chosen": -1.513292670249939,
      "logps/rejected": -1.9217733144760132,
      "loss": 1.5893,
      "nll_loss": 1.540812373161316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15132926404476166,
      "rewards/margins": 0.040848080068826675,
      "rewards/rejected": -0.19217734038829803,
      "step": 509
    },
    {
      "epoch": 0.8037825059101655,
      "grad_norm": 0.18507680296897888,
      "learning_rate": 2.9952356004802813e-06,
      "log_odds_chosen": 0.4633612632751465,
      "log_odds_ratio": -0.49390658736228943,
      "logits/chosen": -0.11722264438867569,
      "logits/rejected": -0.9260642528533936,
      "logps/chosen": -1.6030315160751343,
      "logps/rejected": -1.9908173084259033,
      "loss": 1.6632,
      "nll_loss": 1.6137889623641968,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16030316054821014,
      "rewards/margins": 0.03877857327461243,
      "rewards/rejected": -0.19908173382282257,
      "step": 510
    },
    {
      "epoch": 0.8053585500394012,
      "grad_norm": 0.18921083211898804,
      "learning_rate": 2.9904636824309625e-06,
      "log_odds_chosen": 0.4750264286994934,
      "log_odds_ratio": -0.48544105887413025,
      "logits/chosen": -0.1430448740720749,
      "logits/rejected": -0.692807137966156,
      "logps/chosen": -1.5226821899414062,
      "logps/rejected": -1.9140468835830688,
      "loss": 1.5969,
      "nll_loss": 1.5483349561691284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15226823091506958,
      "rewards/margins": 0.03913646191358566,
      "rewards/rejected": -0.19140468537807465,
      "step": 511
    },
    {
      "epoch": 0.8069345941686368,
      "grad_norm": 0.21483613550662994,
      "learning_rate": 2.985684281901603e-06,
      "log_odds_chosen": 0.5119884014129639,
      "log_odds_ratio": -0.47515565156936646,
      "logits/chosen": -0.10890363156795502,
      "logits/rejected": -0.7739498019218445,
      "logps/chosen": -1.6445868015289307,
      "logps/rejected": -2.0738658905029297,
      "loss": 1.6983,
      "nll_loss": 1.650826334953308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16445867717266083,
      "rewards/margins": 0.042927902191877365,
      "rewards/rejected": -0.2073865830898285,
      "step": 512
    },
    {
      "epoch": 0.8085106382978723,
      "grad_norm": 0.20813430845737457,
      "learning_rate": 2.980897434998293e-06,
      "log_odds_chosen": 0.7414675354957581,
      "log_odds_ratio": -0.3971366882324219,
      "logits/chosen": -0.16981634497642517,
      "logits/rejected": -0.8147495985031128,
      "logps/chosen": -1.491377592086792,
      "logps/rejected": -2.1044163703918457,
      "loss": 1.563,
      "nll_loss": 1.5232783555984497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14913776516914368,
      "rewards/margins": 0.061303891241550446,
      "rewards/rejected": -0.21044164896011353,
      "step": 513
    },
    {
      "epoch": 0.8100866824271079,
      "grad_norm": 0.22016926109790802,
      "learning_rate": 2.976103177883374e-06,
      "log_odds_chosen": 0.5122324228286743,
      "log_odds_ratio": -0.48156213760375977,
      "logits/chosen": -0.1261415332555771,
      "logits/rejected": -0.7446667551994324,
      "logps/chosen": -1.6461725234985352,
      "logps/rejected": -2.082578659057617,
      "loss": 1.7079,
      "nll_loss": 1.65975821018219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16461724042892456,
      "rewards/margins": 0.0436406210064888,
      "rewards/rejected": -0.20825788378715515,
      "step": 514
    },
    {
      "epoch": 0.8116627265563435,
      "grad_norm": 0.19069533050060272,
      "learning_rate": 2.971301546775167e-06,
      "log_odds_chosen": 0.6997944116592407,
      "log_odds_ratio": -0.40974050760269165,
      "logits/chosen": -0.18052856624126434,
      "logits/rejected": -0.9556470513343811,
      "logps/chosen": -1.548624038696289,
      "logps/rejected": -2.13440203666687,
      "loss": 1.6187,
      "nll_loss": 1.5777188539505005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548624038696289,
      "rewards/margins": 0.05857778713107109,
      "rewards/rejected": -0.2134401947259903,
      "step": 515
    },
    {
      "epoch": 0.8132387706855791,
      "grad_norm": 0.20404374599456787,
      "learning_rate": 2.966492577947704e-06,
      "log_odds_chosen": 0.6551504135131836,
      "log_odds_ratio": -0.42468035221099854,
      "logits/chosen": -0.17244477570056915,
      "logits/rejected": -0.9414330124855042,
      "logps/chosen": -1.6585347652435303,
      "logps/rejected": -2.2181200981140137,
      "loss": 1.7312,
      "nll_loss": 1.6887321472167969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16585347056388855,
      "rewards/margins": 0.05595855414867401,
      "rewards/rejected": -0.22181202471256256,
      "step": 516
    },
    {
      "epoch": 0.8148148148148148,
      "grad_norm": 0.19115567207336426,
      "learning_rate": 2.9616763077304457e-06,
      "log_odds_chosen": 0.678066611289978,
      "log_odds_ratio": -0.41563379764556885,
      "logits/chosen": -0.11129939556121826,
      "logits/rejected": -0.9936434030532837,
      "logps/chosen": -1.5809935331344604,
      "logps/rejected": -2.151888132095337,
      "loss": 1.6612,
      "nll_loss": 1.6196848154067993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15809936821460724,
      "rewards/margins": 0.057089436799287796,
      "rewards/rejected": -0.21518880128860474,
      "step": 517
    },
    {
      "epoch": 0.8163908589440504,
      "grad_norm": 0.2233276665210724,
      "learning_rate": 2.956852772508014e-06,
      "log_odds_chosen": 0.5954594612121582,
      "log_odds_ratio": -0.4597008228302002,
      "logits/chosen": -0.11558226495981216,
      "logits/rejected": -0.6093869209289551,
      "logps/chosen": -1.4810335636138916,
      "logps/rejected": -1.9798085689544678,
      "loss": 1.5507,
      "nll_loss": 1.5047768354415894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14810335636138916,
      "rewards/margins": 0.04987748712301254,
      "rewards/rejected": -0.1979808509349823,
      "step": 518
    },
    {
      "epoch": 0.817966903073286,
      "grad_norm": 0.2145313322544098,
      "learning_rate": 2.952022008719914e-06,
      "log_odds_chosen": 0.39059752225875854,
      "log_odds_ratio": -0.5197114944458008,
      "logits/chosen": -0.09284790605306625,
      "logits/rejected": -0.6995605826377869,
      "logps/chosen": -1.632112979888916,
      "logps/rejected": -1.9580031633377075,
      "loss": 1.6978,
      "nll_loss": 1.6458323001861572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16321128606796265,
      "rewards/margins": 0.03258903697133064,
      "rewards/rejected": -0.195800319314003,
      "step": 519
    },
    {
      "epoch": 0.8195429472025216,
      "grad_norm": 0.1915220469236374,
      "learning_rate": 2.9471840528602573e-06,
      "log_odds_chosen": 0.5446378588676453,
      "log_odds_ratio": -0.4655386805534363,
      "logits/chosen": -0.2712858319282532,
      "logits/rejected": -0.8173853754997253,
      "logps/chosen": -1.5713272094726562,
      "logps/rejected": -2.024519920349121,
      "loss": 1.636,
      "nll_loss": 1.5894588232040405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15713272988796234,
      "rewards/margins": 0.045319270342588425,
      "rewards/rejected": -0.20245198905467987,
      "step": 520
    },
    {
      "epoch": 0.8211189913317573,
      "grad_norm": 0.20975013077259064,
      "learning_rate": 2.9423389414774914e-06,
      "log_odds_chosen": 0.620283842086792,
      "log_odds_ratio": -0.43707895278930664,
      "logits/chosen": -0.233064204454422,
      "logits/rejected": -0.975511908531189,
      "logps/chosen": -1.5373528003692627,
      "logps/rejected": -2.055075168609619,
      "loss": 1.6058,
      "nll_loss": 1.562140941619873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15373528003692627,
      "rewards/margins": 0.051772233098745346,
      "rewards/rejected": -0.20550751686096191,
      "step": 521
    },
    {
      "epoch": 0.8226950354609929,
      "grad_norm": 0.19371354579925537,
      "learning_rate": 2.9374867111741174e-06,
      "log_odds_chosen": 0.6126106381416321,
      "log_odds_ratio": -0.4390874207019806,
      "logits/chosen": -0.10471879690885544,
      "logits/rejected": -0.9505351781845093,
      "logps/chosen": -1.564954400062561,
      "logps/rejected": -2.0767931938171387,
      "loss": 1.6266,
      "nll_loss": 1.582688331604004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15649543702602386,
      "rewards/margins": 0.05118388682603836,
      "rewards/rejected": -0.20767931640148163,
      "step": 522
    },
    {
      "epoch": 0.8242710795902285,
      "grad_norm": 0.20733517408370972,
      "learning_rate": 2.9326273986064177e-06,
      "log_odds_chosen": 0.6589902639389038,
      "log_odds_ratio": -0.4235922396183014,
      "logits/chosen": -0.15513229370117188,
      "logits/rejected": -0.9118279218673706,
      "logps/chosen": -1.5880587100982666,
      "logps/rejected": -2.1406936645507812,
      "loss": 1.663,
      "nll_loss": 1.620676040649414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15880587697029114,
      "rewards/margins": 0.05526348575949669,
      "rewards/rejected": -0.21406936645507812,
      "step": 523
    },
    {
      "epoch": 0.8258471237194641,
      "grad_norm": 0.2172606885433197,
      "learning_rate": 2.9277610404841787e-06,
      "log_odds_chosen": 0.551877498626709,
      "log_odds_ratio": -0.45637860894203186,
      "logits/chosen": -0.19774095714092255,
      "logits/rejected": -0.749905526638031,
      "logps/chosen": -1.561869502067566,
      "logps/rejected": -2.0207343101501465,
      "loss": 1.6293,
      "nll_loss": 1.5836267471313477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15618693828582764,
      "rewards/margins": 0.04588647186756134,
      "rewards/rejected": -0.20207341015338898,
      "step": 524
    },
    {
      "epoch": 0.8274231678486997,
      "grad_norm": 0.2113419771194458,
      "learning_rate": 2.9228876735704107e-06,
      "log_odds_chosen": 0.508874773979187,
      "log_odds_ratio": -0.4748789668083191,
      "logits/chosen": -0.16925950348377228,
      "logits/rejected": -0.8467004299163818,
      "logps/chosen": -1.449049711227417,
      "logps/rejected": -1.8597774505615234,
      "loss": 1.5272,
      "nll_loss": 1.4797534942626953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1449049711227417,
      "rewards/margins": 0.0410727895796299,
      "rewards/rejected": -0.1859777420759201,
      "step": 525
    },
    {
      "epoch": 0.8289992119779354,
      "grad_norm": 0.27638667821884155,
      "learning_rate": 2.9180073346810738e-06,
      "log_odds_chosen": 0.5058953762054443,
      "log_odds_ratio": -0.47599464654922485,
      "logits/chosen": -0.16779431700706482,
      "logits/rejected": -0.8963111042976379,
      "logps/chosen": -1.725398302078247,
      "logps/rejected": -2.158155918121338,
      "loss": 1.7714,
      "nll_loss": 1.7238061428070068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1725398451089859,
      "rewards/margins": 0.04327574372291565,
      "rewards/rejected": -0.21581557393074036,
      "step": 526
    },
    {
      "epoch": 0.830575256107171,
      "grad_norm": 0.19077715277671814,
      "learning_rate": 2.9131200606847957e-06,
      "log_odds_chosen": 0.579885721206665,
      "log_odds_ratio": -0.45431891083717346,
      "logits/chosen": -0.13277938961982727,
      "logits/rejected": -0.8479864597320557,
      "logps/chosen": -1.5826200246810913,
      "logps/rejected": -2.0684566497802734,
      "loss": 1.6499,
      "nll_loss": 1.60448157787323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15826202929019928,
      "rewards/margins": 0.04858363792300224,
      "rewards/rejected": -0.20684565603733063,
      "step": 527
    },
    {
      "epoch": 0.8321513002364066,
      "grad_norm": 0.2045188695192337,
      "learning_rate": 2.9082258885025995e-06,
      "log_odds_chosen": 0.6600230932235718,
      "log_odds_ratio": -0.42511871457099915,
      "logits/chosen": -0.27393457293510437,
      "logits/rejected": -0.9078572392463684,
      "logps/chosen": -1.6413564682006836,
      "logps/rejected": -2.201584577560425,
      "loss": 1.6953,
      "nll_loss": 1.6527621746063232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1641356348991394,
      "rewards/margins": 0.05602281540632248,
      "rewards/rejected": -0.2201584428548813,
      "step": 528
    },
    {
      "epoch": 0.8337273443656422,
      "grad_norm": 0.20276007056236267,
      "learning_rate": 2.9033248551076167e-06,
      "log_odds_chosen": 0.5023425221443176,
      "log_odds_ratio": -0.4759887456893921,
      "logits/chosen": -0.12282951921224594,
      "logits/rejected": -0.7607415914535522,
      "logps/chosen": -1.693946123123169,
      "logps/rejected": -2.1194987297058105,
      "loss": 1.7581,
      "nll_loss": 1.7104847431182861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1693945974111557,
      "rewards/margins": 0.042555272579193115,
      "rewards/rejected": -0.21194986999034882,
      "step": 529
    },
    {
      "epoch": 0.8353033884948778,
      "grad_norm": 0.21133460104465485,
      "learning_rate": 2.8984169975248138e-06,
      "log_odds_chosen": 0.6771699786186218,
      "log_odds_ratio": -0.41483455896377563,
      "logits/chosen": -0.1729290634393692,
      "logits/rejected": -0.8104040622711182,
      "logps/chosen": -1.5635886192321777,
      "logps/rejected": -2.1283154487609863,
      "loss": 1.6396,
      "nll_loss": 1.5981507301330566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15635888278484344,
      "rewards/margins": 0.05647267401218414,
      "rewards/rejected": -0.2128315567970276,
      "step": 530
    },
    {
      "epoch": 0.8368794326241135,
      "grad_norm": 0.18891829252243042,
      "learning_rate": 2.893502352830712e-06,
      "log_odds_chosen": 0.8367829918861389,
      "log_odds_ratio": -0.36633914709091187,
      "logits/chosen": -0.2112298309803009,
      "logits/rejected": -0.9446803331375122,
      "logps/chosen": -1.458343267440796,
      "logps/rejected": -2.154154062271118,
      "loss": 1.5149,
      "nll_loss": 1.4782843589782715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1458343267440796,
      "rewards/margins": 0.06958108395338058,
      "rewards/rejected": -0.21541538834571838,
      "step": 531
    },
    {
      "epoch": 0.8384554767533491,
      "grad_norm": 0.19002628326416016,
      "learning_rate": 2.888580958153103e-06,
      "log_odds_chosen": 0.6682854890823364,
      "log_odds_ratio": -0.41821667551994324,
      "logits/chosen": -0.2233610898256302,
      "logits/rejected": -0.902778148651123,
      "logps/chosen": -1.509924054145813,
      "logps/rejected": -2.0646915435791016,
      "loss": 1.5893,
      "nll_loss": 1.547492504119873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15099239349365234,
      "rewards/margins": 0.055476751178503036,
      "rewards/rejected": -0.20646914839744568,
      "step": 532
    },
    {
      "epoch": 0.8400315208825847,
      "grad_norm": 0.2150018960237503,
      "learning_rate": 2.8836528506707733e-06,
      "log_odds_chosen": 0.7340890765190125,
      "log_odds_ratio": -0.39939868450164795,
      "logits/chosen": -0.19120405614376068,
      "logits/rejected": -0.8698283433914185,
      "logps/chosen": -1.592369794845581,
      "logps/rejected": -2.215693473815918,
      "loss": 1.6508,
      "nll_loss": 1.6108508110046387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15923698246479034,
      "rewards/margins": 0.06233234703540802,
      "rewards/rejected": -0.22156932950019836,
      "step": 533
    },
    {
      "epoch": 0.8416075650118203,
      "grad_norm": 0.18880720436573029,
      "learning_rate": 2.878718067613222e-06,
      "log_odds_chosen": 0.5389696359634399,
      "log_odds_ratio": -0.46682363748550415,
      "logits/chosen": -0.1631683111190796,
      "logits/rejected": -0.818675696849823,
      "logps/chosen": -1.6331545114517212,
      "logps/rejected": -2.0882396697998047,
      "loss": 1.6961,
      "nll_loss": 1.6494261026382446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16331546008586884,
      "rewards/margins": 0.04550851136445999,
      "rewards/rejected": -0.20882396399974823,
      "step": 534
    },
    {
      "epoch": 0.843183609141056,
      "grad_norm": 0.2102164924144745,
      "learning_rate": 2.8737766462603763e-06,
      "log_odds_chosen": 0.7052509188652039,
      "log_odds_ratio": -0.42012929916381836,
      "logits/chosen": -0.137245774269104,
      "logits/rejected": -0.6852982640266418,
      "logps/chosen": -1.5779880285263062,
      "logps/rejected": -2.179982900619507,
      "loss": 1.635,
      "nll_loss": 1.5930135250091553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15779879689216614,
      "rewards/margins": 0.06019951403141022,
      "rewards/rejected": -0.21799831092357635,
      "step": 535
    },
    {
      "epoch": 0.8447596532702916,
      "grad_norm": 0.19189202785491943,
      "learning_rate": 2.8688286239423167e-06,
      "log_odds_chosen": 0.5648595690727234,
      "log_odds_ratio": -0.460971862077713,
      "logits/chosen": -0.15599645674228668,
      "logits/rejected": -0.875238299369812,
      "logps/chosen": -1.6101949214935303,
      "logps/rejected": -2.088616132736206,
      "loss": 1.6629,
      "nll_loss": 1.616776704788208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16101950407028198,
      "rewards/margins": 0.047842107713222504,
      "rewards/rejected": -0.20886161923408508,
      "step": 536
    },
    {
      "epoch": 0.8463356973995272,
      "grad_norm": 0.1954520344734192,
      "learning_rate": 2.8638740380389862e-06,
      "log_odds_chosen": 0.6521760821342468,
      "log_odds_ratio": -0.4279599189758301,
      "logits/chosen": -0.18855759501457214,
      "logits/rejected": -0.9558929204940796,
      "logps/chosen": -1.5204603672027588,
      "logps/rejected": -2.0634326934814453,
      "loss": 1.5931,
      "nll_loss": 1.5503343343734741,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15204603970050812,
      "rewards/margins": 0.05429724603891373,
      "rewards/rejected": -0.20634329319000244,
      "step": 537
    },
    {
      "epoch": 0.8479117415287628,
      "grad_norm": 0.20151342451572418,
      "learning_rate": 2.8589129259799164e-06,
      "log_odds_chosen": 0.7290222644805908,
      "log_odds_ratio": -0.40753090381622314,
      "logits/chosen": -0.19693735241889954,
      "logits/rejected": -0.7818280458450317,
      "logps/chosen": -1.5971534252166748,
      "logps/rejected": -2.21592378616333,
      "loss": 1.6524,
      "nll_loss": 1.6116957664489746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15971533954143524,
      "rewards/margins": 0.061877019703388214,
      "rewards/rejected": -0.22159235179424286,
      "step": 538
    },
    {
      "epoch": 0.8494877856579984,
      "grad_norm": 0.1942114531993866,
      "learning_rate": 2.853945325243938e-06,
      "log_odds_chosen": 0.6367645859718323,
      "log_odds_ratio": -0.4276583194732666,
      "logits/chosen": -0.20440728962421417,
      "logits/rejected": -0.7571648955345154,
      "logps/chosen": -1.6106340885162354,
      "logps/rejected": -2.1478118896484375,
      "loss": 1.663,
      "nll_loss": 1.6202239990234375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16106338798999786,
      "rewards/margins": 0.05371779948472977,
      "rewards/rejected": -0.21478119492530823,
      "step": 539
    },
    {
      "epoch": 0.851063829787234,
      "grad_norm": 0.19356879591941833,
      "learning_rate": 2.848971273358903e-06,
      "log_odds_chosen": 0.7260472774505615,
      "log_odds_ratio": -0.3975798189640045,
      "logits/chosen": -0.25415971875190735,
      "logits/rejected": -0.8466203808784485,
      "logps/chosen": -1.5488154888153076,
      "logps/rejected": -2.1581711769104004,
      "loss": 1.5916,
      "nll_loss": 1.5518642663955688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548815220594406,
      "rewards/margins": 0.06093557924032211,
      "rewards/rejected": -0.21581712365150452,
      "step": 540
    },
    {
      "epoch": 0.8526398739164697,
      "grad_norm": 0.21764245629310608,
      "learning_rate": 2.843990807901397e-06,
      "log_odds_chosen": 0.5283202528953552,
      "log_odds_ratio": -0.46983450651168823,
      "logits/chosen": -0.14203937351703644,
      "logits/rejected": -0.6407403945922852,
      "logps/chosen": -1.6387572288513184,
      "logps/rejected": -2.0832724571228027,
      "loss": 1.7019,
      "nll_loss": 1.654943823814392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16387571394443512,
      "rewards/margins": 0.0444515198469162,
      "rewards/rejected": -0.20832723379135132,
      "step": 541
    },
    {
      "epoch": 0.8542159180457053,
      "grad_norm": 0.20828819274902344,
      "learning_rate": 2.839003966496458e-06,
      "log_odds_chosen": 0.7138378620147705,
      "log_odds_ratio": -0.41416510939598083,
      "logits/chosen": -0.07021275162696838,
      "logits/rejected": -0.8677617311477661,
      "logps/chosen": -1.5991896390914917,
      "logps/rejected": -2.202282190322876,
      "loss": 1.6659,
      "nll_loss": 1.6244930028915405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15991896390914917,
      "rewards/margins": 0.06030925735831261,
      "rewards/rejected": -0.22022822499275208,
      "step": 542
    },
    {
      "epoch": 0.8557919621749409,
      "grad_norm": 0.18856418132781982,
      "learning_rate": 2.8340107868172905e-06,
      "log_odds_chosen": 0.6527149081230164,
      "log_odds_ratio": -0.4253949820995331,
      "logits/chosen": -0.18893197178840637,
      "logits/rejected": -0.8991619348526001,
      "logps/chosen": -1.433607578277588,
      "logps/rejected": -1.9691213369369507,
      "loss": 1.5033,
      "nll_loss": 1.4607951641082764,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14336076378822327,
      "rewards/margins": 0.05355136841535568,
      "rewards/rejected": -0.19691210985183716,
      "step": 543
    },
    {
      "epoch": 0.8573680063041765,
      "grad_norm": 0.18478965759277344,
      "learning_rate": 2.8290113065849826e-06,
      "log_odds_chosen": 0.7859818935394287,
      "log_odds_ratio": -0.3904573917388916,
      "logits/chosen": -0.17060324549674988,
      "logits/rejected": -1.0058189630508423,
      "logps/chosen": -1.5429413318634033,
      "logps/rejected": -2.206505537033081,
      "loss": 1.6085,
      "nll_loss": 1.5694491863250732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15429413318634033,
      "rewards/margins": 0.06635642796754837,
      "rewards/rejected": -0.2206505686044693,
      "step": 544
    },
    {
      "epoch": 0.8589440504334122,
      "grad_norm": 0.19526703655719757,
      "learning_rate": 2.8240055635682193e-06,
      "log_odds_chosen": 0.8691021800041199,
      "log_odds_ratio": -0.36303502321243286,
      "logits/chosen": -0.19786478579044342,
      "logits/rejected": -0.9055451154708862,
      "logps/chosen": -1.5170778036117554,
      "logps/rejected": -2.249812364578247,
      "loss": 1.563,
      "nll_loss": 1.52670419216156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15170776844024658,
      "rewards/margins": 0.07327346503734589,
      "rewards/rejected": -0.22498123347759247,
      "step": 545
    },
    {
      "epoch": 0.8605200945626478,
      "grad_norm": 0.21169361472129822,
      "learning_rate": 2.8189935955829973e-06,
      "log_odds_chosen": 0.8027443885803223,
      "log_odds_ratio": -0.3792360723018646,
      "logits/chosen": -0.17718231678009033,
      "logits/rejected": -0.7358558177947998,
      "logps/chosen": -1.5833489894866943,
      "logps/rejected": -2.267822504043579,
      "loss": 1.6353,
      "nll_loss": 1.5973656177520752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1583349108695984,
      "rewards/margins": 0.06844733655452728,
      "rewards/rejected": -0.22678223252296448,
      "step": 546
    },
    {
      "epoch": 0.8620961386918834,
      "grad_norm": 0.19900372624397278,
      "learning_rate": 2.813975440492342e-06,
      "log_odds_chosen": 0.6525126695632935,
      "log_odds_ratio": -0.42862603068351746,
      "logits/chosen": -0.23090821504592896,
      "logits/rejected": -0.9543328881263733,
      "logps/chosen": -1.5641371011734009,
      "logps/rejected": -2.108736276626587,
      "loss": 1.635,
      "nll_loss": 1.5921066999435425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1564137041568756,
      "rewards/margins": 0.05445992201566696,
      "rewards/rejected": -0.21087361872196198,
      "step": 547
    },
    {
      "epoch": 0.863672182821119,
      "grad_norm": 0.21935023367404938,
      "learning_rate": 2.8089511362060182e-06,
      "log_odds_chosen": 0.5071982741355896,
      "log_odds_ratio": -0.47420525550842285,
      "logits/chosen": -0.23329834640026093,
      "logits/rejected": -0.8297457695007324,
      "logps/chosen": -1.710657000541687,
      "logps/rejected": -2.143946886062622,
      "loss": 1.7492,
      "nll_loss": 1.7017606496810913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17106570303440094,
      "rewards/margins": 0.04332900792360306,
      "rewards/rejected": -0.2143947035074234,
      "step": 548
    },
    {
      "epoch": 0.8652482269503546,
      "grad_norm": 0.20773950219154358,
      "learning_rate": 2.803920720680244e-06,
      "log_odds_chosen": 0.6489558219909668,
      "log_odds_ratio": -0.424747109413147,
      "logits/chosen": -0.17254123091697693,
      "logits/rejected": -0.9058185815811157,
      "logps/chosen": -1.560959815979004,
      "logps/rejected": -2.1028122901916504,
      "loss": 1.624,
      "nll_loss": 1.5815582275390625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15609599649906158,
      "rewards/margins": 0.05418524891138077,
      "rewards/rejected": -0.21028123795986176,
      "step": 549
    },
    {
      "epoch": 0.8668242710795903,
      "grad_norm": 0.21484144032001495,
      "learning_rate": 2.7988842319174075e-06,
      "log_odds_chosen": 0.6866887807846069,
      "log_odds_ratio": -0.4149416387081146,
      "logits/chosen": -0.2942747473716736,
      "logits/rejected": -0.9815259575843811,
      "logps/chosen": -1.636769413948059,
      "logps/rejected": -2.2215113639831543,
      "loss": 1.6835,
      "nll_loss": 1.6419565677642822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16367694735527039,
      "rewards/margins": 0.05847420543432236,
      "rewards/rejected": -0.22215114533901215,
      "step": 550
    },
    {
      "epoch": 0.8684003152088259,
      "grad_norm": 0.18475672602653503,
      "learning_rate": 2.7938417079657743e-06,
      "log_odds_chosen": 0.7217756509780884,
      "log_odds_ratio": -0.40571483969688416,
      "logits/chosen": -0.25556260347366333,
      "logits/rejected": -0.9915696978569031,
      "logps/chosen": -1.4683786630630493,
      "logps/rejected": -2.0679759979248047,
      "loss": 1.5334,
      "nll_loss": 1.492841124534607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14683787524700165,
      "rewards/margins": 0.05995973199605942,
      "rewards/rejected": -0.20679758489131927,
      "step": 551
    },
    {
      "epoch": 0.8699763593380615,
      "grad_norm": 0.1895190179347992,
      "learning_rate": 2.7887931869192047e-06,
      "log_odds_chosen": 0.6034907698631287,
      "log_odds_ratio": -0.43990734219551086,
      "logits/chosen": -0.24089497327804565,
      "logits/rejected": -0.9087793827056885,
      "logps/chosen": -1.562459945678711,
      "logps/rejected": -2.067342758178711,
      "loss": 1.6206,
      "nll_loss": 1.5765740871429443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15624599158763885,
      "rewards/margins": 0.05048830062150955,
      "rewards/rejected": -0.2067342847585678,
      "step": 552
    },
    {
      "epoch": 0.8715524034672971,
      "grad_norm": 0.2013726532459259,
      "learning_rate": 2.783738706916865e-06,
      "log_odds_chosen": 0.7799487709999084,
      "log_odds_ratio": -0.3866681158542633,
      "logits/chosen": -0.18822821974754333,
      "logits/rejected": -0.988959789276123,
      "logps/chosen": -1.6002718210220337,
      "logps/rejected": -2.260967969894409,
      "loss": 1.6469,
      "nll_loss": 1.6082266569137573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1600271761417389,
      "rewards/margins": 0.06606961786746979,
      "rewards/rejected": -0.22609679400920868,
      "step": 553
    },
    {
      "epoch": 0.8731284475965327,
      "grad_norm": 0.3795514404773712,
      "learning_rate": 2.7786783061429356e-06,
      "log_odds_chosen": 0.5683963894844055,
      "log_odds_ratio": -0.45066073536872864,
      "logits/chosen": -0.27111372351646423,
      "logits/rejected": -0.7333752512931824,
      "logps/chosen": -1.531263828277588,
      "logps/rejected": -2.0012807846069336,
      "loss": 1.5784,
      "nll_loss": 1.533337116241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15312638878822327,
      "rewards/margins": 0.04700169339776039,
      "rewards/rejected": -0.20012807846069336,
      "step": 554
    },
    {
      "epoch": 0.8747044917257684,
      "grad_norm": 0.2037513703107834,
      "learning_rate": 2.7736120228263287e-06,
      "log_odds_chosen": 0.7218413949012756,
      "log_odds_ratio": -0.4056497812271118,
      "logits/chosen": -0.1924748420715332,
      "logits/rejected": -0.8713416457176208,
      "logps/chosen": -1.5259437561035156,
      "logps/rejected": -2.1311612129211426,
      "loss": 1.5855,
      "nll_loss": 1.5449249744415283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15259435772895813,
      "rewards/margins": 0.06052176654338837,
      "rewards/rejected": -0.2131161391735077,
      "step": 555
    },
    {
      "epoch": 0.876280535855004,
      "grad_norm": 0.17655427753925323,
      "learning_rate": 2.768539895240394e-06,
      "log_odds_chosen": 0.6988765001296997,
      "log_odds_ratio": -0.4172660708427429,
      "logits/chosen": -0.22002924978733063,
      "logits/rejected": -0.9279561638832092,
      "logps/chosen": -1.479904055595398,
      "logps/rejected": -2.054877758026123,
      "loss": 1.5366,
      "nll_loss": 1.4948430061340332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479904055595398,
      "rewards/margins": 0.05749736726284027,
      "rewards/rejected": -0.20548778772354126,
      "step": 556
    },
    {
      "epoch": 0.8778565799842396,
      "grad_norm": 0.20609180629253387,
      "learning_rate": 2.763461961702633e-06,
      "log_odds_chosen": 0.6753147840499878,
      "log_odds_ratio": -0.42125439643859863,
      "logits/chosen": -0.21649648249149323,
      "logits/rejected": -0.9744517803192139,
      "logps/chosen": -1.6015154123306274,
      "logps/rejected": -2.1722609996795654,
      "loss": 1.6348,
      "nll_loss": 1.592639446258545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16015154123306274,
      "rewards/margins": 0.05707456171512604,
      "rewards/rejected": -0.21722610294818878,
      "step": 557
    },
    {
      "epoch": 0.8794326241134752,
      "grad_norm": 0.20336881279945374,
      "learning_rate": 2.758378260574409e-06,
      "log_odds_chosen": 0.6111657619476318,
      "log_odds_ratio": -0.43575984239578247,
      "logits/chosen": -0.20938482880592346,
      "logits/rejected": -0.8878787755966187,
      "logps/chosen": -1.5844966173171997,
      "logps/rejected": -2.0953845977783203,
      "loss": 1.6521,
      "nll_loss": 1.6085572242736816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15844964981079102,
      "rewards/margins": 0.05108879879117012,
      "rewards/rejected": -0.20953845977783203,
      "step": 558
    },
    {
      "epoch": 0.8810086682427108,
      "grad_norm": 0.19161473214626312,
      "learning_rate": 2.753288830260655e-06,
      "log_odds_chosen": 0.9316143989562988,
      "log_odds_ratio": -0.3507584035396576,
      "logits/chosen": -0.2052924931049347,
      "logits/rejected": -1.0359580516815186,
      "logps/chosen": -1.5796922445297241,
      "logps/rejected": -2.379153251647949,
      "loss": 1.6248,
      "nll_loss": 1.58968985080719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15796923637390137,
      "rewards/margins": 0.0799461230635643,
      "rewards/rejected": -0.23791533708572388,
      "step": 559
    },
    {
      "epoch": 0.8825847123719465,
      "grad_norm": 0.20678529143333435,
      "learning_rate": 2.7481937092095866e-06,
      "log_odds_chosen": 0.4201931953430176,
      "log_odds_ratio": -0.512961208820343,
      "logits/chosen": -0.14778494834899902,
      "logits/rejected": -0.8609280586242676,
      "logps/chosen": -1.6391096115112305,
      "logps/rejected": -1.996466040611267,
      "loss": 1.6808,
      "nll_loss": 1.6295074224472046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16391095519065857,
      "rewards/margins": 0.035735633224248886,
      "rewards/rejected": -0.19964659214019775,
      "step": 560
    },
    {
      "epoch": 0.8841607565011821,
      "grad_norm": 0.18909314274787903,
      "learning_rate": 2.7430929359124086e-06,
      "log_odds_chosen": 0.5517322421073914,
      "log_odds_ratio": -0.45902585983276367,
      "logits/chosen": -0.22820569574832916,
      "logits/rejected": -0.7544287443161011,
      "logps/chosen": -1.4732322692871094,
      "logps/rejected": -1.9251489639282227,
      "loss": 1.5494,
      "nll_loss": 1.503477931022644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14732322096824646,
      "rewards/margins": 0.04519166052341461,
      "rewards/rejected": -0.19251491129398346,
      "step": 561
    },
    {
      "epoch": 0.8857368006304176,
      "grad_norm": 0.20790156722068787,
      "learning_rate": 2.737986548903029e-06,
      "log_odds_chosen": 0.9357652068138123,
      "log_odds_ratio": -0.33474409580230713,
      "logits/chosen": -0.40204527974128723,
      "logits/rejected": -0.9759889245033264,
      "logps/chosen": -1.459343433380127,
      "logps/rejected": -2.243154764175415,
      "loss": 1.5244,
      "nll_loss": 1.490889310836792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1459343433380127,
      "rewards/margins": 0.07838112115859985,
      "rewards/rejected": -0.22431546449661255,
      "step": 562
    },
    {
      "epoch": 0.8873128447596532,
      "grad_norm": 0.19877144694328308,
      "learning_rate": 2.7328745867577604e-06,
      "log_odds_chosen": 0.557167112827301,
      "log_odds_ratio": -0.45478886365890503,
      "logits/chosen": -0.19379200041294098,
      "logits/rejected": -0.9167407155036926,
      "logps/chosen": -1.6136691570281982,
      "logps/rejected": -2.081575393676758,
      "loss": 1.6765,
      "nll_loss": 1.6310021877288818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16136692464351654,
      "rewards/margins": 0.04679061099886894,
      "rewards/rejected": -0.20815753936767578,
      "step": 563
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 0.21385206282138824,
      "learning_rate": 2.727757088095037e-06,
      "log_odds_chosen": 0.7267173528671265,
      "log_odds_ratio": -0.39710837602615356,
      "logits/chosen": -0.20846852660179138,
      "logits/rejected": -0.9595527052879333,
      "logps/chosen": -1.6061768531799316,
      "logps/rejected": -2.220876932144165,
      "loss": 1.6546,
      "nll_loss": 1.6148784160614014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16061770915985107,
      "rewards/margins": 0.061469994485378265,
      "rewards/rejected": -0.22208771109580994,
      "step": 564
    },
    {
      "epoch": 0.8904649330181245,
      "grad_norm": 0.201381117105484,
      "learning_rate": 2.7226340915751156e-06,
      "log_odds_chosen": 0.6429829001426697,
      "log_odds_ratio": -0.43055054545402527,
      "logits/chosen": -0.2115229070186615,
      "logits/rejected": -0.8014180660247803,
      "logps/chosen": -1.6048914194107056,
      "logps/rejected": -2.1478219032287598,
      "loss": 1.6677,
      "nll_loss": 1.6246284246444702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16048914194107056,
      "rewards/margins": 0.054293058812618256,
      "rewards/rejected": -0.2147822082042694,
      "step": 565
    },
    {
      "epoch": 0.8920409771473601,
      "grad_norm": 0.19357822835445404,
      "learning_rate": 2.7175056358997887e-06,
      "log_odds_chosen": 0.49270471930503845,
      "log_odds_ratio": -0.4904627501964569,
      "logits/chosen": -0.13550208508968353,
      "logits/rejected": -1.0521605014801025,
      "logps/chosen": -1.5758802890777588,
      "logps/rejected": -1.9880424737930298,
      "loss": 1.6311,
      "nll_loss": 1.5820876359939575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15758806467056274,
      "rewards/margins": 0.04121619462966919,
      "rewards/rejected": -0.19880424439907074,
      "step": 566
    },
    {
      "epoch": 0.8936170212765957,
      "grad_norm": 0.19979268312454224,
      "learning_rate": 2.7123717598120892e-06,
      "log_odds_chosen": 0.7791353464126587,
      "log_odds_ratio": -0.3947252929210663,
      "logits/chosen": -0.31340545415878296,
      "logits/rejected": -0.9945352077484131,
      "logps/chosen": -1.53523588180542,
      "logps/rejected": -2.19802188873291,
      "loss": 1.5896,
      "nll_loss": 1.5501768589019775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15352359414100647,
      "rewards/margins": 0.0662786215543747,
      "rewards/rejected": -0.21980220079421997,
      "step": 567
    },
    {
      "epoch": 0.8951930654058313,
      "grad_norm": 0.1915675550699234,
      "learning_rate": 2.7072325020959985e-06,
      "log_odds_chosen": 0.7476974725723267,
      "log_odds_ratio": -0.3926428258419037,
      "logits/chosen": -0.14248669147491455,
      "logits/rejected": -0.75220787525177,
      "logps/chosen": -1.4232914447784424,
      "logps/rejected": -2.03520131111145,
      "loss": 1.4987,
      "nll_loss": 1.4594628810882568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1423291563987732,
      "rewards/margins": 0.06119098886847496,
      "rewards/rejected": -0.20352013409137726,
      "step": 568
    },
    {
      "epoch": 0.8967691095350669,
      "grad_norm": 0.20593954622745514,
      "learning_rate": 2.702087901576155e-06,
      "log_odds_chosen": 0.7687501907348633,
      "log_odds_ratio": -0.38312792778015137,
      "logits/chosen": -0.08590427041053772,
      "logits/rejected": -0.810374915599823,
      "logps/chosen": -1.560744047164917,
      "logps/rejected": -2.2092761993408203,
      "loss": 1.5991,
      "nll_loss": 1.560828685760498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1560744047164917,
      "rewards/margins": 0.06485321372747421,
      "rewards/rejected": -0.22092761099338531,
      "step": 569
    },
    {
      "epoch": 0.8983451536643026,
      "grad_norm": 0.20341774821281433,
      "learning_rate": 2.6969379971175576e-06,
      "log_odds_chosen": 0.7138971090316772,
      "log_odds_ratio": -0.4054667353630066,
      "logits/chosen": -0.21046526730060577,
      "logits/rejected": -0.9977098107337952,
      "logps/chosen": -1.6091594696044922,
      "logps/rejected": -2.2165942192077637,
      "loss": 1.6676,
      "nll_loss": 1.6270349025726318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16091594099998474,
      "rewards/margins": 0.06074347719550133,
      "rewards/rejected": -0.22165942192077637,
      "step": 570
    },
    {
      "epoch": 0.8999211977935382,
      "grad_norm": 0.207947239279747,
      "learning_rate": 2.6917828276252745e-06,
      "log_odds_chosen": 0.8034748435020447,
      "log_odds_ratio": -0.376839280128479,
      "logits/chosen": -0.19248998165130615,
      "logits/rejected": -0.9158348441123962,
      "logps/chosen": -1.522568702697754,
      "logps/rejected": -2.19545841217041,
      "loss": 1.5745,
      "nll_loss": 1.5367984771728516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15225687623023987,
      "rewards/margins": 0.06728895008563995,
      "rewards/rejected": -0.21954582631587982,
      "step": 571
    },
    {
      "epoch": 0.9014972419227738,
      "grad_norm": 0.1965232938528061,
      "learning_rate": 2.686622432044149e-06,
      "log_odds_chosen": 0.6131253242492676,
      "log_odds_ratio": -0.4381985366344452,
      "logits/chosen": -0.19718916714191437,
      "logits/rejected": -0.9542604684829712,
      "logps/chosen": -1.4703774452209473,
      "logps/rejected": -1.976022481918335,
      "loss": 1.5373,
      "nll_loss": 1.4934675693511963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14703774452209473,
      "rewards/margins": 0.050564512610435486,
      "rewards/rejected": -0.1976022571325302,
      "step": 572
    },
    {
      "epoch": 0.9030732860520094,
      "grad_norm": 0.1894710510969162,
      "learning_rate": 2.681456849358505e-06,
      "log_odds_chosen": 0.6845194101333618,
      "log_odds_ratio": -0.4263882339000702,
      "logits/chosen": -0.289983332157135,
      "logits/rejected": -1.058085560798645,
      "logps/chosen": -1.5097733736038208,
      "logps/rejected": -2.0822525024414062,
      "loss": 1.5681,
      "nll_loss": 1.5254263877868652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15097735822200775,
      "rewards/margins": 0.05724789947271347,
      "rewards/rejected": -0.20822525024414062,
      "step": 573
    },
    {
      "epoch": 0.904649330181245,
      "grad_norm": 0.1847888082265854,
      "learning_rate": 2.6762861185918528e-06,
      "log_odds_chosen": 1.0263214111328125,
      "log_odds_ratio": -0.33447879552841187,
      "logits/chosen": -0.3058614730834961,
      "logits/rejected": -0.9919390678405762,
      "logps/chosen": -1.419980764389038,
      "logps/rejected": -2.2853784561157227,
      "loss": 1.4766,
      "nll_loss": 1.4431376457214355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419980674982071,
      "rewards/margins": 0.08653976768255234,
      "rewards/rejected": -0.22853784263134003,
      "step": 574
    },
    {
      "epoch": 0.9062253743104807,
      "grad_norm": 0.20123165845870972,
      "learning_rate": 2.6711102788065934e-06,
      "log_odds_chosen": 0.8114575147628784,
      "log_odds_ratio": -0.38528120517730713,
      "logits/chosen": -0.21853798627853394,
      "logits/rejected": -0.9192309379577637,
      "logps/chosen": -1.6038267612457275,
      "logps/rejected": -2.302663564682007,
      "loss": 1.6605,
      "nll_loss": 1.6220036745071411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16038267314434052,
      "rewards/margins": 0.06988368928432465,
      "rewards/rejected": -0.23026636242866516,
      "step": 575
    },
    {
      "epoch": 0.9078014184397163,
      "grad_norm": 0.21373559534549713,
      "learning_rate": 2.665929369103724e-06,
      "log_odds_chosen": 0.49332332611083984,
      "log_odds_ratio": -0.4839211106300354,
      "logits/chosen": -0.17559638619422913,
      "logits/rejected": -0.7130711078643799,
      "logps/chosen": -1.6556646823883057,
      "logps/rejected": -2.067650318145752,
      "loss": 1.6986,
      "nll_loss": 1.6501877307891846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16556645929813385,
      "rewards/margins": 0.04119856655597687,
      "rewards/rejected": -0.20676502585411072,
      "step": 576
    },
    {
      "epoch": 0.9093774625689519,
      "grad_norm": 0.20101001858711243,
      "learning_rate": 2.6607434286225427e-06,
      "log_odds_chosen": 0.7052325010299683,
      "log_odds_ratio": -0.40979117155075073,
      "logits/chosen": -0.17108431458473206,
      "logits/rejected": -0.9418731331825256,
      "logps/chosen": -1.5434072017669678,
      "logps/rejected": -2.137641191482544,
      "loss": 1.6034,
      "nll_loss": 1.5624289512634277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543407291173935,
      "rewards/margins": 0.05942339450120926,
      "rewards/rejected": -0.21376413106918335,
      "step": 577
    },
    {
      "epoch": 0.9109535066981875,
      "grad_norm": 0.19382938742637634,
      "learning_rate": 2.6555524965403533e-06,
      "log_odds_chosen": 0.8655090928077698,
      "log_odds_ratio": -0.35367241501808167,
      "logits/chosen": -0.2748557925224304,
      "logits/rejected": -0.9562632441520691,
      "logps/chosen": -1.459355354309082,
      "logps/rejected": -2.1797609329223633,
      "loss": 1.5252,
      "nll_loss": 1.489851713180542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1459355503320694,
      "rewards/margins": 0.07204054296016693,
      "rewards/rejected": -0.21797609329223633,
      "step": 578
    },
    {
      "epoch": 0.9125295508274232,
      "grad_norm": 0.1983761489391327,
      "learning_rate": 2.6503566120721683e-06,
      "log_odds_chosen": 0.8486279845237732,
      "log_odds_ratio": -0.3594459891319275,
      "logits/chosen": -0.21223284304141998,
      "logits/rejected": -0.9987262487411499,
      "logps/chosen": -1.553979516029358,
      "logps/rejected": -2.272264003753662,
      "loss": 1.6098,
      "nll_loss": 1.5738502740859985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553979367017746,
      "rewards/margins": 0.0718284472823143,
      "rewards/rejected": -0.2272263765335083,
      "step": 579
    },
    {
      "epoch": 0.9141055949566588,
      "grad_norm": 0.1951398104429245,
      "learning_rate": 2.6451558144704126e-06,
      "log_odds_chosen": 0.7717759609222412,
      "log_odds_ratio": -0.3853806257247925,
      "logits/chosen": -0.3385624587535858,
      "logits/rejected": -0.9782055020332336,
      "logps/chosen": -1.516585111618042,
      "logps/rejected": -2.162315607070923,
      "loss": 1.5614,
      "nll_loss": 1.5228731632232666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15165852010250092,
      "rewards/margins": 0.06457304954528809,
      "rewards/rejected": -0.2162315547466278,
      "step": 580
    },
    {
      "epoch": 0.9156816390858944,
      "grad_norm": 0.19969116151332855,
      "learning_rate": 2.6399501430246286e-06,
      "log_odds_chosen": 0.6388647556304932,
      "log_odds_ratio": -0.42486095428466797,
      "logits/chosen": -0.21140539646148682,
      "logits/rejected": -0.9106035828590393,
      "logps/chosen": -1.4945226907730103,
      "logps/rejected": -2.0187599658966064,
      "loss": 1.5456,
      "nll_loss": 1.5030843019485474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14945226907730103,
      "rewards/margins": 0.05242374166846275,
      "rewards/rejected": -0.20187601447105408,
      "step": 581
    },
    {
      "epoch": 0.91725768321513,
      "grad_norm": 0.19611087441444397,
      "learning_rate": 2.634739637061177e-06,
      "log_odds_chosen": 0.5196619629859924,
      "log_odds_ratio": -0.47210627794265747,
      "logits/chosen": -0.23403318226337433,
      "logits/rejected": -0.9205057621002197,
      "logps/chosen": -1.5480612516403198,
      "logps/rejected": -1.9782025814056396,
      "loss": 1.608,
      "nll_loss": 1.560773253440857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15480613708496094,
      "rewards/margins": 0.043014127761125565,
      "rewards/rejected": -0.1978202611207962,
      "step": 582
    },
    {
      "epoch": 0.9188337273443656,
      "grad_norm": 0.2043604999780655,
      "learning_rate": 2.6295243359429423e-06,
      "log_odds_chosen": 0.8300765752792358,
      "log_odds_ratio": -0.36730027198791504,
      "logits/chosen": -0.22673654556274414,
      "logits/rejected": -0.9699738621711731,
      "logps/chosen": -1.5623438358306885,
      "logps/rejected": -2.266706705093384,
      "loss": 1.6115,
      "nll_loss": 1.5747452974319458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15623438358306885,
      "rewards/margins": 0.0704362764954567,
      "rewards/rejected": -0.22667065262794495,
      "step": 583
    },
    {
      "epoch": 0.9204097714736013,
      "grad_norm": 0.19517943263053894,
      "learning_rate": 2.624304279069033e-06,
      "log_odds_chosen": 0.828274130821228,
      "log_odds_ratio": -0.36922022700309753,
      "logits/chosen": -0.18819034099578857,
      "logits/rejected": -0.8624467253684998,
      "logps/chosen": -1.4918920993804932,
      "logps/rejected": -2.182048797607422,
      "loss": 1.5658,
      "nll_loss": 1.5289130210876465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14918920397758484,
      "rewards/margins": 0.06901565194129944,
      "rewards/rejected": -0.21820485591888428,
      "step": 584
    },
    {
      "epoch": 0.9219858156028369,
      "grad_norm": 0.20722360908985138,
      "learning_rate": 2.6190795058744854e-06,
      "log_odds_chosen": 0.8151520490646362,
      "log_odds_ratio": -0.3838111460208893,
      "logits/chosen": -0.25330740213394165,
      "logits/rejected": -0.8803038597106934,
      "logps/chosen": -1.5780651569366455,
      "logps/rejected": -2.2760934829711914,
      "loss": 1.6281,
      "nll_loss": 1.5897554159164429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15780651569366455,
      "rewards/margins": 0.06980285048484802,
      "rewards/rejected": -0.22760935127735138,
      "step": 585
    },
    {
      "epoch": 0.9235618597320725,
      "grad_norm": 0.21773649752140045,
      "learning_rate": 2.6138500558299664e-06,
      "log_odds_chosen": 0.5938032865524292,
      "log_odds_ratio": -0.44755733013153076,
      "logits/chosen": -0.21899095177650452,
      "logits/rejected": -0.7907366752624512,
      "logps/chosen": -1.574753761291504,
      "logps/rejected": -2.0744011402130127,
      "loss": 1.6253,
      "nll_loss": 1.5805370807647705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15747538208961487,
      "rewards/margins": 0.04996473342180252,
      "rewards/rejected": -0.20744013786315918,
      "step": 586
    },
    {
      "epoch": 0.9251379038613081,
      "grad_norm": 0.21164314448833466,
      "learning_rate": 2.608615968441472e-06,
      "log_odds_chosen": 0.7227214574813843,
      "log_odds_ratio": -0.40597063302993774,
      "logits/chosen": -0.25326594710350037,
      "logits/rejected": -0.9068828821182251,
      "logps/chosen": -1.605982780456543,
      "logps/rejected": -2.2230467796325684,
      "loss": 1.6594,
      "nll_loss": 1.6187876462936401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1605982780456543,
      "rewards/margins": 0.06170640140771866,
      "rewards/rejected": -0.22230468690395355,
      "step": 587
    },
    {
      "epoch": 0.9267139479905437,
      "grad_norm": 0.1992940753698349,
      "learning_rate": 2.6033772832500333e-06,
      "log_odds_chosen": 0.6828200817108154,
      "log_odds_ratio": -0.41489946842193604,
      "logits/chosen": -0.301583856344223,
      "logits/rejected": -0.8837338089942932,
      "logps/chosen": -1.6066690683364868,
      "logps/rejected": -2.1857612133026123,
      "loss": 1.6576,
      "nll_loss": 1.616064190864563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16066691279411316,
      "rewards/margins": 0.05790921300649643,
      "rewards/rejected": -0.21857613325119019,
      "step": 588
    },
    {
      "epoch": 0.9282899921197794,
      "grad_norm": 0.19454245269298553,
      "learning_rate": 2.5981340398314146e-06,
      "log_odds_chosen": 0.7048467397689819,
      "log_odds_ratio": -0.41048330068588257,
      "logits/chosen": -0.32221171259880066,
      "logits/rejected": -1.0305137634277344,
      "logps/chosen": -1.4869518280029297,
      "logps/rejected": -2.069021701812744,
      "loss": 1.5455,
      "nll_loss": 1.5044976472854614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486952006816864,
      "rewards/margins": 0.05820697546005249,
      "rewards/rejected": -0.20690219104290009,
      "step": 589
    },
    {
      "epoch": 0.929866036249015,
      "grad_norm": 0.20854075253009796,
      "learning_rate": 2.592886277795815e-06,
      "log_odds_chosen": 0.7627784609794617,
      "log_odds_ratio": -0.39803192019462585,
      "logits/chosen": -0.2654706537723541,
      "logits/rejected": -1.1287604570388794,
      "logps/chosen": -1.6166093349456787,
      "logps/rejected": -2.2710747718811035,
      "loss": 1.6748,
      "nll_loss": 1.6350226402282715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16166093945503235,
      "rewards/margins": 0.06544653326272964,
      "rewards/rejected": -0.2271074801683426,
      "step": 590
    },
    {
      "epoch": 0.9314420803782506,
      "grad_norm": 0.20009207725524902,
      "learning_rate": 2.5876340367875706e-06,
      "log_odds_chosen": 0.9041686058044434,
      "log_odds_ratio": -0.34814539551734924,
      "logits/chosen": -0.27487215399742126,
      "logits/rejected": -1.0577666759490967,
      "logps/chosen": -1.562727928161621,
      "logps/rejected": -2.3339860439300537,
      "loss": 1.6169,
      "nll_loss": 1.5820958614349365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1562727987766266,
      "rewards/margins": 0.07712581753730774,
      "rewards/rejected": -0.23339863121509552,
      "step": 591
    },
    {
      "epoch": 0.9330181245074862,
      "grad_norm": 0.18890726566314697,
      "learning_rate": 2.582377356484853e-06,
      "log_odds_chosen": 0.6838082075119019,
      "log_odds_ratio": -0.4133846163749695,
      "logits/chosen": -0.2822187840938568,
      "logits/rejected": -0.9121863842010498,
      "logps/chosen": -1.458941102027893,
      "logps/rejected": -2.0181310176849365,
      "loss": 1.5256,
      "nll_loss": 1.4842967987060547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1458941102027893,
      "rewards/margins": 0.055918991565704346,
      "rewards/rejected": -0.20181310176849365,
      "step": 592
    },
    {
      "epoch": 0.9345941686367218,
      "grad_norm": 0.19878967106342316,
      "learning_rate": 2.577116276599373e-06,
      "log_odds_chosen": 0.6385331749916077,
      "log_odds_ratio": -0.4308171570301056,
      "logits/chosen": -0.2593928575515747,
      "logits/rejected": -0.7261518239974976,
      "logps/chosen": -1.4897356033325195,
      "logps/rejected": -2.014805555343628,
      "loss": 1.5694,
      "nll_loss": 1.526312232017517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14897355437278748,
      "rewards/margins": 0.052506983280181885,
      "rewards/rejected": -0.20148055255413055,
      "step": 593
    },
    {
      "epoch": 0.9361702127659575,
      "grad_norm": 0.20385335385799408,
      "learning_rate": 2.5718508368760737e-06,
      "log_odds_chosen": 0.6622017621994019,
      "log_odds_ratio": -0.43777352571487427,
      "logits/chosen": -0.28589704632759094,
      "logits/rejected": -0.9581348896026611,
      "logps/chosen": -1.4968966245651245,
      "logps/rejected": -2.0486605167388916,
      "loss": 1.5665,
      "nll_loss": 1.5227015018463135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1496896892786026,
      "rewards/margins": 0.05517636612057686,
      "rewards/rejected": -0.20486603677272797,
      "step": 594
    },
    {
      "epoch": 0.9377462568951931,
      "grad_norm": 0.19222359359264374,
      "learning_rate": 2.5665810770928386e-06,
      "log_odds_chosen": 0.810360312461853,
      "log_odds_ratio": -0.3729667663574219,
      "logits/chosen": -0.3185596466064453,
      "logits/rejected": -1.0900532007217407,
      "logps/chosen": -1.4856997728347778,
      "logps/rejected": -2.1604537963867188,
      "loss": 1.5603,
      "nll_loss": 1.523039698600769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1485699564218521,
      "rewards/margins": 0.06747540831565857,
      "rewards/rejected": -0.21604537963867188,
      "step": 595
    },
    {
      "epoch": 0.9393223010244287,
      "grad_norm": 0.19499744474887848,
      "learning_rate": 2.5613070370601863e-06,
      "log_odds_chosen": 0.8030418157577515,
      "log_odds_ratio": -0.3729785084724426,
      "logits/chosen": -0.24502594769001007,
      "logits/rejected": -0.9063122868537903,
      "logps/chosen": -1.5408481359481812,
      "logps/rejected": -2.2181851863861084,
      "loss": 1.5835,
      "nll_loss": 1.5461995601654053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15408481657505035,
      "rewards/margins": 0.06773370504379272,
      "rewards/rejected": -0.22181853652000427,
      "step": 596
    },
    {
      "epoch": 0.9408983451536643,
      "grad_norm": 0.20074626803398132,
      "learning_rate": 2.556028756620969e-06,
      "log_odds_chosen": 0.660226047039032,
      "log_odds_ratio": -0.42320913076400757,
      "logits/chosen": -0.25374382734298706,
      "logits/rejected": -1.059732437133789,
      "logps/chosen": -1.5710662603378296,
      "logps/rejected": -2.1250038146972656,
      "loss": 1.6104,
      "nll_loss": 1.5680400133132935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15710663795471191,
      "rewards/margins": 0.0553937628865242,
      "rewards/rejected": -0.21250039339065552,
      "step": 597
    },
    {
      "epoch": 0.9424743892829,
      "grad_norm": 0.20810863375663757,
      "learning_rate": 2.5507462756500747e-06,
      "log_odds_chosen": 0.8217098116874695,
      "log_odds_ratio": -0.379741907119751,
      "logits/chosen": -0.12592369318008423,
      "logits/rejected": -0.8605210185050964,
      "logps/chosen": -1.5777266025543213,
      "logps/rejected": -2.2759249210357666,
      "loss": 1.623,
      "nll_loss": 1.5849889516830444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15777264535427094,
      "rewards/margins": 0.06981983780860901,
      "rewards/rejected": -0.22759249806404114,
      "step": 598
    },
    {
      "epoch": 0.9440504334121356,
      "grad_norm": 0.22460423409938812,
      "learning_rate": 2.5454596340541245e-06,
      "log_odds_chosen": 0.716874897480011,
      "log_odds_ratio": -0.4105355143547058,
      "logits/chosen": -0.3449729084968567,
      "logits/rejected": -0.9790999293327332,
      "logps/chosen": -1.6166168451309204,
      "logps/rejected": -2.2282192707061768,
      "loss": 1.6601,
      "nll_loss": 1.6190063953399658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16166169941425323,
      "rewards/margins": 0.061160244047641754,
      "rewards/rejected": -0.22282195091247559,
      "step": 599
    },
    {
      "epoch": 0.9456264775413712,
      "grad_norm": 0.20120826363563538,
      "learning_rate": 2.5401688717711702e-06,
      "log_odds_chosen": 0.6365572810173035,
      "log_odds_ratio": -0.4298497438430786,
      "logits/chosen": -0.23544873297214508,
      "logits/rejected": -0.9360511898994446,
      "logps/chosen": -1.5445044040679932,
      "logps/rejected": -2.0757927894592285,
      "loss": 1.6053,
      "nll_loss": 1.5622787475585938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544504314661026,
      "rewards/margins": 0.05312883108854294,
      "rewards/rejected": -0.20757928490638733,
      "step": 600
    },
    {
      "epoch": 0.9472025216706068,
      "grad_norm": 0.19064421951770782,
      "learning_rate": 2.5348740287703937e-06,
      "log_odds_chosen": 0.8292121887207031,
      "log_odds_ratio": -0.36574628949165344,
      "logits/chosen": -0.24441444873809814,
      "logits/rejected": -0.8810250759124756,
      "logps/chosen": -1.4425519704818726,
      "logps/rejected": -2.1281933784484863,
      "loss": 1.5143,
      "nll_loss": 1.4777559041976929,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14425520598888397,
      "rewards/margins": 0.06856414675712585,
      "rewards/rejected": -0.21281933784484863,
      "step": 601
    },
    {
      "epoch": 0.9487785657998424,
      "grad_norm": 0.2183743417263031,
      "learning_rate": 2.529575145051805e-06,
      "log_odds_chosen": 0.7649248838424683,
      "log_odds_ratio": -0.3955337405204773,
      "logits/chosen": -0.2556244432926178,
      "logits/rejected": -0.8571820259094238,
      "logps/chosen": -1.522550106048584,
      "logps/rejected": -2.165855884552002,
      "loss": 1.5987,
      "nll_loss": 1.5591237545013428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15225501358509064,
      "rewards/margins": 0.0643305629491806,
      "rewards/rejected": -0.21658559143543243,
      "step": 602
    },
    {
      "epoch": 0.950354609929078,
      "grad_norm": 0.2045803964138031,
      "learning_rate": 2.52427226064594e-06,
      "log_odds_chosen": 0.8493075370788574,
      "log_odds_ratio": -0.3637474775314331,
      "logits/chosen": -0.34642696380615234,
      "logits/rejected": -0.9698687791824341,
      "logps/chosen": -1.5536282062530518,
      "logps/rejected": -2.2717180252075195,
      "loss": 1.5894,
      "nll_loss": 1.5530749559402466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553628146648407,
      "rewards/margins": 0.07180898636579514,
      "rewards/rejected": -0.22717180848121643,
      "step": 603
    },
    {
      "epoch": 0.9519306540583137,
      "grad_norm": 0.18781255185604095,
      "learning_rate": 2.518965415613557e-06,
      "log_odds_chosen": 0.966554582118988,
      "log_odds_ratio": -0.3394841253757477,
      "logits/chosen": -0.2721576392650604,
      "logits/rejected": -0.9227450489997864,
      "logps/chosen": -1.451041340827942,
      "logps/rejected": -2.260777711868286,
      "loss": 1.5167,
      "nll_loss": 1.4827251434326172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14510415494441986,
      "rewards/margins": 0.08097360283136368,
      "rewards/rejected": -0.22607775032520294,
      "step": 604
    },
    {
      "epoch": 0.9535066981875493,
      "grad_norm": 0.2103288471698761,
      "learning_rate": 2.513654650045336e-06,
      "log_odds_chosen": 0.7449389100074768,
      "log_odds_ratio": -0.392122745513916,
      "logits/chosen": -0.27897876501083374,
      "logits/rejected": -1.109695315361023,
      "logps/chosen": -1.481645941734314,
      "logps/rejected": -2.1000876426696777,
      "loss": 1.5365,
      "nll_loss": 1.4973164796829224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14816460013389587,
      "rewards/margins": 0.06184417009353638,
      "rewards/rejected": -0.21000875532627106,
      "step": 605
    },
    {
      "epoch": 0.9550827423167849,
      "grad_norm": 0.19499583542346954,
      "learning_rate": 2.508340004061574e-06,
      "log_odds_chosen": 0.8943713307380676,
      "log_odds_ratio": -0.35206788778305054,
      "logits/chosen": -0.2978722155094147,
      "logits/rejected": -0.7795621752738953,
      "logps/chosen": -1.4949980974197388,
      "logps/rejected": -2.246958017349243,
      "loss": 1.5555,
      "nll_loss": 1.5202827453613281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1494998186826706,
      "rewards/margins": 0.07519599795341492,
      "rewards/rejected": -0.22469580173492432,
      "step": 606
    },
    {
      "epoch": 0.9566587864460205,
      "grad_norm": 0.2042934149503708,
      "learning_rate": 2.503021517811882e-06,
      "log_odds_chosen": 0.8187891244888306,
      "log_odds_ratio": -0.3694564998149872,
      "logits/chosen": -0.34403425455093384,
      "logits/rejected": -1.0705429315567017,
      "logps/chosen": -1.5671286582946777,
      "logps/rejected": -2.2610132694244385,
      "loss": 1.622,
      "nll_loss": 1.5850613117218018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1567128598690033,
      "rewards/margins": 0.0693884789943695,
      "rewards/rejected": -0.2261013388633728,
      "step": 607
    },
    {
      "epoch": 0.9582348305752562,
      "grad_norm": 0.194716677069664,
      "learning_rate": 2.497699231474885e-06,
      "log_odds_chosen": 0.9213505387306213,
      "log_odds_ratio": -0.35156625509262085,
      "logits/chosen": -0.31200534105300903,
      "logits/rejected": -1.0640758275985718,
      "logps/chosen": -1.45862877368927,
      "logps/rejected": -2.238149881362915,
      "loss": 1.5054,
      "nll_loss": 1.4702261686325073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1458628922700882,
      "rewards/margins": 0.07795208692550659,
      "rewards/rejected": -0.2238149791955948,
      "step": 608
    },
    {
      "epoch": 0.9598108747044918,
      "grad_norm": 0.20470517873764038,
      "learning_rate": 2.4923731852579127e-06,
      "log_odds_chosen": 0.8817616105079651,
      "log_odds_ratio": -0.3616253733634949,
      "logits/chosen": -0.31070253252983093,
      "logits/rejected": -0.767755389213562,
      "logps/chosen": -1.4946589469909668,
      "logps/rejected": -2.2430834770202637,
      "loss": 1.552,
      "nll_loss": 1.5158417224884033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1494658887386322,
      "rewards/margins": 0.0748424306511879,
      "rewards/rejected": -0.2243083268404007,
      "step": 609
    },
    {
      "epoch": 0.9613869188337274,
      "grad_norm": 0.18764522671699524,
      "learning_rate": 2.4870434193967017e-06,
      "log_odds_chosen": 0.9560897946357727,
      "log_odds_ratio": -0.3333515524864197,
      "logits/chosen": -0.2520322799682617,
      "logits/rejected": -1.0898277759552002,
      "logps/chosen": -1.4886752367019653,
      "logps/rejected": -2.293381452560425,
      "loss": 1.5254,
      "nll_loss": 1.492051362991333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14886753261089325,
      "rewards/margins": 0.08047063648700714,
      "rewards/rejected": -0.2293381541967392,
      "step": 610
    },
    {
      "epoch": 0.9629629629629629,
      "grad_norm": 0.2105409801006317,
      "learning_rate": 2.481709974155086e-06,
      "log_odds_chosen": 1.0217981338500977,
      "log_odds_ratio": -0.32593169808387756,
      "logits/chosen": -0.22246496379375458,
      "logits/rejected": -0.8673331141471863,
      "logps/chosen": -1.5121766328811646,
      "logps/rejected": -2.382678270339966,
      "loss": 1.5556,
      "nll_loss": 1.5230402946472168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15121768414974213,
      "rewards/margins": 0.08705013990402222,
      "rewards/rejected": -0.23826780915260315,
      "step": 611
    },
    {
      "epoch": 0.9645390070921985,
      "grad_norm": 0.21617871522903442,
      "learning_rate": 2.4763728898246983e-06,
      "log_odds_chosen": 1.0673617124557495,
      "log_odds_ratio": -0.30890581011772156,
      "logits/chosen": -0.10703454911708832,
      "logits/rejected": -0.9467811584472656,
      "logps/chosen": -1.3957353830337524,
      "logps/rejected": -2.2724063396453857,
      "loss": 1.4345,
      "nll_loss": 1.4036459922790527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13957354426383972,
      "rewards/margins": 0.0876670628786087,
      "rewards/rejected": -0.22724059224128723,
      "step": 612
    },
    {
      "epoch": 0.9661150512214342,
      "grad_norm": 0.17770403623580933,
      "learning_rate": 2.4710322067246607e-06,
      "log_odds_chosen": 0.8337207436561584,
      "log_odds_ratio": -0.3655989468097687,
      "logits/chosen": -0.2130187749862671,
      "logits/rejected": -0.9335604310035706,
      "logps/chosen": -1.52428138256073,
      "logps/rejected": -2.2232112884521484,
      "loss": 1.5607,
      "nll_loss": 1.524143099784851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15242813527584076,
      "rewards/margins": 0.06989297270774841,
      "rewards/rejected": -0.22232110798358917,
      "step": 613
    },
    {
      "epoch": 0.9676910953506698,
      "grad_norm": 0.1831359565258026,
      "learning_rate": 2.465687965201283e-06,
      "log_odds_chosen": 0.9036546945571899,
      "log_odds_ratio": -0.3479987382888794,
      "logits/chosen": -0.25526103377342224,
      "logits/rejected": -0.9284212589263916,
      "logps/chosen": -1.4246617555618286,
      "logps/rejected": -2.175575017929077,
      "loss": 1.4904,
      "nll_loss": 1.4555717706680298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14246618747711182,
      "rewards/margins": 0.07509131729602814,
      "rewards/rejected": -0.21755750477313995,
      "step": 614
    },
    {
      "epoch": 0.9692671394799054,
      "grad_norm": 0.19739262759685516,
      "learning_rate": 2.4603402056277577e-06,
      "log_odds_chosen": 0.8270947337150574,
      "log_odds_ratio": -0.370461642742157,
      "logits/chosen": -0.3928753733634949,
      "logits/rejected": -0.9402602910995483,
      "logps/chosen": -1.4891471862792969,
      "logps/rejected": -2.178040027618408,
      "loss": 1.5517,
      "nll_loss": 1.5146777629852295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14891472458839417,
      "rewards/margins": 0.06888928264379501,
      "rewards/rejected": -0.21780399978160858,
      "step": 615
    },
    {
      "epoch": 0.970843183609141,
      "grad_norm": 0.21897821128368378,
      "learning_rate": 2.454988968403854e-06,
      "log_odds_chosen": 0.6489979028701782,
      "log_odds_ratio": -0.42528313398361206,
      "logits/chosen": -0.21522563695907593,
      "logits/rejected": -0.9565899968147278,
      "logps/chosen": -1.5508010387420654,
      "logps/rejected": -2.0939066410064697,
      "loss": 1.589,
      "nll_loss": 1.5464531183242798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15508009493350983,
      "rewards/margins": 0.05431056767702103,
      "rewards/rejected": -0.20939067006111145,
      "step": 616
    },
    {
      "epoch": 0.9724192277383766,
      "grad_norm": 0.19598649442195892,
      "learning_rate": 2.4496342939556133e-06,
      "log_odds_chosen": 0.84560626745224,
      "log_odds_ratio": -0.3649125099182129,
      "logits/chosen": -0.1577945500612259,
      "logits/rejected": -0.8966969847679138,
      "logps/chosen": -1.500002384185791,
      "logps/rejected": -2.208306312561035,
      "loss": 1.5554,
      "nll_loss": 1.5189388990402222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15000022947788239,
      "rewards/margins": 0.07083040475845337,
      "rewards/rejected": -0.22083064913749695,
      "step": 617
    },
    {
      "epoch": 0.9739952718676123,
      "grad_norm": 0.20053473114967346,
      "learning_rate": 2.444276222735043e-06,
      "log_odds_chosen": 0.8209749460220337,
      "log_odds_ratio": -0.3673669099807739,
      "logits/chosen": -0.3390085697174072,
      "logits/rejected": -1.0487425327301025,
      "logps/chosen": -1.463447093963623,
      "logps/rejected": -2.1447691917419434,
      "loss": 1.5398,
      "nll_loss": 1.5030690431594849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14634472131729126,
      "rewards/margins": 0.0681321993470192,
      "rewards/rejected": -0.21447692811489105,
      "step": 618
    },
    {
      "epoch": 0.9755713159968479,
      "grad_norm": 0.2156509906053543,
      "learning_rate": 2.4389147952198127e-06,
      "log_odds_chosen": 0.8275145292282104,
      "log_odds_ratio": -0.36603498458862305,
      "logits/chosen": -0.35737529397010803,
      "logits/rejected": -0.9917409420013428,
      "logps/chosen": -1.5070266723632812,
      "logps/rejected": -2.199937582015991,
      "loss": 1.5399,
      "nll_loss": 1.5032652616500854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15070267021656036,
      "rewards/margins": 0.06929107755422592,
      "rewards/rejected": -0.21999377012252808,
      "step": 619
    },
    {
      "epoch": 0.9771473601260835,
      "grad_norm": 0.1865287870168686,
      "learning_rate": 2.433550051912946e-06,
      "log_odds_chosen": 0.9182285070419312,
      "log_odds_ratio": -0.34175238013267517,
      "logits/chosen": -0.28525999188423157,
      "logits/rejected": -1.043125033378601,
      "logps/chosen": -1.395874261856079,
      "logps/rejected": -2.1492919921875,
      "loss": 1.4507,
      "nll_loss": 1.4165234565734863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13958744704723358,
      "rewards/margins": 0.07534177601337433,
      "rewards/rejected": -0.21492920815944672,
      "step": 620
    },
    {
      "epoch": 0.9787234042553191,
      "grad_norm": 0.2098291963338852,
      "learning_rate": 2.4281820333425167e-06,
      "log_odds_chosen": 0.8603261709213257,
      "log_odds_ratio": -0.3645764887332916,
      "logits/chosen": -0.27089592814445496,
      "logits/rejected": -0.8807480335235596,
      "logps/chosen": -1.6115281581878662,
      "logps/rejected": -2.3515753746032715,
      "loss": 1.6413,
      "nll_loss": 1.6048469543457031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16115280985832214,
      "rewards/margins": 0.07400472462177277,
      "rewards/rejected": -0.2351575493812561,
      "step": 621
    },
    {
      "epoch": 0.9802994483845547,
      "grad_norm": 0.20614829659461975,
      "learning_rate": 2.42281078006134e-06,
      "log_odds_chosen": 1.07462739944458,
      "log_odds_ratio": -0.2980968952178955,
      "logits/chosen": -0.30630022287368774,
      "logits/rejected": -1.14458167552948,
      "logps/chosen": -1.598561406135559,
      "logps/rejected": -2.526106357574463,
      "loss": 1.6359,
      "nll_loss": 1.6060421466827393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1598561406135559,
      "rewards/margins": 0.09275451302528381,
      "rewards/rejected": -0.2526106536388397,
      "step": 622
    },
    {
      "epoch": 0.9818754925137904,
      "grad_norm": 0.2179221510887146,
      "learning_rate": 2.4174363326466703e-06,
      "log_odds_chosen": 0.8943939805030823,
      "log_odds_ratio": -0.36915120482444763,
      "logits/chosen": -0.30196717381477356,
      "logits/rejected": -1.0717780590057373,
      "logps/chosen": -1.5215718746185303,
      "logps/rejected": -2.2911746501922607,
      "loss": 1.5694,
      "nll_loss": 1.532444953918457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15215721726417542,
      "rewards/margins": 0.0769602507352829,
      "rewards/rejected": -0.22911745309829712,
      "step": 623
    },
    {
      "epoch": 0.983451536643026,
      "grad_norm": 0.19984939694404602,
      "learning_rate": 2.41205873169989e-06,
      "log_odds_chosen": 0.7326567769050598,
      "log_odds_ratio": -0.40043237805366516,
      "logits/chosen": -0.1846848428249359,
      "logits/rejected": -0.7317571043968201,
      "logps/chosen": -1.5202010869979858,
      "logps/rejected": -2.1315598487854004,
      "loss": 1.5763,
      "nll_loss": 1.536268949508667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15202011168003082,
      "rewards/margins": 0.061135854572057724,
      "rewards/rejected": -0.21315596997737885,
      "step": 624
    },
    {
      "epoch": 0.9850275807722616,
      "grad_norm": 0.19462138414382935,
      "learning_rate": 2.4066780178462058e-06,
      "log_odds_chosen": 0.9225372076034546,
      "log_odds_ratio": -0.34987640380859375,
      "logits/chosen": -0.17803911864757538,
      "logits/rejected": -0.8712078928947449,
      "logps/chosen": -1.5460710525512695,
      "logps/rejected": -2.335981845855713,
      "loss": 1.5874,
      "nll_loss": 1.5524508953094482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1546071171760559,
      "rewards/margins": 0.07899108529090881,
      "rewards/rejected": -0.23359820246696472,
      "step": 625
    },
    {
      "epoch": 0.9866036249014972,
      "grad_norm": 0.41452720761299133,
      "learning_rate": 2.40129423173434e-06,
      "log_odds_chosen": 0.8286516070365906,
      "log_odds_ratio": -0.371588796377182,
      "logits/chosen": -0.24122776091098785,
      "logits/rejected": -1.0151088237762451,
      "logps/chosen": -1.4925408363342285,
      "logps/rejected": -2.1837525367736816,
      "loss": 1.5558,
      "nll_loss": 1.5186827182769775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14925408363342285,
      "rewards/margins": 0.06912118196487427,
      "rewards/rejected": -0.2183752804994583,
      "step": 626
    },
    {
      "epoch": 0.9881796690307328,
      "grad_norm": 0.21910816431045532,
      "learning_rate": 2.3959074140362274e-06,
      "log_odds_chosen": 0.8413971662521362,
      "log_odds_ratio": -0.36324429512023926,
      "logits/chosen": -0.3605331778526306,
      "logits/rejected": -0.9486165642738342,
      "logps/chosen": -1.4744913578033447,
      "logps/rejected": -2.1747500896453857,
      "loss": 1.5232,
      "nll_loss": 1.4869177341461182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14744913578033447,
      "rewards/margins": 0.07002587616443634,
      "rewards/rejected": -0.2174750119447708,
      "step": 627
    },
    {
      "epoch": 0.9897557131599685,
      "grad_norm": 0.22606877982616425,
      "learning_rate": 2.3905176054467007e-06,
      "log_odds_chosen": 1.0594252347946167,
      "log_odds_ratio": -0.3114800453186035,
      "logits/chosen": -0.33363643288612366,
      "logits/rejected": -0.9470556974411011,
      "logps/chosen": -1.4619425535202026,
      "logps/rejected": -2.361726999282837,
      "loss": 1.5208,
      "nll_loss": 1.4896818399429321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461942493915558,
      "rewards/margins": 0.08997844904661179,
      "rewards/rejected": -0.23617270588874817,
      "step": 628
    },
    {
      "epoch": 0.9913317572892041,
      "grad_norm": 0.21481728553771973,
      "learning_rate": 2.3851248466831905e-06,
      "log_odds_chosen": 0.8987281322479248,
      "log_odds_ratio": -0.361965537071228,
      "logits/chosen": -0.20962606370449066,
      "logits/rejected": -0.835357129573822,
      "logps/chosen": -1.4602702856063843,
      "logps/rejected": -2.2177577018737793,
      "loss": 1.5019,
      "nll_loss": 1.4656609296798706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14602702856063843,
      "rewards/margins": 0.0757487490773201,
      "rewards/rejected": -0.22177578508853912,
      "step": 629
    },
    {
      "epoch": 0.9929078014184397,
      "grad_norm": 0.19942425191402435,
      "learning_rate": 2.379729178485412e-06,
      "log_odds_chosen": 0.8199061751365662,
      "log_odds_ratio": -0.3686443567276001,
      "logits/chosen": -0.29476356506347656,
      "logits/rejected": -1.058532476425171,
      "logps/chosen": -1.5569266080856323,
      "logps/rejected": -2.2516047954559326,
      "loss": 1.6036,
      "nll_loss": 1.5667665004730225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1556926667690277,
      "rewards/margins": 0.06946781277656555,
      "rewards/rejected": -0.22516046464443207,
      "step": 630
    },
    {
      "epoch": 0.9944838455476753,
      "grad_norm": 0.23968850076198578,
      "learning_rate": 2.3743306416150636e-06,
      "log_odds_chosen": 0.8157881498336792,
      "log_odds_ratio": -0.3680221736431122,
      "logits/chosen": -0.21403929591178894,
      "logits/rejected": -0.9583814144134521,
      "logps/chosen": -1.5242538452148438,
      "logps/rejected": -2.2058591842651367,
      "loss": 1.5722,
      "nll_loss": 1.5354090929031372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1524253934621811,
      "rewards/margins": 0.06816056370735168,
      "rewards/rejected": -0.22058594226837158,
      "step": 631
    },
    {
      "epoch": 0.996059889676911,
      "grad_norm": 0.1970617175102234,
      "learning_rate": 2.368929276855512e-06,
      "log_odds_chosen": 0.6617365479469299,
      "log_odds_ratio": -0.4178897738456726,
      "logits/chosen": -0.2291412055492401,
      "logits/rejected": -0.91761714220047,
      "logps/chosen": -1.625699758529663,
      "logps/rejected": -2.187096357345581,
      "loss": 1.6498,
      "nll_loss": 1.6079752445220947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16256996989250183,
      "rewards/margins": 0.056139662861824036,
      "rewards/rejected": -0.21870963275432587,
      "step": 632
    },
    {
      "epoch": 0.9976359338061466,
      "grad_norm": 0.21298566460609436,
      "learning_rate": 2.363525125011487e-06,
      "log_odds_chosen": 0.8577976226806641,
      "log_odds_ratio": -0.3713403046131134,
      "logits/chosen": -0.2584449350833893,
      "logits/rejected": -0.9506880044937134,
      "logps/chosen": -1.6527642011642456,
      "logps/rejected": -2.397233009338379,
      "loss": 1.6968,
      "nll_loss": 1.6596896648406982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.165276437997818,
      "rewards/margins": 0.07444687932729721,
      "rewards/rejected": -0.2397232949733734,
      "step": 633
    },
    {
      "epoch": 0.9992119779353822,
      "grad_norm": 0.19464780390262604,
      "learning_rate": 2.3581182269087755e-06,
      "log_odds_chosen": 1.2316921949386597,
      "log_odds_ratio": -0.3090118169784546,
      "logits/chosen": -0.24409595131874084,
      "logits/rejected": -1.0376189947128296,
      "logps/chosen": -1.4501967430114746,
      "logps/rejected": -2.4870381355285645,
      "loss": 1.5055,
      "nll_loss": 1.474576711654663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14501968026161194,
      "rewards/margins": 0.10368414223194122,
      "rewards/rejected": -0.24870382249355316,
      "step": 634
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.2554992735385895,
      "learning_rate": 2.3527086233939097e-06,
      "log_odds_chosen": 0.9453415870666504,
      "log_odds_ratio": -0.33290183544158936,
      "logits/chosen": -0.2778136134147644,
      "logits/rejected": -1.237735390663147,
      "logps/chosen": -1.5283219814300537,
      "logps/rejected": -2.3322157859802246,
      "loss": 1.5811,
      "nll_loss": 1.5478403568267822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15283219516277313,
      "rewards/margins": 0.08038939535617828,
      "rewards/rejected": -0.2332216054201126,
      "step": 635
    },
    {
      "epoch": 1.0015760441292356,
      "grad_norm": 0.23139451444149017,
      "learning_rate": 2.347296355333861e-06,
      "log_odds_chosen": 0.8247041702270508,
      "log_odds_ratio": -0.3687278628349304,
      "logits/chosen": -0.3295940160751343,
      "logits/rejected": -0.8597362637519836,
      "logps/chosen": -1.5452543497085571,
      "logps/rejected": -2.2433595657348633,
      "loss": 1.5931,
      "nll_loss": 1.5561856031417847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15452544391155243,
      "rewards/margins": 0.06981053203344345,
      "rewards/rejected": -0.2243359535932541,
      "step": 636
    },
    {
      "epoch": 1.0031520882584712,
      "grad_norm": 0.20446458458900452,
      "learning_rate": 2.3418814636157283e-06,
      "log_odds_chosen": 0.7290414571762085,
      "log_odds_ratio": -0.40019094944000244,
      "logits/chosen": -0.2882698178291321,
      "logits/rejected": -1.0735361576080322,
      "logps/chosen": -1.581707239151001,
      "logps/rejected": -2.195343494415283,
      "loss": 1.619,
      "nll_loss": 1.5790104866027832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15817072987556458,
      "rewards/margins": 0.061363644897937775,
      "rewards/rejected": -0.21953438222408295,
      "step": 637
    },
    {
      "epoch": 1.0047281323877069,
      "grad_norm": 0.5818917751312256,
      "learning_rate": 2.336463989146434e-06,
      "log_odds_chosen": 0.8472689986228943,
      "log_odds_ratio": -0.36595940589904785,
      "logits/chosen": -0.2873977720737457,
      "logits/rejected": -0.8676982522010803,
      "logps/chosen": -1.4776124954223633,
      "logps/rejected": -2.1879258155822754,
      "loss": 1.5217,
      "nll_loss": 1.4851521253585815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477612406015396,
      "rewards/margins": 0.07103133946657181,
      "rewards/rejected": -0.2187926024198532,
      "step": 638
    },
    {
      "epoch": 1.0063041765169425,
      "grad_norm": 0.20081207156181335,
      "learning_rate": 2.3310439728524074e-06,
      "log_odds_chosen": 0.9249904155731201,
      "log_odds_ratio": -0.33683958649635315,
      "logits/chosen": -0.3330644965171814,
      "logits/rejected": -0.8969684839248657,
      "logps/chosen": -1.498591423034668,
      "logps/rejected": -2.2777352333068848,
      "loss": 1.5528,
      "nll_loss": 1.519068956375122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14985914528369904,
      "rewards/margins": 0.07791437953710556,
      "rewards/rejected": -0.2277735322713852,
      "step": 639
    },
    {
      "epoch": 1.007880220646178,
      "grad_norm": 0.20605570077896118,
      "learning_rate": 2.325621455679286e-06,
      "log_odds_chosen": 0.9802895784378052,
      "log_odds_ratio": -0.32932159304618835,
      "logits/chosen": -0.22372624278068542,
      "logits/rejected": -0.9513578414916992,
      "logps/chosen": -1.4824607372283936,
      "logps/rejected": -2.310763120651245,
      "loss": 1.5242,
      "nll_loss": 1.4912447929382324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14824607968330383,
      "rewards/margins": 0.08283022791147232,
      "rewards/rejected": -0.23107630014419556,
      "step": 640
    },
    {
      "epoch": 1.0094562647754137,
      "grad_norm": 0.19742731750011444,
      "learning_rate": 2.3201964785915953e-06,
      "log_odds_chosen": 0.7533868551254272,
      "log_odds_ratio": -0.38784900307655334,
      "logits/chosen": -0.3235701322555542,
      "logits/rejected": -0.8257976770401001,
      "logps/chosen": -1.5435073375701904,
      "logps/rejected": -2.1762735843658447,
      "loss": 1.5905,
      "nll_loss": 1.5516694784164429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15435072779655457,
      "rewards/margins": 0.06327663362026215,
      "rewards/rejected": -0.2176273614168167,
      "step": 641
    },
    {
      "epoch": 1.0110323089046493,
      "grad_norm": 0.20521683990955353,
      "learning_rate": 2.3147690825724457e-06,
      "log_odds_chosen": 0.6707542538642883,
      "log_odds_ratio": -0.4190235733985901,
      "logits/chosen": -0.27176862955093384,
      "logits/rejected": -0.9075352549552917,
      "logps/chosen": -1.588667631149292,
      "logps/rejected": -2.1577951908111572,
      "loss": 1.63,
      "nll_loss": 1.5880699157714844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1588667631149292,
      "rewards/margins": 0.056912779808044434,
      "rewards/rejected": -0.21577952802181244,
      "step": 642
    },
    {
      "epoch": 1.012608353033885,
      "grad_norm": 0.20128387212753296,
      "learning_rate": 2.3093393086232226e-06,
      "log_odds_chosen": 0.9791814684867859,
      "log_odds_ratio": -0.3240292966365814,
      "logits/chosen": -0.22348952293395996,
      "logits/rejected": -0.8576082587242126,
      "logps/chosen": -1.4472434520721436,
      "logps/rejected": -2.267815351486206,
      "loss": 1.505,
      "nll_loss": 1.4725701808929443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14472433924674988,
      "rewards/margins": 0.08205720037221909,
      "rewards/rejected": -0.22678154706954956,
      "step": 643
    },
    {
      "epoch": 1.0141843971631206,
      "grad_norm": 0.19670970737934113,
      "learning_rate": 2.3039071977632748e-06,
      "log_odds_chosen": 0.9264999032020569,
      "log_odds_ratio": -0.34527403116226196,
      "logits/chosen": -0.3282894492149353,
      "logits/rejected": -1.1630949974060059,
      "logps/chosen": -1.5073062181472778,
      "logps/rejected": -2.295166492462158,
      "loss": 1.5483,
      "nll_loss": 1.5137276649475098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15073063969612122,
      "rewards/margins": 0.07878601551055908,
      "rewards/rejected": -0.2295166552066803,
      "step": 644
    },
    {
      "epoch": 1.0157604412923562,
      "grad_norm": 0.19094307720661163,
      "learning_rate": 2.2984727910296044e-06,
      "log_odds_chosen": 0.7740219831466675,
      "log_odds_ratio": -0.38529035449028015,
      "logits/chosen": -0.332872599363327,
      "logits/rejected": -1.006569266319275,
      "logps/chosen": -1.565941572189331,
      "logps/rejected": -2.2239296436309814,
      "loss": 1.6248,
      "nll_loss": 1.5862404108047485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1565941572189331,
      "rewards/margins": 0.0657988116145134,
      "rewards/rejected": -0.2223929613828659,
      "step": 645
    },
    {
      "epoch": 1.0173364854215918,
      "grad_norm": 0.2220807820558548,
      "learning_rate": 2.2930361294765594e-06,
      "log_odds_chosen": 0.8859462141990662,
      "log_odds_ratio": -0.3504306972026825,
      "logits/chosen": -0.2045348733663559,
      "logits/rejected": -1.0126723051071167,
      "logps/chosen": -1.577898383140564,
      "logps/rejected": -2.335003137588501,
      "loss": 1.6343,
      "nll_loss": 1.5992114543914795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15778982639312744,
      "rewards/margins": 0.0757104754447937,
      "rewards/rejected": -0.23350030183792114,
      "step": 646
    },
    {
      "epoch": 1.0189125295508275,
      "grad_norm": 0.19843855500221252,
      "learning_rate": 2.287597254175521e-06,
      "log_odds_chosen": 1.015138864517212,
      "log_odds_ratio": -0.33159783482551575,
      "logits/chosen": -0.2682275176048279,
      "logits/rejected": -0.9703023433685303,
      "logps/chosen": -1.4326711893081665,
      "logps/rejected": -2.2867417335510254,
      "loss": 1.5033,
      "nll_loss": 1.4700952768325806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14326712489128113,
      "rewards/margins": 0.0854070782661438,
      "rewards/rejected": -0.22867418825626373,
      "step": 647
    },
    {
      "epoch": 1.020488573680063,
      "grad_norm": 0.19806131720542908,
      "learning_rate": 2.2821562062145938e-06,
      "log_odds_chosen": 1.052594542503357,
      "log_odds_ratio": -0.318314790725708,
      "logits/chosen": -0.2569073736667633,
      "logits/rejected": -1.0673884153366089,
      "logps/chosen": -1.5933895111083984,
      "logps/rejected": -2.5108981132507324,
      "loss": 1.6119,
      "nll_loss": 1.5801138877868652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15933895111083984,
      "rewards/margins": 0.09175088256597519,
      "rewards/rejected": -0.25108984112739563,
      "step": 648
    },
    {
      "epoch": 1.0220646178092987,
      "grad_norm": 0.2218894362449646,
      "learning_rate": 2.2767130266982967e-06,
      "log_odds_chosen": 0.6992781162261963,
      "log_odds_ratio": -0.4133135676383972,
      "logits/chosen": -0.31421294808387756,
      "logits/rejected": -0.8303437829017639,
      "logps/chosen": -1.5169445276260376,
      "logps/rejected": -2.1018807888031006,
      "loss": 1.568,
      "nll_loss": 1.5266555547714233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15169446170330048,
      "rewards/margins": 0.05849362164735794,
      "rewards/rejected": -0.210188090801239,
      "step": 649
    },
    {
      "epoch": 1.0236406619385343,
      "grad_norm": 0.19644376635551453,
      "learning_rate": 2.271267756747251e-06,
      "log_odds_chosen": 0.8789330720901489,
      "log_odds_ratio": -0.3566223978996277,
      "logits/chosen": -0.23688815534114838,
      "logits/rejected": -0.9716790914535522,
      "logps/chosen": -1.4732593297958374,
      "logps/rejected": -2.2074530124664307,
      "loss": 1.5125,
      "nll_loss": 1.4767979383468628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14732591807842255,
      "rewards/margins": 0.07341939210891724,
      "rewards/rejected": -0.22074531018733978,
      "step": 650
    },
    {
      "epoch": 1.02521670606777,
      "grad_norm": 0.18773488700389862,
      "learning_rate": 2.265820437497871e-06,
      "log_odds_chosen": 0.7397662997245789,
      "log_odds_ratio": -0.40189120173454285,
      "logits/chosen": -0.24222803115844727,
      "logits/rejected": -0.8547966480255127,
      "logps/chosen": -1.4620825052261353,
      "logps/rejected": -2.0761170387268066,
      "loss": 1.532,
      "nll_loss": 1.491844892501831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.146208256483078,
      "rewards/margins": 0.06140345335006714,
      "rewards/rejected": -0.20761169493198395,
      "step": 651
    },
    {
      "epoch": 1.0267927501970056,
      "grad_norm": 0.20862142741680145,
      "learning_rate": 2.26037111010205e-06,
      "log_odds_chosen": 0.7649354338645935,
      "log_odds_ratio": -0.38638901710510254,
      "logits/chosen": -0.25058722496032715,
      "logits/rejected": -0.9417374730110168,
      "logps/chosen": -1.4618744850158691,
      "logps/rejected": -2.0932345390319824,
      "loss": 1.5156,
      "nll_loss": 1.4769532680511475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461874544620514,
      "rewards/margins": 0.06313599646091461,
      "rewards/rejected": -0.209323450922966,
      "step": 652
    },
    {
      "epoch": 1.0283687943262412,
      "grad_norm": 0.20174525678157806,
      "learning_rate": 2.254919815726856e-06,
      "log_odds_chosen": 0.7451000809669495,
      "log_odds_ratio": -0.3972141742706299,
      "logits/chosen": -0.30902013182640076,
      "logits/rejected": -0.973990797996521,
      "logps/chosen": -1.5594556331634521,
      "logps/rejected": -2.1896677017211914,
      "loss": 1.6205,
      "nll_loss": 1.580798864364624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1559455692768097,
      "rewards/margins": 0.06302119791507721,
      "rewards/rejected": -0.2189667671918869,
      "step": 653
    },
    {
      "epoch": 1.0299448384554768,
      "grad_norm": 0.2052079141139984,
      "learning_rate": 2.2494665955542127e-06,
      "log_odds_chosen": 1.0554144382476807,
      "log_odds_ratio": -0.31652897596359253,
      "logits/chosen": -0.3758982717990875,
      "logits/rejected": -1.0485985279083252,
      "logps/chosen": -1.4716399908065796,
      "logps/rejected": -2.3626677989959717,
      "loss": 1.5185,
      "nll_loss": 1.4868149757385254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.147163987159729,
      "rewards/margins": 0.08910278230905533,
      "rewards/rejected": -0.23626677691936493,
      "step": 654
    },
    {
      "epoch": 1.0315208825847124,
      "grad_norm": 0.2286987155675888,
      "learning_rate": 2.2440114907805942e-06,
      "log_odds_chosen": 0.7839126586914062,
      "log_odds_ratio": -0.3824114203453064,
      "logits/chosen": -0.3396918773651123,
      "logits/rejected": -0.9772771000862122,
      "logps/chosen": -1.6120754480361938,
      "logps/rejected": -2.2784581184387207,
      "loss": 1.6363,
      "nll_loss": 1.598050832748413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16120754182338715,
      "rewards/margins": 0.06663825362920761,
      "rewards/rejected": -0.22784578800201416,
      "step": 655
    },
    {
      "epoch": 1.033096926713948,
      "grad_norm": 0.2109273076057434,
      "learning_rate": 2.2385545426167112e-06,
      "log_odds_chosen": 0.8177002668380737,
      "log_odds_ratio": -0.37445881962776184,
      "logits/chosen": -0.1846223920583725,
      "logits/rejected": -1.0761255025863647,
      "logps/chosen": -1.5889091491699219,
      "logps/rejected": -2.2808661460876465,
      "loss": 1.6395,
      "nll_loss": 1.6020262241363525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15889091789722443,
      "rewards/margins": 0.0691957026720047,
      "rewards/rejected": -0.22808662056922913,
      "step": 656
    },
    {
      "epoch": 1.0346729708431837,
      "grad_norm": 0.2063363790512085,
      "learning_rate": 2.2330957922872016e-06,
      "log_odds_chosen": 1.2240403890609741,
      "log_odds_ratio": -0.2735121250152588,
      "logits/chosen": -0.3476215898990631,
      "logits/rejected": -1.0773109197616577,
      "logps/chosen": -1.4977139234542847,
      "logps/rejected": -2.552717685699463,
      "loss": 1.5409,
      "nll_loss": 1.5135314464569092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14977139234542847,
      "rewards/margins": 0.10550037026405334,
      "rewards/rejected": -0.2552717626094818,
      "step": 657
    },
    {
      "epoch": 1.0362490149724193,
      "grad_norm": 0.19179211556911469,
      "learning_rate": 2.2276352810303166e-06,
      "log_odds_chosen": 1.0539443492889404,
      "log_odds_ratio": -0.3063228726387024,
      "logits/chosen": -0.27733543515205383,
      "logits/rejected": -1.0490702390670776,
      "logps/chosen": -1.4026516675949097,
      "logps/rejected": -2.2836122512817383,
      "loss": 1.4579,
      "nll_loss": 1.427275538444519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14026516675949097,
      "rewards/margins": 0.08809608966112137,
      "rewards/rejected": -0.22836124897003174,
      "step": 658
    },
    {
      "epoch": 1.037825059101655,
      "grad_norm": 0.20735010504722595,
      "learning_rate": 2.222173050097609e-06,
      "log_odds_chosen": 0.8399420976638794,
      "log_odds_ratio": -0.36545124650001526,
      "logits/chosen": -0.294746994972229,
      "logits/rejected": -1.0786052942276,
      "logps/chosen": -1.5219407081604004,
      "logps/rejected": -2.2292397022247314,
      "loss": 1.5749,
      "nll_loss": 1.5383909940719604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.152194082736969,
      "rewards/margins": 0.07072988897562027,
      "rewards/rejected": -0.22292396426200867,
      "step": 659
    },
    {
      "epoch": 1.0394011032308905,
      "grad_norm": 0.21655794978141785,
      "learning_rate": 2.2167091407536272e-06,
      "log_odds_chosen": 0.8468115329742432,
      "log_odds_ratio": -0.36164629459381104,
      "logits/chosen": -0.3030907213687897,
      "logits/rejected": -0.9068984985351562,
      "logps/chosen": -1.545032024383545,
      "logps/rejected": -2.258321762084961,
      "loss": 1.6033,
      "nll_loss": 1.5671807527542114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15450319647789001,
      "rewards/margins": 0.07132896780967712,
      "rewards/rejected": -0.22583216428756714,
      "step": 660
    },
    {
      "epoch": 1.0409771473601261,
      "grad_norm": 0.2175927758216858,
      "learning_rate": 2.211243594275594e-06,
      "log_odds_chosen": 0.7824351787567139,
      "log_odds_ratio": -0.3850528597831726,
      "logits/chosen": -0.296817421913147,
      "logits/rejected": -0.8058379292488098,
      "logps/chosen": -1.5780060291290283,
      "logps/rejected": -2.2393832206726074,
      "loss": 1.6033,
      "nll_loss": 1.5647971630096436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1578006148338318,
      "rewards/margins": 0.06613771617412567,
      "rewards/rejected": -0.22393831610679626,
      "step": 661
    },
    {
      "epoch": 1.0425531914893618,
      "grad_norm": 0.21112869679927826,
      "learning_rate": 2.2057764519531034e-06,
      "log_odds_chosen": 0.8649223446846008,
      "log_odds_ratio": -0.355747789144516,
      "logits/chosen": -0.2525985538959503,
      "logits/rejected": -0.961216151714325,
      "logps/chosen": -1.568381428718567,
      "logps/rejected": -2.303539752960205,
      "loss": 1.5945,
      "nll_loss": 1.5589532852172852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15683814883232117,
      "rewards/margins": 0.07351582497358322,
      "rewards/rejected": -0.2303539514541626,
      "step": 662
    },
    {
      "epoch": 1.0441292356185974,
      "grad_norm": 0.21360249817371368,
      "learning_rate": 2.2003077550878053e-06,
      "log_odds_chosen": 0.9827125072479248,
      "log_odds_ratio": -0.3207847774028778,
      "logits/chosen": -0.24310600757598877,
      "logits/rejected": -1.005952000617981,
      "logps/chosen": -1.5527591705322266,
      "logps/rejected": -2.3932976722717285,
      "loss": 1.5761,
      "nll_loss": 1.544018030166626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15527592599391937,
      "rewards/margins": 0.08405381441116333,
      "rewards/rejected": -0.2393297404050827,
      "step": 663
    },
    {
      "epoch": 1.045705279747833,
      "grad_norm": 0.21486958861351013,
      "learning_rate": 2.1948375449930915e-06,
      "log_odds_chosen": 0.8869554996490479,
      "log_odds_ratio": -0.3553463816642761,
      "logits/chosen": -0.25793159008026123,
      "logits/rejected": -0.9235754609107971,
      "logps/chosen": -1.587700605392456,
      "logps/rejected": -2.3497979640960693,
      "loss": 1.617,
      "nll_loss": 1.5814210176467896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15877006947994232,
      "rewards/margins": 0.07620973885059357,
      "rewards/rejected": -0.23497982323169708,
      "step": 664
    },
    {
      "epoch": 1.0472813238770686,
      "grad_norm": 0.2001623511314392,
      "learning_rate": 2.189365862993787e-06,
      "log_odds_chosen": 0.8695443868637085,
      "log_odds_ratio": -0.35637232661247253,
      "logits/chosen": -0.21846145391464233,
      "logits/rejected": -0.9603075981140137,
      "logps/chosen": -1.574625015258789,
      "logps/rejected": -2.3161492347717285,
      "loss": 1.6208,
      "nll_loss": 1.5851972103118896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15746250748634338,
      "rewards/margins": 0.07415242493152618,
      "rewards/rejected": -0.23161491751670837,
      "step": 665
    },
    {
      "epoch": 1.0488573680063042,
      "grad_norm": 0.21913884580135345,
      "learning_rate": 2.1838927504258354e-06,
      "log_odds_chosen": 0.871741771697998,
      "log_odds_ratio": -0.35455983877182007,
      "logits/chosen": -0.23453892767429352,
      "logits/rejected": -1.0257761478424072,
      "logps/chosen": -1.5272916555404663,
      "logps/rejected": -2.2643685340881348,
      "loss": 1.5659,
      "nll_loss": 1.5304679870605469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15272918343544006,
      "rewards/margins": 0.0737076848745346,
      "rewards/rejected": -0.22643686830997467,
      "step": 666
    },
    {
      "epoch": 1.0504334121355399,
      "grad_norm": 0.23528745770454407,
      "learning_rate": 2.178418248635988e-06,
      "log_odds_chosen": 0.8076845407485962,
      "log_odds_ratio": -0.3807538151741028,
      "logits/chosen": -0.2612028121948242,
      "logits/rejected": -0.7296571135520935,
      "logps/chosen": -1.6030538082122803,
      "logps/rejected": -2.2961909770965576,
      "loss": 1.6746,
      "nll_loss": 1.6365177631378174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16030538082122803,
      "rewards/margins": 0.06931371986865997,
      "rewards/rejected": -0.229619100689888,
      "step": 667
    },
    {
      "epoch": 1.0520094562647755,
      "grad_norm": 0.20690131187438965,
      "learning_rate": 2.1729423989814912e-06,
      "log_odds_chosen": 0.7662758231163025,
      "log_odds_ratio": -0.39079123735427856,
      "logits/chosen": -0.2992761731147766,
      "logits/rejected": -0.9287897944450378,
      "logps/chosen": -1.5230937004089355,
      "logps/rejected": -2.1684110164642334,
      "loss": 1.5555,
      "nll_loss": 1.516397476196289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1523093730211258,
      "rewards/margins": 0.06453172862529755,
      "rewards/rejected": -0.21684110164642334,
      "step": 668
    },
    {
      "epoch": 1.053585500394011,
      "grad_norm": 0.20352652668952942,
      "learning_rate": 2.167465242829774e-06,
      "log_odds_chosen": 0.8630669116973877,
      "log_odds_ratio": -0.35757502913475037,
      "logits/chosen": -0.3036842942237854,
      "logits/rejected": -0.9055747985839844,
      "logps/chosen": -1.5679224729537964,
      "logps/rejected": -2.3057737350463867,
      "loss": 1.6097,
      "nll_loss": 1.5739659070968628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15679225325584412,
      "rewards/margins": 0.07378511130809784,
      "rewards/rejected": -0.23057737946510315,
      "step": 669
    },
    {
      "epoch": 1.0551615445232467,
      "grad_norm": 0.2021772712469101,
      "learning_rate": 2.1619868215581343e-06,
      "log_odds_chosen": 0.9089421033859253,
      "log_odds_ratio": -0.34314393997192383,
      "logits/chosen": -0.3053356409072876,
      "logits/rejected": -0.8859040141105652,
      "logps/chosen": -1.4944835901260376,
      "logps/rejected": -2.256105899810791,
      "loss": 1.5583,
      "nll_loss": 1.5239697694778442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14944835007190704,
      "rewards/margins": 0.07616224884986877,
      "rewards/rejected": -0.22561059892177582,
      "step": 670
    },
    {
      "epoch": 1.0567375886524824,
      "grad_norm": 0.2011970430612564,
      "learning_rate": 2.1565071765534287e-06,
      "log_odds_chosen": 0.8158732056617737,
      "log_odds_ratio": -0.3711245059967041,
      "logits/chosen": -0.2421168088912964,
      "logits/rejected": -0.9611644744873047,
      "logps/chosen": -1.5732707977294922,
      "logps/rejected": -2.2663521766662598,
      "loss": 1.6112,
      "nll_loss": 1.57405686378479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573270708322525,
      "rewards/margins": 0.06930813938379288,
      "rewards/rejected": -0.22663521766662598,
      "step": 671
    },
    {
      "epoch": 1.058313632781718,
      "grad_norm": 0.2069043666124344,
      "learning_rate": 2.1510263492117574e-06,
      "log_odds_chosen": 0.6853798627853394,
      "log_odds_ratio": -0.4132938086986542,
      "logits/chosen": -0.22250482439994812,
      "logits/rejected": -0.7960903644561768,
      "logps/chosen": -1.563555121421814,
      "logps/rejected": -2.1420722007751465,
      "loss": 1.6223,
      "nll_loss": 1.5810190439224243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15635551512241364,
      "rewards/margins": 0.05785169452428818,
      "rewards/rejected": -0.2142072170972824,
      "step": 672
    },
    {
      "epoch": 1.0598896769109536,
      "grad_norm": 0.19049504399299622,
      "learning_rate": 2.1455443809381535e-06,
      "log_odds_chosen": 1.0065072774887085,
      "log_odds_ratio": -0.32247740030288696,
      "logits/chosen": -0.3116475045681,
      "logits/rejected": -1.1138675212860107,
      "logps/chosen": -1.4903415441513062,
      "logps/rejected": -2.3419151306152344,
      "loss": 1.5339,
      "nll_loss": 1.5016934871673584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14903415739536285,
      "rewards/margins": 0.0851573571562767,
      "rewards/rejected": -0.23419150710105896,
      "step": 673
    },
    {
      "epoch": 1.0614657210401892,
      "grad_norm": 0.2111048400402069,
      "learning_rate": 2.1400613131462697e-06,
      "log_odds_chosen": 0.950046181678772,
      "log_odds_ratio": -0.3349469304084778,
      "logits/chosen": -0.2336883544921875,
      "logits/rejected": -0.9827866554260254,
      "logps/chosen": -1.6412674188613892,
      "logps/rejected": -2.4668800830841064,
      "loss": 1.6664,
      "nll_loss": 1.6329472064971924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16412676870822906,
      "rewards/margins": 0.08256123960018158,
      "rewards/rejected": -0.24668799340724945,
      "step": 674
    },
    {
      "epoch": 1.0630417651694248,
      "grad_norm": 0.2013227492570877,
      "learning_rate": 2.1345771872580628e-06,
      "log_odds_chosen": 0.9758560657501221,
      "log_odds_ratio": -0.348794162273407,
      "logits/chosen": -0.2733922004699707,
      "logits/rejected": -0.9526365399360657,
      "logps/chosen": -1.5516276359558105,
      "logps/rejected": -2.3978376388549805,
      "loss": 1.5825,
      "nll_loss": 1.5476202964782715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1551627516746521,
      "rewards/margins": 0.08462101221084595,
      "rewards/rejected": -0.23978376388549805,
      "step": 675
    },
    {
      "epoch": 1.0646178092986605,
      "grad_norm": 0.19003072381019592,
      "learning_rate": 2.1290920447034846e-06,
      "log_odds_chosen": 0.9035107493400574,
      "log_odds_ratio": -0.34451401233673096,
      "logits/chosen": -0.2237972617149353,
      "logits/rejected": -0.9341007471084595,
      "logps/chosen": -1.5158449411392212,
      "logps/rejected": -2.276766300201416,
      "loss": 1.5597,
      "nll_loss": 1.525288701057434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15158450603485107,
      "rewards/margins": 0.07609213143587112,
      "rewards/rejected": -0.2276766300201416,
      "step": 676
    },
    {
      "epoch": 1.066193853427896,
      "grad_norm": 0.2086423635482788,
      "learning_rate": 2.1236059269201683e-06,
      "log_odds_chosen": 0.7209897041320801,
      "log_odds_ratio": -0.4030327796936035,
      "logits/chosen": -0.3177185654640198,
      "logits/rejected": -0.8937892913818359,
      "logps/chosen": -1.5658420324325562,
      "logps/rejected": -2.175645589828491,
      "loss": 1.5949,
      "nll_loss": 1.554572582244873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15658420324325562,
      "rewards/margins": 0.06098034232854843,
      "rewards/rejected": -0.21756455302238464,
      "step": 677
    },
    {
      "epoch": 1.0677698975571317,
      "grad_norm": 0.26009270548820496,
      "learning_rate": 2.1181188753531124e-06,
      "log_odds_chosen": 0.870098352432251,
      "log_odds_ratio": -0.3560430407524109,
      "logits/chosen": -0.3291873037815094,
      "logits/rejected": -0.8631235957145691,
      "logps/chosen": -1.7124642133712769,
      "logps/rejected": -2.472269058227539,
      "loss": 1.7353,
      "nll_loss": 1.6996697187423706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17124643921852112,
      "rewards/margins": 0.07598045468330383,
      "rewards/rejected": -0.24722687900066376,
      "step": 678
    },
    {
      "epoch": 1.0693459416863673,
      "grad_norm": 0.21179570257663727,
      "learning_rate": 2.112630931454371e-06,
      "log_odds_chosen": 0.9536169767379761,
      "log_odds_ratio": -0.33382853865623474,
      "logits/chosen": -0.24859049916267395,
      "logits/rejected": -0.9111310243606567,
      "logps/chosen": -1.463700532913208,
      "logps/rejected": -2.2605843544006348,
      "loss": 1.5156,
      "nll_loss": 1.482250690460205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1463700532913208,
      "rewards/margins": 0.07968837022781372,
      "rewards/rejected": -0.22605842351913452,
      "step": 679
    },
    {
      "epoch": 1.070921985815603,
      "grad_norm": 0.19382344186306,
      "learning_rate": 2.10714213668274e-06,
      "log_odds_chosen": 0.8163297772407532,
      "log_odds_ratio": -0.3690479099750519,
      "logits/chosen": -0.2804459035396576,
      "logits/rejected": -0.8734225034713745,
      "logps/chosen": -1.5064940452575684,
      "logps/rejected": -2.1909472942352295,
      "loss": 1.5632,
      "nll_loss": 1.5263135433197021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15064938366413116,
      "rewards/margins": 0.06844533234834671,
      "rewards/rejected": -0.21909473836421967,
      "step": 680
    },
    {
      "epoch": 1.0724980299448386,
      "grad_norm": 0.2072845995426178,
      "learning_rate": 2.1016525325034403e-06,
      "log_odds_chosen": 1.1502344608306885,
      "log_odds_ratio": -0.30130699276924133,
      "logits/chosen": -0.24809864163398743,
      "logits/rejected": -0.8717442154884338,
      "logps/chosen": -1.5165177583694458,
      "logps/rejected": -2.509082317352295,
      "loss": 1.5508,
      "nll_loss": 1.5206247568130493,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1516517847776413,
      "rewards/margins": 0.0992564707994461,
      "rewards/rejected": -0.2509082555770874,
      "step": 681
    },
    {
      "epoch": 1.074074074074074,
      "grad_norm": 0.192153200507164,
      "learning_rate": 2.096162160387811e-06,
      "log_odds_chosen": 0.7491259574890137,
      "log_odds_ratio": -0.39266830682754517,
      "logits/chosen": -0.21124041080474854,
      "logits/rejected": -0.8732647895812988,
      "logps/chosen": -1.446552038192749,
      "logps/rejected": -2.0558524131774902,
      "loss": 1.5143,
      "nll_loss": 1.4750535488128662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14465519785881042,
      "rewards/margins": 0.0609300434589386,
      "rewards/rejected": -0.20558524131774902,
      "step": 682
    },
    {
      "epoch": 1.0756501182033098,
      "grad_norm": 0.18908655643463135,
      "learning_rate": 2.09067106181299e-06,
      "log_odds_chosen": 0.9639242887496948,
      "log_odds_ratio": -0.33013150095939636,
      "logits/chosen": -0.2597469687461853,
      "logits/rejected": -0.939673662185669,
      "logps/chosen": -1.393530249595642,
      "logps/rejected": -2.1889231204986572,
      "loss": 1.4464,
      "nll_loss": 1.4133532047271729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13935300707817078,
      "rewards/margins": 0.07953930646181107,
      "rewards/rejected": -0.21889230608940125,
      "step": 683
    },
    {
      "epoch": 1.0772261623325452,
      "grad_norm": 0.19600100815296173,
      "learning_rate": 2.0851792782616052e-06,
      "log_odds_chosen": 1.0824642181396484,
      "log_odds_ratio": -0.3076101541519165,
      "logits/chosen": -0.17360028624534607,
      "logits/rejected": -1.0133646726608276,
      "logps/chosen": -1.4955987930297852,
      "logps/rejected": -2.4212093353271484,
      "loss": 1.5411,
      "nll_loss": 1.5103636980056763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149559885263443,
      "rewards/margins": 0.09256107360124588,
      "rewards/rejected": -0.24212095141410828,
      "step": 684
    },
    {
      "epoch": 1.078802206461781,
      "grad_norm": 0.20125281810760498,
      "learning_rate": 2.0796868512214576e-06,
      "log_odds_chosen": 0.97855544090271,
      "log_odds_ratio": -0.33637991547584534,
      "logits/chosen": -0.36037150025367737,
      "logits/rejected": -0.9488298892974854,
      "logps/chosen": -1.486480474472046,
      "logps/rejected": -2.312894344329834,
      "loss": 1.5469,
      "nll_loss": 1.5132901668548584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14864806830883026,
      "rewards/margins": 0.0826413705945015,
      "rewards/rejected": -0.23128941655158997,
      "step": 685
    },
    {
      "epoch": 1.0803782505910164,
      "grad_norm": 0.19159035384655,
      "learning_rate": 2.0741938221852103e-06,
      "log_odds_chosen": 0.8979485630989075,
      "log_odds_ratio": -0.35111314058303833,
      "logits/chosen": -0.24698059260845184,
      "logits/rejected": -0.9577164649963379,
      "logps/chosen": -1.485425591468811,
      "logps/rejected": -2.244891405105591,
      "loss": 1.5411,
      "nll_loss": 1.5060123205184937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14854256808757782,
      "rewards/margins": 0.07594658434391022,
      "rewards/rejected": -0.22448915243148804,
      "step": 686
    },
    {
      "epoch": 1.081954294720252,
      "grad_norm": 0.2218235433101654,
      "learning_rate": 2.0687002326500743e-06,
      "log_odds_chosen": 0.9697452783584595,
      "log_odds_ratio": -0.325246125459671,
      "logits/chosen": -0.3256666958332062,
      "logits/rejected": -0.9953600168228149,
      "logps/chosen": -1.4858152866363525,
      "logps/rejected": -2.303402900695801,
      "loss": 1.5272,
      "nll_loss": 1.4946974515914917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14858151972293854,
      "rewards/margins": 0.08175875246524811,
      "rewards/rejected": -0.23034028708934784,
      "step": 687
    },
    {
      "epoch": 1.0835303388494877,
      "grad_norm": 0.25145718455314636,
      "learning_rate": 2.0632061241174942e-06,
      "log_odds_chosen": 0.9677091836929321,
      "log_odds_ratio": -0.33493050932884216,
      "logits/chosen": -0.297545462846756,
      "logits/rejected": -0.9273019433021545,
      "logps/chosen": -1.5455886125564575,
      "logps/rejected": -2.3722074031829834,
      "loss": 1.581,
      "nll_loss": 1.547481894493103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15455885231494904,
      "rewards/margins": 0.08266190439462662,
      "rewards/rejected": -0.23722077906131744,
      "step": 688
    },
    {
      "epoch": 1.0851063829787233,
      "grad_norm": 0.21306590735912323,
      "learning_rate": 2.0577115380928364e-06,
      "log_odds_chosen": 0.9169594645500183,
      "log_odds_ratio": -0.3488909900188446,
      "logits/chosen": -0.26201170682907104,
      "logits/rejected": -0.9411023259162903,
      "logps/chosen": -1.5122214555740356,
      "logps/rejected": -2.2935245037078857,
      "loss": 1.5445,
      "nll_loss": 1.5096161365509033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1512221395969391,
      "rewards/margins": 0.0781303197145462,
      "rewards/rejected": -0.2293524444103241,
      "step": 689
    },
    {
      "epoch": 1.086682427107959,
      "grad_norm": 0.20546680688858032,
      "learning_rate": 2.052216516085073e-06,
      "log_odds_chosen": 0.831648588180542,
      "log_odds_ratio": -0.3817562460899353,
      "logits/chosen": -0.2853247821331024,
      "logits/rejected": -0.9388511776924133,
      "logps/chosen": -1.4612269401550293,
      "logps/rejected": -2.1631882190704346,
      "loss": 1.5206,
      "nll_loss": 1.482445478439331,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14612270891666412,
      "rewards/margins": 0.07019612938165665,
      "rewards/rejected": -0.21631881594657898,
      "step": 690
    },
    {
      "epoch": 1.0882584712371945,
      "grad_norm": 0.22151832282543182,
      "learning_rate": 2.0467210996064707e-06,
      "log_odds_chosen": 0.7678408026695251,
      "log_odds_ratio": -0.3888075053691864,
      "logits/chosen": -0.3574616611003876,
      "logits/rejected": -0.6472020745277405,
      "logps/chosen": -1.5306328535079956,
      "logps/rejected": -2.177694320678711,
      "loss": 1.567,
      "nll_loss": 1.5281471014022827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15306328237056732,
      "rewards/margins": 0.06470613181591034,
      "rewards/rejected": -0.21776944398880005,
      "step": 691
    },
    {
      "epoch": 1.0898345153664302,
      "grad_norm": 0.19064006209373474,
      "learning_rate": 2.0412253301722774e-06,
      "log_odds_chosen": 1.0286555290222168,
      "log_odds_ratio": -0.3092900514602661,
      "logits/chosen": -0.2332383543252945,
      "logits/rejected": -1.010955572128296,
      "logps/chosen": -1.5690068006515503,
      "logps/rejected": -2.454373598098755,
      "loss": 1.6097,
      "nll_loss": 1.5788038969039917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15690067410469055,
      "rewards/margins": 0.08853667229413986,
      "rewards/rejected": -0.2454373836517334,
      "step": 692
    },
    {
      "epoch": 1.0914105594956658,
      "grad_norm": 0.19384533166885376,
      "learning_rate": 2.0357292493004044e-06,
      "log_odds_chosen": 0.9699249267578125,
      "log_odds_ratio": -0.3262019753456116,
      "logits/chosen": -0.2196933627128601,
      "logits/rejected": -0.9613053798675537,
      "logps/chosen": -1.4959990978240967,
      "logps/rejected": -2.3170418739318848,
      "loss": 1.5247,
      "nll_loss": 1.4920899868011475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14959992468357086,
      "rewards/margins": 0.08210429549217224,
      "rewards/rejected": -0.2317042201757431,
      "step": 693
    },
    {
      "epoch": 1.0929866036249014,
      "grad_norm": 0.2010764479637146,
      "learning_rate": 2.0302328985111193e-06,
      "log_odds_chosen": 1.022120714187622,
      "log_odds_ratio": -0.3276897370815277,
      "logits/chosen": -0.17788267135620117,
      "logits/rejected": -0.9701160192489624,
      "logps/chosen": -1.4484479427337646,
      "logps/rejected": -2.315668821334839,
      "loss": 1.5056,
      "nll_loss": 1.472790241241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14484478533267975,
      "rewards/margins": 0.08672208338975906,
      "rewards/rejected": -0.2315668910741806,
      "step": 694
    },
    {
      "epoch": 1.094562647754137,
      "grad_norm": 0.2163962572813034,
      "learning_rate": 2.0247363193267256e-06,
      "log_odds_chosen": 0.909433126449585,
      "log_odds_ratio": -0.34329187870025635,
      "logits/chosen": -0.18117782473564148,
      "logits/rejected": -0.92548006772995,
      "logps/chosen": -1.5629791021347046,
      "logps/rejected": -2.3399498462677,
      "loss": 1.5955,
      "nll_loss": 1.5611307621002197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15629792213439941,
      "rewards/margins": 0.07769709080457687,
      "rewards/rejected": -0.2339950054883957,
      "step": 695
    },
    {
      "epoch": 1.0961386918833727,
      "grad_norm": 0.21157033741474152,
      "learning_rate": 2.019239553271255e-06,
      "log_odds_chosen": 1.024816870689392,
      "log_odds_ratio": -0.31641706824302673,
      "logits/chosen": -0.25130489468574524,
      "logits/rejected": -1.0264472961425781,
      "logps/chosen": -1.497871994972229,
      "logps/rejected": -2.3680384159088135,
      "loss": 1.5425,
      "nll_loss": 1.5108526945114136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14978720247745514,
      "rewards/margins": 0.08701664954423904,
      "rewards/rejected": -0.23680385947227478,
      "step": 696
    },
    {
      "epoch": 1.0977147360126083,
      "grad_norm": 0.19864477217197418,
      "learning_rate": 2.0137426418701488e-06,
      "log_odds_chosen": 1.0378433465957642,
      "log_odds_ratio": -0.31480899453163147,
      "logits/chosen": -0.23130860924720764,
      "logits/rejected": -0.9598379731178284,
      "logps/chosen": -1.5000468492507935,
      "logps/rejected": -2.3893837928771973,
      "loss": 1.5448,
      "nll_loss": 1.5133565664291382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15000468492507935,
      "rewards/margins": 0.08893368393182755,
      "rewards/rejected": -0.2389383763074875,
      "step": 697
    },
    {
      "epoch": 1.099290780141844,
      "grad_norm": 0.1923292577266693,
      "learning_rate": 2.008245626649947e-06,
      "log_odds_chosen": 0.9621240496635437,
      "log_odds_ratio": -0.3462398052215576,
      "logits/chosen": -0.27533259987831116,
      "logits/rejected": -1.1044375896453857,
      "logps/chosen": -1.4487746953964233,
      "logps/rejected": -2.26387095451355,
      "loss": 1.5081,
      "nll_loss": 1.4734283685684204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14487746357917786,
      "rewards/margins": 0.08150961995124817,
      "rewards/rejected": -0.22638706862926483,
      "step": 698
    },
    {
      "epoch": 1.1008668242710795,
      "grad_norm": 0.22848764061927795,
      "learning_rate": 2.0027485491379746e-06,
      "log_odds_chosen": 1.075317144393921,
      "log_odds_ratio": -0.3047947287559509,
      "logits/chosen": -0.40208151936531067,
      "logits/rejected": -0.8406577110290527,
      "logps/chosen": -1.6428619623184204,
      "logps/rejected": -2.5771408081054688,
      "loss": 1.6691,
      "nll_loss": 1.6386264562606812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16428619623184204,
      "rewards/margins": 0.093427874147892,
      "rewards/rejected": -0.25771406292915344,
      "step": 699
    },
    {
      "epoch": 1.1024428684003151,
      "grad_norm": 0.21458838880062103,
      "learning_rate": 1.9972514508620256e-06,
      "log_odds_chosen": 0.9174166321754456,
      "log_odds_ratio": -0.34811967611312866,
      "logits/chosen": -0.24462562799453735,
      "logits/rejected": -0.9917095303535461,
      "logps/chosen": -1.4866465330123901,
      "logps/rejected": -2.2594070434570312,
      "loss": 1.5358,
      "nll_loss": 1.5010063648223877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.148664653301239,
      "rewards/margins": 0.07727605849504471,
      "rewards/rejected": -0.22594070434570312,
      "step": 700
    },
    {
      "epoch": 1.1040189125295508,
      "grad_norm": 0.20889812707901,
      "learning_rate": 1.991754373350053e-06,
      "log_odds_chosen": 0.6961079835891724,
      "log_odds_ratio": -0.4143297076225281,
      "logits/chosen": -0.2887740433216095,
      "logits/rejected": -0.9712682962417603,
      "logps/chosen": -1.5484859943389893,
      "logps/rejected": -2.1341640949249268,
      "loss": 1.6002,
      "nll_loss": 1.5587782859802246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548486053943634,
      "rewards/margins": 0.058567821979522705,
      "rewards/rejected": -0.2134164422750473,
      "step": 701
    },
    {
      "epoch": 1.1055949566587864,
      "grad_norm": 0.20063714683055878,
      "learning_rate": 1.986257358129852e-06,
      "log_odds_chosen": 0.8949864506721497,
      "log_odds_ratio": -0.34919092059135437,
      "logits/chosen": -0.2694525420665741,
      "logits/rejected": -0.9621233940124512,
      "logps/chosen": -1.4433969259262085,
      "logps/rejected": -2.1830191612243652,
      "loss": 1.4935,
      "nll_loss": 1.4585883617401123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1443396806716919,
      "rewards/margins": 0.07396221160888672,
      "rewards/rejected": -0.2183019071817398,
      "step": 702
    },
    {
      "epoch": 1.107171000788022,
      "grad_norm": 0.2723228335380554,
      "learning_rate": 1.9807604467287453e-06,
      "log_odds_chosen": 0.8745384812355042,
      "log_odds_ratio": -0.3553420603275299,
      "logits/chosen": -0.331906795501709,
      "logits/rejected": -0.788597583770752,
      "logps/chosen": -1.5835316181182861,
      "logps/rejected": -2.3282837867736816,
      "loss": 1.6112,
      "nll_loss": 1.5756691694259644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15835314989089966,
      "rewards/margins": 0.07447522133588791,
      "rewards/rejected": -0.23282836377620697,
      "step": 703
    },
    {
      "epoch": 1.1087470449172576,
      "grad_norm": 0.20463499426841736,
      "learning_rate": 1.9752636806732742e-06,
      "log_odds_chosen": 0.8855620622634888,
      "log_odds_ratio": -0.34919407963752747,
      "logits/chosen": -0.20010122656822205,
      "logits/rejected": -0.9473900198936462,
      "logps/chosen": -1.4821786880493164,
      "logps/rejected": -2.2254881858825684,
      "loss": 1.5184,
      "nll_loss": 1.4834460020065308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14821787178516388,
      "rewards/margins": 0.07433094084262848,
      "rewards/rejected": -0.22254881262779236,
      "step": 704
    },
    {
      "epoch": 1.1103230890464932,
      "grad_norm": 0.21732135117053986,
      "learning_rate": 1.9697671014888805e-06,
      "log_odds_chosen": 0.9765762090682983,
      "log_odds_ratio": -0.32620343565940857,
      "logits/chosen": -0.3735521733760834,
      "logits/rejected": -0.8439749479293823,
      "logps/chosen": -1.5605196952819824,
      "logps/rejected": -2.396658182144165,
      "loss": 1.5958,
      "nll_loss": 1.5631530284881592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15605197846889496,
      "rewards/margins": 0.08361382782459259,
      "rewards/rejected": -0.23966582119464874,
      "step": 705
    },
    {
      "epoch": 1.1118991331757289,
      "grad_norm": 0.20080871880054474,
      "learning_rate": 1.9642707506995954e-06,
      "log_odds_chosen": 1.0879112482070923,
      "log_odds_ratio": -0.2983154356479645,
      "logits/chosen": -0.29149124026298523,
      "logits/rejected": -1.0468382835388184,
      "logps/chosen": -1.4993736743927002,
      "logps/rejected": -2.430717706680298,
      "loss": 1.5383,
      "nll_loss": 1.5085018873214722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14993737637996674,
      "rewards/margins": 0.09313439577817917,
      "rewards/rejected": -0.2430717647075653,
      "step": 706
    },
    {
      "epoch": 1.1134751773049645,
      "grad_norm": 0.1945836991071701,
      "learning_rate": 1.9587746698277232e-06,
      "log_odds_chosen": 0.8970012664794922,
      "log_odds_ratio": -0.36084216833114624,
      "logits/chosen": -0.315437912940979,
      "logits/rejected": -0.8308150172233582,
      "logps/chosen": -1.499241828918457,
      "logps/rejected": -2.262544631958008,
      "loss": 1.5398,
      "nll_loss": 1.5036754608154297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14992420375347137,
      "rewards/margins": 0.0763302743434906,
      "rewards/rejected": -0.22625446319580078,
      "step": 707
    },
    {
      "epoch": 1.1150512214342,
      "grad_norm": 0.23695383965969086,
      "learning_rate": 1.953278900393529e-06,
      "log_odds_chosen": 0.8935959339141846,
      "log_odds_ratio": -0.35870230197906494,
      "logits/chosen": -0.3227505683898926,
      "logits/rejected": -0.7964525818824768,
      "logps/chosen": -1.5599826574325562,
      "logps/rejected": -2.323190212249756,
      "loss": 1.5751,
      "nll_loss": 1.53925621509552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15599827468395233,
      "rewards/margins": 0.07632075995206833,
      "rewards/rejected": -0.23231901228427887,
      "step": 708
    },
    {
      "epoch": 1.1166272655634357,
      "grad_norm": 0.23588566482067108,
      "learning_rate": 1.9477834839149274e-06,
      "log_odds_chosen": 0.9891924262046814,
      "log_odds_ratio": -0.32777783274650574,
      "logits/chosen": -0.3270997703075409,
      "logits/rejected": -0.9252603054046631,
      "logps/chosen": -1.4509809017181396,
      "logps/rejected": -2.285118818283081,
      "loss": 1.5148,
      "nll_loss": 1.4820170402526855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14509810507297516,
      "rewards/margins": 0.08341380208730698,
      "rewards/rejected": -0.22851189970970154,
      "step": 709
    },
    {
      "epoch": 1.1182033096926713,
      "grad_norm": 0.23068565130233765,
      "learning_rate": 1.942288461907164e-06,
      "log_odds_chosen": 0.9291799068450928,
      "log_odds_ratio": -0.34601297974586487,
      "logits/chosen": -0.30701199173927307,
      "logits/rejected": -0.9044516682624817,
      "logps/chosen": -1.4564368724822998,
      "logps/rejected": -2.240225076675415,
      "loss": 1.5165,
      "nll_loss": 1.4818742275238037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1456436961889267,
      "rewards/margins": 0.0783788189291954,
      "rewards/rejected": -0.2240224927663803,
      "step": 710
    },
    {
      "epoch": 1.119779353821907,
      "grad_norm": 0.22865253686904907,
      "learning_rate": 1.936793875882505e-06,
      "log_odds_chosen": 0.9833695888519287,
      "log_odds_ratio": -0.32182392477989197,
      "logits/chosen": -0.13819369673728943,
      "logits/rejected": -0.9392806887626648,
      "logps/chosen": -1.6067181825637817,
      "logps/rejected": -2.4564261436462402,
      "loss": 1.6598,
      "nll_loss": 1.6276073455810547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16067183017730713,
      "rewards/margins": 0.08497080951929092,
      "rewards/rejected": -0.24564263224601746,
      "step": 711
    },
    {
      "epoch": 1.1213553979511426,
      "grad_norm": 0.21684178709983826,
      "learning_rate": 1.931299767349926e-06,
      "log_odds_chosen": 0.9651028513908386,
      "log_odds_ratio": -0.33438360691070557,
      "logits/chosen": -0.31118014454841614,
      "logits/rejected": -0.8043417930603027,
      "logps/chosen": -1.4366124868392944,
      "logps/rejected": -2.2357497215270996,
      "loss": 1.4886,
      "nll_loss": 1.4551756381988525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1436612755060196,
      "rewards/margins": 0.07991372793912888,
      "rewards/rejected": -0.22357499599456787,
      "step": 712
    },
    {
      "epoch": 1.1229314420803782,
      "grad_norm": 0.22175389528274536,
      "learning_rate": 1.9258061778147895e-06,
      "log_odds_chosen": 0.8915292620658875,
      "log_odds_ratio": -0.3518582284450531,
      "logits/chosen": -0.3330654203891754,
      "logits/rejected": -1.0483332872390747,
      "logps/chosen": -1.5444999933242798,
      "logps/rejected": -2.304286003112793,
      "loss": 1.5909,
      "nll_loss": 1.5557161569595337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15445001423358917,
      "rewards/margins": 0.0759785920381546,
      "rewards/rejected": -0.23042860627174377,
      "step": 713
    },
    {
      "epoch": 1.1245074862096138,
      "grad_norm": 0.2109820544719696,
      "learning_rate": 1.9203131487785426e-06,
      "log_odds_chosen": 1.100347638130188,
      "log_odds_ratio": -0.2922804653644562,
      "logits/chosen": -0.4427320063114166,
      "logits/rejected": -0.9111467003822327,
      "logps/chosen": -1.430337905883789,
      "logps/rejected": -2.356025218963623,
      "loss": 1.4837,
      "nll_loss": 1.4544481039047241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14303377270698547,
      "rewards/margins": 0.09256873279809952,
      "rewards/rejected": -0.23560252785682678,
      "step": 714
    },
    {
      "epoch": 1.1260835303388494,
      "grad_norm": 0.32483428716659546,
      "learning_rate": 1.9148207217383946e-06,
      "log_odds_chosen": 1.2034516334533691,
      "log_odds_ratio": -0.265217125415802,
      "logits/chosen": -0.2711378335952759,
      "logits/rejected": -1.0504577159881592,
      "logps/chosen": -1.542511224746704,
      "logps/rejected": -2.584134817123413,
      "loss": 1.5595,
      "nll_loss": 1.5330171585083008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1542511284351349,
      "rewards/margins": 0.10416235029697418,
      "rewards/rejected": -0.2584134638309479,
      "step": 715
    },
    {
      "epoch": 1.127659574468085,
      "grad_norm": 0.2173331081867218,
      "learning_rate": 1.9093289381870094e-06,
      "log_odds_chosen": 0.7955161333084106,
      "log_odds_ratio": -0.37909385561943054,
      "logits/chosen": -0.30374932289123535,
      "logits/rejected": -0.9530816674232483,
      "logps/chosen": -1.5344806909561157,
      "logps/rejected": -2.206887722015381,
      "loss": 1.5818,
      "nll_loss": 1.543928623199463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15344806015491486,
      "rewards/margins": 0.06724072247743607,
      "rewards/rejected": -0.2206888049840927,
      "step": 716
    },
    {
      "epoch": 1.1292356185973207,
      "grad_norm": 0.2083769589662552,
      "learning_rate": 1.9038378396121895e-06,
      "log_odds_chosen": 1.0013920068740845,
      "log_odds_ratio": -0.3208934962749481,
      "logits/chosen": -0.1274857223033905,
      "logits/rejected": -1.0461782217025757,
      "logps/chosen": -1.5757423639297485,
      "logps/rejected": -2.441706895828247,
      "loss": 1.6138,
      "nll_loss": 1.5817304849624634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15757423639297485,
      "rewards/margins": 0.08659645915031433,
      "rewards/rejected": -0.24417069554328918,
      "step": 717
    },
    {
      "epoch": 1.1308116627265563,
      "grad_norm": 0.1972358375787735,
      "learning_rate": 1.8983474674965597e-06,
      "log_odds_chosen": 0.9565795660018921,
      "log_odds_ratio": -0.34190720319747925,
      "logits/chosen": -0.2967897057533264,
      "logits/rejected": -1.0658279657363892,
      "logps/chosen": -1.506929874420166,
      "logps/rejected": -2.317199945449829,
      "loss": 1.5496,
      "nll_loss": 1.515379548072815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15069298446178436,
      "rewards/margins": 0.08102700859308243,
      "rewards/rejected": -0.2317199856042862,
      "step": 718
    },
    {
      "epoch": 1.132387706855792,
      "grad_norm": 0.21626442670822144,
      "learning_rate": 1.8928578633172603e-06,
      "log_odds_chosen": 1.1320643424987793,
      "log_odds_ratio": -0.28411558270454407,
      "logits/chosen": -0.2692641615867615,
      "logits/rejected": -1.0467296838760376,
      "logps/chosen": -1.5431231260299683,
      "logps/rejected": -2.518969774246216,
      "loss": 1.5854,
      "nll_loss": 1.5569418668746948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15431231260299683,
      "rewards/margins": 0.09758466482162476,
      "rewards/rejected": -0.2518969774246216,
      "step": 719
    },
    {
      "epoch": 1.1339637509850276,
      "grad_norm": 0.20519238710403442,
      "learning_rate": 1.8873690685456283e-06,
      "log_odds_chosen": 1.0468000173568726,
      "log_odds_ratio": -0.3102174401283264,
      "logits/chosen": -0.3111383020877838,
      "logits/rejected": -0.9187269806861877,
      "logps/chosen": -1.4898278713226318,
      "logps/rejected": -2.3791775703430176,
      "loss": 1.5274,
      "nll_loss": 1.496414303779602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14898279309272766,
      "rewards/margins": 0.08893495053052902,
      "rewards/rejected": -0.2379177361726761,
      "step": 720
    },
    {
      "epoch": 1.1355397951142632,
      "grad_norm": 0.2034110724925995,
      "learning_rate": 1.8818811246468872e-06,
      "log_odds_chosen": 1.186771035194397,
      "log_odds_ratio": -0.29065656661987305,
      "logits/chosen": -0.2709793448448181,
      "logits/rejected": -0.9832121133804321,
      "logps/chosen": -1.477704644203186,
      "logps/rejected": -2.4977824687957764,
      "loss": 1.5182,
      "nll_loss": 1.4890930652618408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477704793214798,
      "rewards/margins": 0.10200777649879456,
      "rewards/rejected": -0.24977825582027435,
      "step": 721
    },
    {
      "epoch": 1.1371158392434988,
      "grad_norm": 0.22037352621555328,
      "learning_rate": 1.876394073079832e-06,
      "log_odds_chosen": 1.1023344993591309,
      "log_odds_ratio": -0.2979393005371094,
      "logits/chosen": -0.35737934708595276,
      "logits/rejected": -0.8673989772796631,
      "logps/chosen": -1.6099216938018799,
      "logps/rejected": -2.570438861846924,
      "loss": 1.6375,
      "nll_loss": 1.6076719760894775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16099217534065247,
      "rewards/margins": 0.09605173021554947,
      "rewards/rejected": -0.25704389810562134,
      "step": 722
    },
    {
      "epoch": 1.1386918833727344,
      "grad_norm": 0.20414580404758453,
      "learning_rate": 1.8709079552965152e-06,
      "log_odds_chosen": 1.0163118839263916,
      "log_odds_ratio": -0.3254520893096924,
      "logits/chosen": -0.2989614009857178,
      "logits/rejected": -0.9564584493637085,
      "logps/chosen": -1.4801301956176758,
      "logps/rejected": -2.3410110473632812,
      "loss": 1.5212,
      "nll_loss": 1.4886314868927002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14801302552223206,
      "rewards/margins": 0.08608809113502502,
      "rewards/rejected": -0.2341011017560959,
      "step": 723
    },
    {
      "epoch": 1.14026792750197,
      "grad_norm": 0.21033595502376556,
      "learning_rate": 1.8654228127419375e-06,
      "log_odds_chosen": 0.8655036687850952,
      "log_odds_ratio": -0.35758084058761597,
      "logits/chosen": -0.35425326228141785,
      "logits/rejected": -0.9153095483779907,
      "logps/chosen": -1.4978307485580444,
      "logps/rejected": -2.228721857070923,
      "loss": 1.5394,
      "nll_loss": 1.5036377906799316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14978307485580444,
      "rewards/margins": 0.0730891227722168,
      "rewards/rejected": -0.22287218272686005,
      "step": 724
    },
    {
      "epoch": 1.1418439716312057,
      "grad_norm": 0.1922226846218109,
      "learning_rate": 1.8599386868537306e-06,
      "log_odds_chosen": 1.155823826789856,
      "log_odds_ratio": -0.27717486023902893,
      "logits/chosen": -0.30529868602752686,
      "logits/rejected": -1.0317051410675049,
      "logps/chosen": -1.370191216468811,
      "logps/rejected": -2.3324990272521973,
      "loss": 1.4254,
      "nll_loss": 1.3976707458496094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13701912760734558,
      "rewards/margins": 0.09623079001903534,
      "rewards/rejected": -0.23324991762638092,
      "step": 725
    },
    {
      "epoch": 1.1434200157604413,
      "grad_norm": 0.2180519700050354,
      "learning_rate": 1.8544556190618464e-06,
      "log_odds_chosen": 0.9957131147384644,
      "log_odds_ratio": -0.31708675622940063,
      "logits/chosen": -0.3554293215274811,
      "logits/rejected": -0.8810909986495972,
      "logps/chosen": -1.5760176181793213,
      "logps/rejected": -2.430025100708008,
      "loss": 1.6068,
      "nll_loss": 1.5751209259033203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15760177373886108,
      "rewards/margins": 0.08540076017379761,
      "rewards/rejected": -0.24300254881381989,
      "step": 726
    },
    {
      "epoch": 1.144996059889677,
      "grad_norm": 0.21728859841823578,
      "learning_rate": 1.8489736507882424e-06,
      "log_odds_chosen": 1.2690489292144775,
      "log_odds_ratio": -0.277741938829422,
      "logits/chosen": -0.2846432626247406,
      "logits/rejected": -0.9452536702156067,
      "logps/chosen": -1.4815127849578857,
      "logps/rejected": -2.5821170806884766,
      "loss": 1.5154,
      "nll_loss": 1.4876456260681152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14815127849578857,
      "rewards/margins": 0.1100604236125946,
      "rewards/rejected": -0.2582116723060608,
      "step": 727
    },
    {
      "epoch": 1.1465721040189125,
      "grad_norm": 0.20500479638576508,
      "learning_rate": 1.8434928234465716e-06,
      "log_odds_chosen": 1.0647529363632202,
      "log_odds_ratio": -0.3047347962856293,
      "logits/chosen": -0.3823509216308594,
      "logits/rejected": -0.9670817852020264,
      "logps/chosen": -1.455940842628479,
      "logps/rejected": -2.357487916946411,
      "loss": 1.5041,
      "nll_loss": 1.4736424684524536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14559407532215118,
      "rewards/margins": 0.090154729783535,
      "rewards/rejected": -0.2357487827539444,
      "step": 728
    },
    {
      "epoch": 1.1481481481481481,
      "grad_norm": 0.23868386447429657,
      "learning_rate": 1.8380131784418657e-06,
      "log_odds_chosen": 1.1269980669021606,
      "log_odds_ratio": -0.3020734190940857,
      "logits/chosen": -0.22707228362560272,
      "logits/rejected": -0.9218549728393555,
      "logps/chosen": -1.505780577659607,
      "logps/rejected": -2.477417469024658,
      "loss": 1.5517,
      "nll_loss": 1.5214866399765015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1505780667066574,
      "rewards/margins": 0.09716369211673737,
      "rewards/rejected": -0.24774175882339478,
      "step": 729
    },
    {
      "epoch": 1.1497241922773838,
      "grad_norm": 0.2116031050682068,
      "learning_rate": 1.8325347571702259e-06,
      "log_odds_chosen": 0.9921974539756775,
      "log_odds_ratio": -0.3236672282218933,
      "logits/chosen": -0.34898027777671814,
      "logits/rejected": -1.022017240524292,
      "logps/chosen": -1.553710699081421,
      "logps/rejected": -2.4056694507598877,
      "loss": 1.5755,
      "nll_loss": 1.5431360006332397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15537108480930328,
      "rewards/margins": 0.0851958766579628,
      "rewards/rejected": -0.2405669391155243,
      "step": 730
    },
    {
      "epoch": 1.1513002364066194,
      "grad_norm": 0.23198233544826508,
      "learning_rate": 1.827057601018509e-06,
      "log_odds_chosen": 1.1416021585464478,
      "log_odds_ratio": -0.2932335436344147,
      "logits/chosen": -0.389062762260437,
      "logits/rejected": -1.0393038988113403,
      "logps/chosen": -1.4419293403625488,
      "logps/rejected": -2.4057106971740723,
      "loss": 1.4989,
      "nll_loss": 1.469563364982605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14419294893741608,
      "rewards/margins": 0.0963781327009201,
      "rewards/rejected": -0.24057108163833618,
      "step": 731
    },
    {
      "epoch": 1.152876280535855,
      "grad_norm": 0.21897879242897034,
      "learning_rate": 1.8215817513640122e-06,
      "log_odds_chosen": 1.1703872680664062,
      "log_odds_ratio": -0.2870897352695465,
      "logits/chosen": -0.29620474576950073,
      "logits/rejected": -0.9849406480789185,
      "logps/chosen": -1.495749831199646,
      "logps/rejected": -2.497593402862549,
      "loss": 1.5258,
      "nll_loss": 1.4971160888671875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14957499504089355,
      "rewards/margins": 0.1001843586564064,
      "rewards/rejected": -0.24975934624671936,
      "step": 732
    },
    {
      "epoch": 1.1544523246650906,
      "grad_norm": 0.20853158831596375,
      "learning_rate": 1.8161072495741647e-06,
      "log_odds_chosen": 1.0997140407562256,
      "log_odds_ratio": -0.2949446737766266,
      "logits/chosen": -0.26218992471694946,
      "logits/rejected": -1.0551815032958984,
      "logps/chosen": -1.4597110748291016,
      "logps/rejected": -2.3889989852905273,
      "loss": 1.5052,
      "nll_loss": 1.475714921951294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14597110450267792,
      "rewards/margins": 0.09292877465486526,
      "rewards/rejected": -0.23889987170696259,
      "step": 733
    },
    {
      "epoch": 1.1560283687943262,
      "grad_norm": 0.21174181997776031,
      "learning_rate": 1.810634137006213e-06,
      "log_odds_chosen": 0.7592455148696899,
      "log_odds_ratio": -0.38611549139022827,
      "logits/chosen": -0.3542156517505646,
      "logits/rejected": -0.7729081511497498,
      "logps/chosen": -1.5473408699035645,
      "logps/rejected": -2.186426877975464,
      "loss": 1.598,
      "nll_loss": 1.5594185590744019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15473410487174988,
      "rewards/margins": 0.06390860676765442,
      "rewards/rejected": -0.2186426967382431,
      "step": 734
    },
    {
      "epoch": 1.1576044129235619,
      "grad_norm": 0.2057434320449829,
      "learning_rate": 1.805162455006908e-06,
      "log_odds_chosen": 0.9340770244598389,
      "log_odds_ratio": -0.34093040227890015,
      "logits/chosen": -0.2892776131629944,
      "logits/rejected": -0.855111300945282,
      "logps/chosen": -1.5582666397094727,
      "logps/rejected": -2.3569624423980713,
      "loss": 1.5788,
      "nll_loss": 1.5447454452514648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15582667291164398,
      "rewards/margins": 0.0798695832490921,
      "rewards/rejected": -0.23569625616073608,
      "step": 735
    },
    {
      "epoch": 1.1591804570527975,
      "grad_norm": 0.2108977735042572,
      "learning_rate": 1.799692244912195e-06,
      "log_odds_chosen": 0.9119284749031067,
      "log_odds_ratio": -0.363698273897171,
      "logits/chosen": -0.30674266815185547,
      "logits/rejected": -0.7863306999206543,
      "logps/chosen": -1.553528904914856,
      "logps/rejected": -2.3382933139801025,
      "loss": 1.6031,
      "nll_loss": 1.5667353868484497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15535292029380798,
      "rewards/margins": 0.0784764289855957,
      "rewards/rejected": -0.2338293194770813,
      "step": 736
    },
    {
      "epoch": 1.160756501182033,
      "grad_norm": 0.2081252932548523,
      "learning_rate": 1.7942235480468964e-06,
      "log_odds_chosen": 1.0234342813491821,
      "log_odds_ratio": -0.3197007477283478,
      "logits/chosen": -0.3504559397697449,
      "logits/rejected": -0.9067404866218567,
      "logps/chosen": -1.4382294416427612,
      "logps/rejected": -2.2984979152679443,
      "loss": 1.4853,
      "nll_loss": 1.4532840251922607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14382295310497284,
      "rewards/margins": 0.08602683246135712,
      "rewards/rejected": -0.22984977066516876,
      "step": 737
    },
    {
      "epoch": 1.1623325453112687,
      "grad_norm": 0.21285904943943024,
      "learning_rate": 1.7887564057244062e-06,
      "log_odds_chosen": 1.1078662872314453,
      "log_odds_ratio": -0.32468411326408386,
      "logits/chosen": -0.295391708612442,
      "logits/rejected": -0.8858685493469238,
      "logps/chosen": -1.3938318490982056,
      "logps/rejected": -2.333885908126831,
      "loss": 1.4452,
      "nll_loss": 1.4126954078674316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1393831968307495,
      "rewards/margins": 0.09400539100170135,
      "rewards/rejected": -0.23338858783245087,
      "step": 738
    },
    {
      "epoch": 1.1639085894405043,
      "grad_norm": 0.21565620601177216,
      "learning_rate": 1.7832908592463732e-06,
      "log_odds_chosen": 0.9507670402526855,
      "log_odds_ratio": -0.3382663428783417,
      "logits/chosen": -0.41738462448120117,
      "logits/rejected": -0.9783397912979126,
      "logps/chosen": -1.5321942567825317,
      "logps/rejected": -2.330233335494995,
      "loss": 1.5811,
      "nll_loss": 1.5472469329833984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15321943163871765,
      "rewards/margins": 0.07980390638113022,
      "rewards/rejected": -0.23302333056926727,
      "step": 739
    },
    {
      "epoch": 1.16548463356974,
      "grad_norm": 0.22341176867485046,
      "learning_rate": 1.7778269499023908e-06,
      "log_odds_chosen": 1.0151731967926025,
      "log_odds_ratio": -0.3210703134536743,
      "logits/chosen": -0.3023481070995331,
      "logits/rejected": -0.9575453996658325,
      "logps/chosen": -1.5715084075927734,
      "logps/rejected": -2.4477741718292236,
      "loss": 1.6079,
      "nll_loss": 1.5758386850357056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15715086460113525,
      "rewards/margins": 0.08762655407190323,
      "rewards/rejected": -0.24477741122245789,
      "step": 740
    },
    {
      "epoch": 1.1670606776989756,
      "grad_norm": 0.23965880274772644,
      "learning_rate": 1.7723647189696843e-06,
      "log_odds_chosen": 0.9646286964416504,
      "log_odds_ratio": -0.33112287521362305,
      "logits/chosen": -0.2760770320892334,
      "logits/rejected": -0.7500057816505432,
      "logps/chosen": -1.4593937397003174,
      "logps/rejected": -2.2675344944000244,
      "loss": 1.5179,
      "nll_loss": 1.4847612380981445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14593937993049622,
      "rewards/margins": 0.08081405609846115,
      "rewards/rejected": -0.22675344347953796,
      "step": 741
    },
    {
      "epoch": 1.1686367218282112,
      "grad_norm": 0.22183716297149658,
      "learning_rate": 1.7669042077127982e-06,
      "log_odds_chosen": 1.064090609550476,
      "log_odds_ratio": -0.31245386600494385,
      "logits/chosen": -0.27123942971229553,
      "logits/rejected": -0.8995223045349121,
      "logps/chosen": -1.527857780456543,
      "logps/rejected": -2.438861846923828,
      "loss": 1.5589,
      "nll_loss": 1.5276998281478882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1527857631444931,
      "rewards/margins": 0.09110042452812195,
      "rewards/rejected": -0.24388618767261505,
      "step": 742
    },
    {
      "epoch": 1.1702127659574468,
      "grad_norm": 0.1992058902978897,
      "learning_rate": 1.7614454573832886e-06,
      "log_odds_chosen": 1.1425700187683105,
      "log_odds_ratio": -0.2829878628253937,
      "logits/chosen": -0.4249573349952698,
      "logits/rejected": -1.0127842426300049,
      "logps/chosen": -1.396596908569336,
      "logps/rejected": -2.3520700931549072,
      "loss": 1.453,
      "nll_loss": 1.4247173070907593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13965968787670135,
      "rewards/margins": 0.09554731845855713,
      "rewards/rejected": -0.23520700633525848,
      "step": 743
    },
    {
      "epoch": 1.1717888100866825,
      "grad_norm": 0.2399207502603531,
      "learning_rate": 1.7559885092194058e-06,
      "log_odds_chosen": 0.8169732689857483,
      "log_odds_ratio": -0.3689773976802826,
      "logits/chosen": -0.2764904499053955,
      "logits/rejected": -0.9976515769958496,
      "logps/chosen": -1.5530779361724854,
      "logps/rejected": -2.2440083026885986,
      "loss": 1.6027,
      "nll_loss": 1.5658522844314575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15530778467655182,
      "rewards/margins": 0.06909306347370148,
      "rewards/rejected": -0.2244008481502533,
      "step": 744
    },
    {
      "epoch": 1.173364854215918,
      "grad_norm": 0.19835339486598969,
      "learning_rate": 1.7505334044457871e-06,
      "log_odds_chosen": 1.0452640056610107,
      "log_odds_ratio": -0.3043675124645233,
      "logits/chosen": -0.3055506646633148,
      "logits/rejected": -0.9725714921951294,
      "logps/chosen": -1.434809923171997,
      "logps/rejected": -2.3075952529907227,
      "loss": 1.4733,
      "nll_loss": 1.4428149461746216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14348098635673523,
      "rewards/margins": 0.08727851510047913,
      "rewards/rejected": -0.23075951635837555,
      "step": 745
    },
    {
      "epoch": 1.1749408983451537,
      "grad_norm": 0.19556960463523865,
      "learning_rate": 1.7450801842731443e-06,
      "log_odds_chosen": 1.0478253364562988,
      "log_odds_ratio": -0.3036004900932312,
      "logits/chosen": -0.24023482203483582,
      "logits/rejected": -0.9085336327552795,
      "logps/chosen": -1.5535818338394165,
      "logps/rejected": -2.454700231552124,
      "loss": 1.5883,
      "nll_loss": 1.5579264163970947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553581953048706,
      "rewards/margins": 0.09011184424161911,
      "rewards/rejected": -0.24547003209590912,
      "step": 746
    },
    {
      "epoch": 1.1765169424743893,
      "grad_norm": 0.21798832714557648,
      "learning_rate": 1.7396288898979497e-06,
      "log_odds_chosen": 1.2348870038986206,
      "log_odds_ratio": -0.26652491092681885,
      "logits/chosen": -0.3237140476703644,
      "logits/rejected": -0.9812942147254944,
      "logps/chosen": -1.4410120248794556,
      "logps/rejected": -2.4953722953796387,
      "loss": 1.4777,
      "nll_loss": 1.4510746002197266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14410121738910675,
      "rewards/margins": 0.10543602705001831,
      "rewards/rejected": -0.24953722953796387,
      "step": 747
    },
    {
      "epoch": 1.178092986603625,
      "grad_norm": 0.21649590134620667,
      "learning_rate": 1.7341795625021292e-06,
      "log_odds_chosen": 1.1358311176300049,
      "log_odds_ratio": -0.28475719690322876,
      "logits/chosen": -0.31285732984542847,
      "logits/rejected": -1.0779106616973877,
      "logps/chosen": -1.4806733131408691,
      "logps/rejected": -2.449423313140869,
      "loss": 1.526,
      "nll_loss": 1.4975168704986572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14806735515594482,
      "rewards/margins": 0.09687498956918716,
      "rewards/rejected": -0.24494235217571259,
      "step": 748
    },
    {
      "epoch": 1.1796690307328606,
      "grad_norm": 0.2152174711227417,
      "learning_rate": 1.7287322432527485e-06,
      "log_odds_chosen": 1.0771859884262085,
      "log_odds_ratio": -0.3045428395271301,
      "logits/chosen": -0.35312578082084656,
      "logits/rejected": -1.044448971748352,
      "logps/chosen": -1.5623332262039185,
      "logps/rejected": -2.4927871227264404,
      "loss": 1.5944,
      "nll_loss": 1.5639197826385498,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15623332560062408,
      "rewards/margins": 0.09304539114236832,
      "rewards/rejected": -0.2492787092924118,
      "step": 749
    },
    {
      "epoch": 1.1812450748620962,
      "grad_norm": 0.22221341729164124,
      "learning_rate": 1.7232869733017038e-06,
      "log_odds_chosen": 1.0230644941329956,
      "log_odds_ratio": -0.3207840323448181,
      "logits/chosen": -0.3168463408946991,
      "logits/rejected": -1.0112690925598145,
      "logps/chosen": -1.519090175628662,
      "logps/rejected": -2.392937183380127,
      "loss": 1.5537,
      "nll_loss": 1.521660327911377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1519090235233307,
      "rewards/margins": 0.08738471567630768,
      "rewards/rejected": -0.23929372429847717,
      "step": 750
    },
    {
      "epoch": 1.1828211189913318,
      "grad_norm": 0.22188982367515564,
      "learning_rate": 1.7178437937854065e-06,
      "log_odds_chosen": 1.2587039470672607,
      "log_odds_ratio": -0.25582072138786316,
      "logits/chosen": -0.40541863441467285,
      "logits/rejected": -1.2030413150787354,
      "logps/chosen": -1.5576139688491821,
      "logps/rejected": -2.6524195671081543,
      "loss": 1.5871,
      "nll_loss": 1.561525821685791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15576137602329254,
      "rewards/margins": 0.10948055982589722,
      "rewards/rejected": -0.26524195075035095,
      "step": 751
    },
    {
      "epoch": 1.1843971631205674,
      "grad_norm": 0.25049683451652527,
      "learning_rate": 1.7124027458244794e-06,
      "log_odds_chosen": 1.092215657234192,
      "log_odds_ratio": -0.29998135566711426,
      "logits/chosen": -0.2870887219905853,
      "logits/rejected": -0.8463888764381409,
      "logps/chosen": -1.4843456745147705,
      "logps/rejected": -2.4180421829223633,
      "loss": 1.5326,
      "nll_loss": 1.5026021003723145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1484345644712448,
      "rewards/margins": 0.09336966276168823,
      "rewards/rejected": -0.24180422723293304,
      "step": 752
    },
    {
      "epoch": 1.185973207249803,
      "grad_norm": 0.19952140748500824,
      "learning_rate": 1.7069638705234407e-06,
      "log_odds_chosen": 0.8061379194259644,
      "log_odds_ratio": -0.37425217032432556,
      "logits/chosen": -0.2131144106388092,
      "logits/rejected": -0.9055432081222534,
      "logps/chosen": -1.5489134788513184,
      "logps/rejected": -2.2303953170776367,
      "loss": 1.5987,
      "nll_loss": 1.5613164901733398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15489134192466736,
      "rewards/margins": 0.068148173391819,
      "rewards/rejected": -0.22303953766822815,
      "step": 753
    },
    {
      "epoch": 1.1875492513790387,
      "grad_norm": 0.21154476702213287,
      "learning_rate": 1.7015272089703954e-06,
      "log_odds_chosen": 1.0792686939239502,
      "log_odds_ratio": -0.30500948429107666,
      "logits/chosen": -0.3588353991508484,
      "logits/rejected": -0.9874804019927979,
      "logps/chosen": -1.520809531211853,
      "logps/rejected": -2.4411962032318115,
      "loss": 1.5492,
      "nll_loss": 1.518701195716858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1520809531211853,
      "rewards/margins": 0.09203866124153137,
      "rewards/rejected": -0.24411962926387787,
      "step": 754
    },
    {
      "epoch": 1.1891252955082743,
      "grad_norm": 0.20993748307228088,
      "learning_rate": 1.6960928022367261e-06,
      "log_odds_chosen": 1.0256482362747192,
      "log_odds_ratio": -0.32212212681770325,
      "logits/chosen": -0.34558719396591187,
      "logits/rejected": -0.942269504070282,
      "logps/chosen": -1.5490738153457642,
      "logps/rejected": -2.436086416244507,
      "loss": 1.5837,
      "nll_loss": 1.5515347719192505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15490739047527313,
      "rewards/margins": 0.08870124816894531,
      "rewards/rejected": -0.24360863864421844,
      "step": 755
    },
    {
      "epoch": 1.19070133963751,
      "grad_norm": 0.19960153102874756,
      "learning_rate": 1.6906606913767776e-06,
      "log_odds_chosen": 0.9694607853889465,
      "log_odds_ratio": -0.33017948269844055,
      "logits/chosen": -0.36202937364578247,
      "logits/rejected": -1.0632708072662354,
      "logps/chosen": -1.445634126663208,
      "logps/rejected": -2.259155035018921,
      "loss": 1.4912,
      "nll_loss": 1.458221435546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14456342160701752,
      "rewards/margins": 0.081352099776268,
      "rewards/rejected": -0.22591552138328552,
      "step": 756
    },
    {
      "epoch": 1.1922773837667455,
      "grad_norm": 0.22965334355831146,
      "learning_rate": 1.6852309174275543e-06,
      "log_odds_chosen": 0.9793221354484558,
      "log_odds_ratio": -0.32375243306159973,
      "logits/chosen": -0.45830801129341125,
      "logits/rejected": -0.9744352698326111,
      "logps/chosen": -1.3762973546981812,
      "logps/rejected": -2.18276309967041,
      "loss": 1.42,
      "nll_loss": 1.3876392841339111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13762974739074707,
      "rewards/margins": 0.08064659684896469,
      "rewards/rejected": -0.21827633678913116,
      "step": 757
    },
    {
      "epoch": 1.1938534278959811,
      "grad_norm": 0.2170998603105545,
      "learning_rate": 1.6798035214084047e-06,
      "log_odds_chosen": 1.030111312866211,
      "log_odds_ratio": -0.31346821784973145,
      "logits/chosen": -0.406008780002594,
      "logits/rejected": -1.1310484409332275,
      "logps/chosen": -1.4797660112380981,
      "logps/rejected": -2.35377836227417,
      "loss": 1.525,
      "nll_loss": 1.493625521659851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479766070842743,
      "rewards/margins": 0.08740124106407166,
      "rewards/rejected": -0.23537784814834595,
      "step": 758
    },
    {
      "epoch": 1.1954294720252168,
      "grad_norm": 0.20952928066253662,
      "learning_rate": 1.674378544320714e-06,
      "log_odds_chosen": 1.2769091129302979,
      "log_odds_ratio": -0.2616688013076782,
      "logits/chosen": -0.2838149070739746,
      "logits/rejected": -1.012681007385254,
      "logps/chosen": -1.4348300695419312,
      "logps/rejected": -2.5192558765411377,
      "loss": 1.4805,
      "nll_loss": 1.454352617263794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1434830129146576,
      "rewards/margins": 0.10844258964061737,
      "rewards/rejected": -0.25192558765411377,
      "step": 759
    },
    {
      "epoch": 1.1970055161544524,
      "grad_norm": 0.21063999831676483,
      "learning_rate": 1.6689560271475922e-06,
      "log_odds_chosen": 1.1777784824371338,
      "log_odds_ratio": -0.27222558856010437,
      "logits/chosen": -0.3464505076408386,
      "logits/rejected": -1.0742390155792236,
      "logps/chosen": -1.4549981355667114,
      "logps/rejected": -2.4551632404327393,
      "loss": 1.49,
      "nll_loss": 1.4627617597579956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1454998105764389,
      "rewards/margins": 0.1000165045261383,
      "rewards/rejected": -0.24551630020141602,
      "step": 760
    },
    {
      "epoch": 1.198581560283688,
      "grad_norm": 0.19259801506996155,
      "learning_rate": 1.6635360108535665e-06,
      "log_odds_chosen": 1.1388490200042725,
      "log_odds_ratio": -0.28851547837257385,
      "logits/chosen": -0.36790522933006287,
      "logits/rejected": -1.033022403717041,
      "logps/chosen": -1.3951414823532104,
      "logps/rejected": -2.3535854816436768,
      "loss": 1.45,
      "nll_loss": 1.4211528301239014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13951414823532104,
      "rewards/margins": 0.09584441035985947,
      "rewards/rejected": -0.2353585809469223,
      "step": 761
    },
    {
      "epoch": 1.2001576044129236,
      "grad_norm": 0.2085634022951126,
      "learning_rate": 1.6581185363842717e-06,
      "log_odds_chosen": 1.2075750827789307,
      "log_odds_ratio": -0.2794080972671509,
      "logits/chosen": -0.40932169556617737,
      "logits/rejected": -0.9855950474739075,
      "logps/chosen": -1.4529056549072266,
      "logps/rejected": -2.486283302307129,
      "loss": 1.4824,
      "nll_loss": 1.4545007944107056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452905833721161,
      "rewards/margins": 0.10333774983882904,
      "rewards/rejected": -0.24862834811210632,
      "step": 762
    },
    {
      "epoch": 1.2017336485421592,
      "grad_norm": 0.23847652971744537,
      "learning_rate": 1.6527036446661393e-06,
      "log_odds_chosen": 1.0412486791610718,
      "log_odds_ratio": -0.31054016947746277,
      "logits/chosen": -0.3471706807613373,
      "logits/rejected": -1.056304931640625,
      "logps/chosen": -1.4727274179458618,
      "logps/rejected": -2.354166030883789,
      "loss": 1.5103,
      "nll_loss": 1.4792630672454834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1472727358341217,
      "rewards/margins": 0.08814386278390884,
      "rewards/rejected": -0.23541662096977234,
      "step": 763
    },
    {
      "epoch": 1.2033096926713949,
      "grad_norm": 0.2165566384792328,
      "learning_rate": 1.6472913766060901e-06,
      "log_odds_chosen": 1.2068313360214233,
      "log_odds_ratio": -0.27096161246299744,
      "logits/chosen": -0.48877644538879395,
      "logits/rejected": -1.0003821849822998,
      "logps/chosen": -1.4027626514434814,
      "logps/rejected": -2.4139833450317383,
      "loss": 1.4479,
      "nll_loss": 1.4208524227142334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14027628302574158,
      "rewards/margins": 0.10112209618091583,
      "rewards/rejected": -0.24139834940433502,
      "step": 764
    },
    {
      "epoch": 1.2048857368006305,
      "grad_norm": 0.21084967255592346,
      "learning_rate": 1.6418817730912252e-06,
      "log_odds_chosen": 0.9497586488723755,
      "log_odds_ratio": -0.34231606125831604,
      "logits/chosen": -0.4161587357521057,
      "logits/rejected": -0.8635929822921753,
      "logps/chosen": -1.5099153518676758,
      "logps/rejected": -2.3155694007873535,
      "loss": 1.5498,
      "nll_loss": 1.5155656337738037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1509915441274643,
      "rewards/margins": 0.08056540042161942,
      "rewards/rejected": -0.2315569519996643,
      "step": 765
    },
    {
      "epoch": 1.2064617809298661,
      "grad_norm": 0.2167016714811325,
      "learning_rate": 1.6364748749885133e-06,
      "log_odds_chosen": 0.8451948761940002,
      "log_odds_ratio": -0.36133021116256714,
      "logits/chosen": -0.40031933784484863,
      "logits/rejected": -0.9021502733230591,
      "logps/chosen": -1.467913031578064,
      "logps/rejected": -2.1741831302642822,
      "loss": 1.5078,
      "nll_loss": 1.4716212749481201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14679132401943207,
      "rewards/margins": 0.07062703371047974,
      "rewards/rejected": -0.21741832792758942,
      "step": 766
    },
    {
      "epoch": 1.2080378250591017,
      "grad_norm": 0.4888826012611389,
      "learning_rate": 1.6310707231444883e-06,
      "log_odds_chosen": 1.0665688514709473,
      "log_odds_ratio": -0.30518874526023865,
      "logits/chosen": -0.3968595266342163,
      "logits/rejected": -0.8935273289680481,
      "logps/chosen": -1.4759632349014282,
      "logps/rejected": -2.382444143295288,
      "loss": 1.5089,
      "nll_loss": 1.4783427715301514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1475963294506073,
      "rewards/margins": 0.09064808487892151,
      "rewards/rejected": -0.2382444143295288,
      "step": 767
    },
    {
      "epoch": 1.2096138691883374,
      "grad_norm": 0.21818922460079193,
      "learning_rate": 1.625669358384936e-06,
      "log_odds_chosen": 0.8351269960403442,
      "log_odds_ratio": -0.3715553283691406,
      "logits/chosen": -0.3066043257713318,
      "logits/rejected": -0.7817680239677429,
      "logps/chosen": -1.5343680381774902,
      "logps/rejected": -2.240647315979004,
      "loss": 1.5789,
      "nll_loss": 1.5417516231536865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1534368097782135,
      "rewards/margins": 0.07062795013189316,
      "rewards/rejected": -0.22406475245952606,
      "step": 768
    },
    {
      "epoch": 1.211189913317573,
      "grad_norm": 0.2206609547138214,
      "learning_rate": 1.620270821514587e-06,
      "log_odds_chosen": 1.1112496852874756,
      "log_odds_ratio": -0.2954919934272766,
      "logits/chosen": -0.27211764454841614,
      "logits/rejected": -0.959216833114624,
      "logps/chosen": -1.5633149147033691,
      "logps/rejected": -2.5282092094421387,
      "loss": 1.5881,
      "nll_loss": 1.5585979223251343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15633150935173035,
      "rewards/margins": 0.09648942202329636,
      "rewards/rejected": -0.2528209090232849,
      "step": 769
    },
    {
      "epoch": 1.2127659574468086,
      "grad_norm": 0.2250218242406845,
      "learning_rate": 1.6148751533168104e-06,
      "log_odds_chosen": 1.4176288843154907,
      "log_odds_ratio": -0.23163941502571106,
      "logits/chosen": -0.3990238904953003,
      "logits/rejected": -1.0994439125061035,
      "logps/chosen": -1.5316133499145508,
      "logps/rejected": -2.772145986557007,
      "loss": 1.5493,
      "nll_loss": 1.526184320449829,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15316133201122284,
      "rewards/margins": 0.12405325472354889,
      "rewards/rejected": -0.27721458673477173,
      "step": 770
    },
    {
      "epoch": 1.2143420015760442,
      "grad_norm": 0.21237164735794067,
      "learning_rate": 1.6094823945532996e-06,
      "log_odds_chosen": 1.1367987394332886,
      "log_odds_ratio": -0.29999420046806335,
      "logits/chosen": -0.4410712420940399,
      "logits/rejected": -0.9614903926849365,
      "logps/chosen": -1.4790409803390503,
      "logps/rejected": -2.4555158615112305,
      "loss": 1.5085,
      "nll_loss": 1.4784656763076782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14790409803390503,
      "rewards/margins": 0.09764750301837921,
      "rewards/rejected": -0.24555160105228424,
      "step": 771
    },
    {
      "epoch": 1.2159180457052798,
      "grad_norm": 0.2274480015039444,
      "learning_rate": 1.6040925859637728e-06,
      "log_odds_chosen": 1.081011414527893,
      "log_odds_ratio": -0.30805137753486633,
      "logits/chosen": -0.40089038014411926,
      "logits/rejected": -0.9306033253669739,
      "logps/chosen": -1.5680983066558838,
      "logps/rejected": -2.5017659664154053,
      "loss": 1.6001,
      "nll_loss": 1.5693408250808716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15680982172489166,
      "rewards/margins": 0.09336678683757782,
      "rewards/rejected": -0.2501766085624695,
      "step": 772
    },
    {
      "epoch": 1.2174940898345155,
      "grad_norm": 0.24597373604774475,
      "learning_rate": 1.5987057682656596e-06,
      "log_odds_chosen": 0.9743691682815552,
      "log_odds_ratio": -0.32275116443634033,
      "logits/chosen": -0.30330225825309753,
      "logits/rejected": -0.8476285934448242,
      "logps/chosen": -1.6149225234985352,
      "logps/rejected": -2.457552909851074,
      "loss": 1.6507,
      "nll_loss": 1.6184513568878174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.161492258310318,
      "rewards/margins": 0.08426303416490555,
      "rewards/rejected": -0.24575528502464294,
      "step": 773
    },
    {
      "epoch": 1.219070133963751,
      "grad_norm": 0.24178926646709442,
      "learning_rate": 1.593321982153795e-06,
      "log_odds_chosen": 1.0440930128097534,
      "log_odds_ratio": -0.30693989992141724,
      "logits/chosen": -0.4126613736152649,
      "logits/rejected": -1.0729434490203857,
      "logps/chosen": -1.6260238885879517,
      "logps/rejected": -2.5329277515411377,
      "loss": 1.6472,
      "nll_loss": 1.6164599657058716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16260237991809845,
      "rewards/margins": 0.09069040417671204,
      "rewards/rejected": -0.2532927989959717,
      "step": 774
    },
    {
      "epoch": 1.2206461780929867,
      "grad_norm": 0.23748508095741272,
      "learning_rate": 1.5879412683001106e-06,
      "log_odds_chosen": 0.8963701128959656,
      "log_odds_ratio": -0.3483182489871979,
      "logits/chosen": -0.41866371035575867,
      "logits/rejected": -0.8604896068572998,
      "logps/chosen": -1.4483559131622314,
      "logps/rejected": -2.1963205337524414,
      "loss": 1.5028,
      "nll_loss": 1.4679374694824219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14483557641506195,
      "rewards/margins": 0.07479647547006607,
      "rewards/rejected": -0.2196320742368698,
      "step": 775
    },
    {
      "epoch": 1.2222222222222223,
      "grad_norm": 0.2218731790781021,
      "learning_rate": 1.5825636673533298e-06,
      "log_odds_chosen": 1.3701053857803345,
      "log_odds_ratio": -0.23631651699543,
      "logits/chosen": -0.46909186244010925,
      "logits/rejected": -1.2042089700698853,
      "logps/chosen": -1.5150604248046875,
      "logps/rejected": -2.706639289855957,
      "loss": 1.5275,
      "nll_loss": 1.503898024559021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15150603652000427,
      "rewards/margins": 0.11915791034698486,
      "rewards/rejected": -0.27066394686698914,
      "step": 776
    },
    {
      "epoch": 1.2237982663514577,
      "grad_norm": 0.23912036418914795,
      "learning_rate": 1.5771892199386598e-06,
      "log_odds_chosen": 1.0191112756729126,
      "log_odds_ratio": -0.31965747475624084,
      "logits/chosen": -0.37492820620536804,
      "logits/rejected": -0.8714897036552429,
      "logps/chosen": -1.5365045070648193,
      "logps/rejected": -2.4136791229248047,
      "loss": 1.5855,
      "nll_loss": 1.553579330444336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1536504477262497,
      "rewards/margins": 0.0877174660563469,
      "rewards/rejected": -0.241367906332016,
      "step": 777
    },
    {
      "epoch": 1.2253743104806936,
      "grad_norm": 0.20285028219223022,
      "learning_rate": 1.5718179666574834e-06,
      "log_odds_chosen": 1.3285276889801025,
      "log_odds_ratio": -0.24557870626449585,
      "logits/chosen": -0.2986782491207123,
      "logits/rejected": -1.091235637664795,
      "logps/chosen": -1.4959912300109863,
      "logps/rejected": -2.6467878818511963,
      "loss": 1.5343,
      "nll_loss": 1.5097694396972656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495991200208664,
      "rewards/margins": 0.11507965624332428,
      "rewards/rejected": -0.2646787762641907,
      "step": 778
    },
    {
      "epoch": 1.226950354609929,
      "grad_norm": 0.22980020940303802,
      "learning_rate": 1.5664499480870539e-06,
      "log_odds_chosen": 1.3184008598327637,
      "log_odds_ratio": -0.24481111764907837,
      "logits/chosen": -0.3474910855293274,
      "logits/rejected": -1.0733544826507568,
      "logps/chosen": -1.5222991704940796,
      "logps/rejected": -2.666215419769287,
      "loss": 1.5637,
      "nll_loss": 1.5392487049102783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.152229905128479,
      "rewards/margins": 0.11439161002635956,
      "rewards/rejected": -0.26662153005599976,
      "step": 779
    },
    {
      "epoch": 1.2285263987391648,
      "grad_norm": 0.22775126993656158,
      "learning_rate": 1.5610852047801875e-06,
      "log_odds_chosen": 0.9218294024467468,
      "log_odds_ratio": -0.340370774269104,
      "logits/chosen": -0.425527960062027,
      "logits/rejected": -0.9289211630821228,
      "logps/chosen": -1.4561243057250977,
      "logps/rejected": -2.230318784713745,
      "loss": 1.4869,
      "nll_loss": 1.4528659582138062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1456124186515808,
      "rewards/margins": 0.07741944491863251,
      "rewards/rejected": -0.2230318784713745,
      "step": 780
    },
    {
      "epoch": 1.2301024428684002,
      "grad_norm": 0.21928805112838745,
      "learning_rate": 1.5557237772649567e-06,
      "log_odds_chosen": 0.8913912773132324,
      "log_odds_ratio": -0.34962329268455505,
      "logits/chosen": -0.31643298268318176,
      "logits/rejected": -0.9967266917228699,
      "logps/chosen": -1.5394333600997925,
      "logps/rejected": -2.295088768005371,
      "loss": 1.5803,
      "nll_loss": 1.5453306436538696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15394333004951477,
      "rewards/margins": 0.07556554675102234,
      "rewards/rejected": -0.22950886189937592,
      "step": 781
    },
    {
      "epoch": 1.231678486997636,
      "grad_norm": 0.20588098466396332,
      "learning_rate": 1.5503657060443866e-06,
      "log_odds_chosen": 1.060440182685852,
      "log_odds_ratio": -0.3107547461986542,
      "logits/chosen": -0.3517206311225891,
      "logits/rejected": -1.0006048679351807,
      "logps/chosen": -1.4581408500671387,
      "logps/rejected": -2.352829933166504,
      "loss": 1.5086,
      "nll_loss": 1.477475643157959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14581409096717834,
      "rewards/margins": 0.08946886658668518,
      "rewards/rejected": -0.23528297245502472,
      "step": 782
    },
    {
      "epoch": 1.2332545311268714,
      "grad_norm": 0.2089521586894989,
      "learning_rate": 1.5450110315961457e-06,
      "log_odds_chosen": 1.2488126754760742,
      "log_odds_ratio": -0.2691247761249542,
      "logits/chosen": -0.3829159736633301,
      "logits/rejected": -1.056909441947937,
      "logps/chosen": -1.4950027465820312,
      "logps/rejected": -2.5737485885620117,
      "loss": 1.5423,
      "nll_loss": 1.5154030323028564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495002657175064,
      "rewards/margins": 0.10787460207939148,
      "rewards/rejected": -0.2573748826980591,
      "step": 783
    },
    {
      "epoch": 1.2348305752561073,
      "grad_norm": 0.21984124183654785,
      "learning_rate": 1.539659794372243e-06,
      "log_odds_chosen": 1.3012444972991943,
      "log_odds_ratio": -0.27216094732284546,
      "logits/chosen": -0.4334256649017334,
      "logits/rejected": -1.0410724878311157,
      "logps/chosen": -1.4989289045333862,
      "logps/rejected": -2.63016676902771,
      "loss": 1.5344,
      "nll_loss": 1.507155179977417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1498928964138031,
      "rewards/margins": 0.11312378942966461,
      "rewards/rejected": -0.2630167007446289,
      "step": 784
    },
    {
      "epoch": 1.2364066193853427,
      "grad_norm": 0.2202092707157135,
      "learning_rate": 1.5343120347987172e-06,
      "log_odds_chosen": 1.211161494255066,
      "log_odds_ratio": -0.2643766701221466,
      "logits/chosen": -0.39297932386398315,
      "logits/rejected": -1.1359959840774536,
      "logps/chosen": -1.4870625734329224,
      "logps/rejected": -2.525190591812134,
      "loss": 1.5385,
      "nll_loss": 1.5120375156402588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14870627224445343,
      "rewards/margins": 0.1038127988576889,
      "rewards/rejected": -0.25251907110214233,
      "step": 785
    },
    {
      "epoch": 1.2379826635145785,
      "grad_norm": 0.21030808985233307,
      "learning_rate": 1.5289677932753398e-06,
      "log_odds_chosen": 0.9153645038604736,
      "log_odds_ratio": -0.3461787700653076,
      "logits/chosen": -0.3987579345703125,
      "logits/rejected": -0.9754417538642883,
      "logps/chosen": -1.5185397863388062,
      "logps/rejected": -2.293330192565918,
      "loss": 1.559,
      "nll_loss": 1.5243672132492065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1518539935350418,
      "rewards/margins": 0.0774790346622467,
      "rewards/rejected": -0.22933301329612732,
      "step": 786
    },
    {
      "epoch": 1.239558707643814,
      "grad_norm": 0.21333414316177368,
      "learning_rate": 1.5236271101753017e-06,
      "log_odds_chosen": 1.1289535760879517,
      "log_odds_ratio": -0.28179311752319336,
      "logits/chosen": -0.4149124026298523,
      "logits/rejected": -1.0306938886642456,
      "logps/chosen": -1.5197263956069946,
      "logps/rejected": -2.486274242401123,
      "loss": 1.544,
      "nll_loss": 1.5158296823501587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15197263658046722,
      "rewards/margins": 0.09665478765964508,
      "rewards/rejected": -0.2486274242401123,
      "step": 787
    },
    {
      "epoch": 1.2411347517730495,
      "grad_norm": 0.22445255517959595,
      "learning_rate": 1.5182900258449135e-06,
      "log_odds_chosen": 1.2380210161209106,
      "log_odds_ratio": -0.2605192959308624,
      "logits/chosen": -0.4028743803501129,
      "logits/rejected": -1.1445603370666504,
      "logps/chosen": -1.502464771270752,
      "logps/rejected": -2.569650173187256,
      "loss": 1.539,
      "nll_loss": 1.5129010677337646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1502464860677719,
      "rewards/margins": 0.10671855509281158,
      "rewards/rejected": -0.2569650411605835,
      "step": 788
    },
    {
      "epoch": 1.2427107959022852,
      "grad_norm": 0.22204920649528503,
      "learning_rate": 1.5129565806032986e-06,
      "log_odds_chosen": 1.222856879234314,
      "log_odds_ratio": -0.269106388092041,
      "logits/chosen": -0.39869534969329834,
      "logits/rejected": -0.9980011582374573,
      "logps/chosen": -1.3870023488998413,
      "logps/rejected": -2.418531894683838,
      "loss": 1.441,
      "nll_loss": 1.41408371925354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13870024681091309,
      "rewards/margins": 0.10315297544002533,
      "rewards/rejected": -0.24185320734977722,
      "step": 789
    },
    {
      "epoch": 1.2442868400315208,
      "grad_norm": 0.24877521395683289,
      "learning_rate": 1.507626814742087e-06,
      "log_odds_chosen": 1.0394840240478516,
      "log_odds_ratio": -0.317719042301178,
      "logits/chosen": -0.4166167378425598,
      "logits/rejected": -0.9083614945411682,
      "logps/chosen": -1.5805855989456177,
      "logps/rejected": -2.477033853530884,
      "loss": 1.6226,
      "nll_loss": 1.5907845497131348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1580585539340973,
      "rewards/margins": 0.08964481949806213,
      "rewards/rejected": -0.24770338833332062,
      "step": 790
    },
    {
      "epoch": 1.2458628841607564,
      "grad_norm": 0.21424908936023712,
      "learning_rate": 1.502300768525115e-06,
      "log_odds_chosen": 1.1430931091308594,
      "log_odds_ratio": -0.3005574345588684,
      "logits/chosen": -0.3877500295639038,
      "logits/rejected": -1.0552709102630615,
      "logps/chosen": -1.590660572052002,
      "logps/rejected": -2.5871505737304688,
      "loss": 1.5983,
      "nll_loss": 1.5682021379470825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1590660661458969,
      "rewards/margins": 0.09964899718761444,
      "rewards/rejected": -0.25871506333351135,
      "step": 791
    },
    {
      "epoch": 1.247438928289992,
      "grad_norm": 0.2195267528295517,
      "learning_rate": 1.4969784821881177e-06,
      "log_odds_chosen": 0.9598576426506042,
      "log_odds_ratio": -0.33298584818840027,
      "logits/chosen": -0.4250534772872925,
      "logits/rejected": -0.8964757919311523,
      "logps/chosen": -1.4540200233459473,
      "logps/rejected": -2.2579970359802246,
      "loss": 1.5156,
      "nll_loss": 1.4823079109191895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1454019993543625,
      "rewards/margins": 0.08039772510528564,
      "rewards/rejected": -0.22579973936080933,
      "step": 792
    },
    {
      "epoch": 1.2490149724192277,
      "grad_norm": 0.21902096271514893,
      "learning_rate": 1.4916599959384262e-06,
      "log_odds_chosen": 0.9551348686218262,
      "log_odds_ratio": -0.330089271068573,
      "logits/chosen": -0.4277741014957428,
      "logits/rejected": -1.0719842910766602,
      "logps/chosen": -1.533909559249878,
      "logps/rejected": -2.346613883972168,
      "loss": 1.5666,
      "nll_loss": 1.5335662364959717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15339095890522003,
      "rewards/margins": 0.08127044141292572,
      "rewards/rejected": -0.23466140031814575,
      "step": 793
    },
    {
      "epoch": 1.2505910165484633,
      "grad_norm": 0.23156818747520447,
      "learning_rate": 1.4863453499546643e-06,
      "log_odds_chosen": 1.30789315700531,
      "log_odds_ratio": -0.25488516688346863,
      "logits/chosen": -0.35893774032592773,
      "logits/rejected": -1.023228406906128,
      "logps/chosen": -1.4720572233200073,
      "logps/rejected": -2.601604461669922,
      "loss": 1.5069,
      "nll_loss": 1.4814491271972656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14720572531223297,
      "rewards/margins": 0.11295472085475922,
      "rewards/rejected": -0.2601604461669922,
      "step": 794
    },
    {
      "epoch": 1.252167060677699,
      "grad_norm": 0.2238602340221405,
      "learning_rate": 1.4810345843864427e-06,
      "log_odds_chosen": 1.102327585220337,
      "log_odds_ratio": -0.29942479729652405,
      "logits/chosen": -0.4538511633872986,
      "logits/rejected": -0.9794531464576721,
      "logps/chosen": -1.529341697692871,
      "logps/rejected": -2.4750523567199707,
      "loss": 1.5681,
      "nll_loss": 1.5381572246551514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15293416380882263,
      "rewards/margins": 0.09457109123468399,
      "rewards/rejected": -0.24750526249408722,
      "step": 795
    },
    {
      "epoch": 1.2537431048069345,
      "grad_norm": 0.22297698259353638,
      "learning_rate": 1.4757277393540598e-06,
      "log_odds_chosen": 1.1080116033554077,
      "log_odds_ratio": -0.30314451456069946,
      "logits/chosen": -0.38277140259742737,
      "logits/rejected": -0.85636305809021,
      "logps/chosen": -1.5176854133605957,
      "logps/rejected": -2.471342086791992,
      "loss": 1.5639,
      "nll_loss": 1.5336213111877441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15176856517791748,
      "rewards/margins": 0.09536562860012054,
      "rewards/rejected": -0.24713419377803802,
      "step": 796
    },
    {
      "epoch": 1.2553191489361701,
      "grad_norm": 0.2200615257024765,
      "learning_rate": 1.4704248549481946e-06,
      "log_odds_chosen": 1.0324604511260986,
      "log_odds_ratio": -0.3196156919002533,
      "logits/chosen": -0.4039994776248932,
      "logits/rejected": -1.145932912826538,
      "logps/chosen": -1.577517032623291,
      "logps/rejected": -2.47335147857666,
      "loss": 1.6089,
      "nll_loss": 1.5769336223602295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1577516794204712,
      "rewards/margins": 0.08958344906568527,
      "rewards/rejected": -0.24733513593673706,
      "step": 797
    },
    {
      "epoch": 1.2568951930654058,
      "grad_norm": 0.20684649050235748,
      "learning_rate": 1.4651259712296063e-06,
      "log_odds_chosen": 1.2410458326339722,
      "log_odds_ratio": -0.25976118445396423,
      "logits/chosen": -0.33508533239364624,
      "logits/rejected": -0.9718801975250244,
      "logps/chosen": -1.5489780902862549,
      "logps/rejected": -2.625112295150757,
      "loss": 1.5773,
      "nll_loss": 1.5513187646865845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15489782392978668,
      "rewards/margins": 0.10761341452598572,
      "rewards/rejected": -0.2625112235546112,
      "step": 798
    },
    {
      "epoch": 1.2584712371946414,
      "grad_norm": 0.2147311121225357,
      "learning_rate": 1.45983112822883e-06,
      "log_odds_chosen": 1.037184238433838,
      "log_odds_ratio": -0.3077075779438019,
      "logits/chosen": -0.3195713758468628,
      "logits/rejected": -0.9489158391952515,
      "logps/chosen": -1.4682114124298096,
      "logps/rejected": -2.345653772354126,
      "loss": 1.5119,
      "nll_loss": 1.481153964996338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14682115614414215,
      "rewards/margins": 0.08774422854185104,
      "rewards/rejected": -0.2345653772354126,
      "step": 799
    },
    {
      "epoch": 1.260047281323877,
      "grad_norm": 0.22592990100383759,
      "learning_rate": 1.4545403659458756e-06,
      "log_odds_chosen": 1.0629757642745972,
      "log_odds_ratio": -0.30687931180000305,
      "logits/chosen": -0.4144957363605499,
      "logits/rejected": -0.9237263202667236,
      "logps/chosen": -1.4850565195083618,
      "logps/rejected": -2.389726161956787,
      "loss": 1.527,
      "nll_loss": 1.4962794780731201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14850565791130066,
      "rewards/margins": 0.0904669538140297,
      "rewards/rejected": -0.23897261917591095,
      "step": 800
    },
    {
      "epoch": 1.2616233254531126,
      "grad_norm": 0.2108520120382309,
      "learning_rate": 1.4492537243499253e-06,
      "log_odds_chosen": 1.2146984338760376,
      "log_odds_ratio": -0.27139827609062195,
      "logits/chosen": -0.4225751757621765,
      "logits/rejected": -0.9720625877380371,
      "logps/chosen": -1.4731156826019287,
      "logps/rejected": -2.5078656673431396,
      "loss": 1.4963,
      "nll_loss": 1.4692060947418213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14731155335903168,
      "rewards/margins": 0.10347502678632736,
      "rewards/rejected": -0.25078657269477844,
      "step": 801
    },
    {
      "epoch": 1.2631993695823482,
      "grad_norm": 0.20737168192863464,
      "learning_rate": 1.443971243379031e-06,
      "log_odds_chosen": 1.155271291732788,
      "log_odds_ratio": -0.2857162058353424,
      "logits/chosen": -0.3342997133731842,
      "logits/rejected": -0.986181378364563,
      "logps/chosen": -1.5114483833312988,
      "logps/rejected": -2.5011544227600098,
      "loss": 1.5425,
      "nll_loss": 1.5138931274414062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1511448472738266,
      "rewards/margins": 0.09897062182426453,
      "rewards/rejected": -0.25011545419692993,
      "step": 802
    },
    {
      "epoch": 1.2647754137115839,
      "grad_norm": 0.21325348317623138,
      "learning_rate": 1.4386929629398144e-06,
      "log_odds_chosen": 1.2907012701034546,
      "log_odds_ratio": -0.24939501285552979,
      "logits/chosen": -0.3950750231742859,
      "logits/rejected": -1.1367483139038086,
      "logps/chosen": -1.540709376335144,
      "logps/rejected": -2.662118673324585,
      "loss": 1.5693,
      "nll_loss": 1.5443140268325806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1540709286928177,
      "rewards/margins": 0.1121409460902214,
      "rewards/rejected": -0.2662118673324585,
      "step": 803
    },
    {
      "epoch": 1.2663514578408195,
      "grad_norm": 0.21124649047851562,
      "learning_rate": 1.4334189229071614e-06,
      "log_odds_chosen": 1.087626338005066,
      "log_odds_ratio": -0.3059476315975189,
      "logits/chosen": -0.4060670733451843,
      "logits/rejected": -0.9539061188697815,
      "logps/chosen": -1.436784029006958,
      "logps/rejected": -2.357896327972412,
      "loss": 1.4738,
      "nll_loss": 1.4432013034820557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14367841184139252,
      "rewards/margins": 0.09211122244596481,
      "rewards/rejected": -0.23578962683677673,
      "step": 804
    },
    {
      "epoch": 1.267927501970055,
      "grad_norm": 0.24813541769981384,
      "learning_rate": 1.4281491631239263e-06,
      "log_odds_chosen": 0.9352160692214966,
      "log_odds_ratio": -0.3426801562309265,
      "logits/chosen": -0.43386709690093994,
      "logits/rejected": -0.9372543096542358,
      "logps/chosen": -1.5825542211532593,
      "logps/rejected": -2.3885130882263184,
      "loss": 1.6043,
      "nll_loss": 1.5700563192367554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1582554131746292,
      "rewards/margins": 0.08059589564800262,
      "rewards/rejected": -0.23885130882263184,
      "step": 805
    },
    {
      "epoch": 1.2695035460992907,
      "grad_norm": 0.23052074015140533,
      "learning_rate": 1.4228837234006272e-06,
      "log_odds_chosen": 1.2309831380844116,
      "log_odds_ratio": -0.2637780010700226,
      "logits/chosen": -0.4448695778846741,
      "logits/rejected": -1.0244659185409546,
      "logps/chosen": -1.442327618598938,
      "logps/rejected": -2.494382381439209,
      "loss": 1.5026,
      "nll_loss": 1.476207971572876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14423276484012604,
      "rewards/margins": 0.1052054688334465,
      "rewards/rejected": -0.24943824112415314,
      "step": 806
    },
    {
      "epoch": 1.2710795902285263,
      "grad_norm": 0.22362235188484192,
      "learning_rate": 1.4176226435151462e-06,
      "log_odds_chosen": 1.0907524824142456,
      "log_odds_ratio": -0.3066290616989136,
      "logits/chosen": -0.41013288497924805,
      "logits/rejected": -1.0688410997390747,
      "logps/chosen": -1.5482828617095947,
      "logps/rejected": -2.492948293685913,
      "loss": 1.5674,
      "nll_loss": 1.5366919040679932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.154828280210495,
      "rewards/margins": 0.09446655213832855,
      "rewards/rejected": -0.24929484724998474,
      "step": 807
    },
    {
      "epoch": 1.272655634357762,
      "grad_norm": 0.20177769660949707,
      "learning_rate": 1.4123659632124298e-06,
      "log_odds_chosen": 1.2619973421096802,
      "log_odds_ratio": -0.25683704018592834,
      "logits/chosen": -0.45332908630371094,
      "logits/rejected": -1.0218454599380493,
      "logps/chosen": -1.5034123659133911,
      "logps/rejected": -2.5928595066070557,
      "loss": 1.5411,
      "nll_loss": 1.515439510345459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503412425518036,
      "rewards/margins": 0.10894472897052765,
      "rewards/rejected": -0.25928595662117004,
      "step": 808
    },
    {
      "epoch": 1.2742316784869976,
      "grad_norm": 0.21717409789562225,
      "learning_rate": 1.4071137222041852e-06,
      "log_odds_chosen": 1.243767261505127,
      "log_odds_ratio": -0.26813173294067383,
      "logits/chosen": -0.3485559821128845,
      "logits/rejected": -1.1612999439239502,
      "logps/chosen": -1.5613025426864624,
      "logps/rejected": -2.6439056396484375,
      "loss": 1.5886,
      "nll_loss": 1.5617454051971436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15613026916980743,
      "rewards/margins": 0.10826030373573303,
      "rewards/rejected": -0.26439058780670166,
      "step": 809
    },
    {
      "epoch": 1.2758077226162332,
      "grad_norm": 0.21898601949214935,
      "learning_rate": 1.4018659601685857e-06,
      "log_odds_chosen": 1.0675541162490845,
      "log_odds_ratio": -0.30271434783935547,
      "logits/chosen": -0.36096829175949097,
      "logits/rejected": -1.0590094327926636,
      "logps/chosen": -1.5490797758102417,
      "logps/rejected": -2.4695169925689697,
      "loss": 1.578,
      "nll_loss": 1.547766923904419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1549079865217209,
      "rewards/margins": 0.09204372763633728,
      "rewards/rejected": -0.24695171415805817,
      "step": 810
    },
    {
      "epoch": 1.2773837667454688,
      "grad_norm": 0.24523155391216278,
      "learning_rate": 1.3966227167499667e-06,
      "log_odds_chosen": 1.0590333938598633,
      "log_odds_ratio": -0.30275577306747437,
      "logits/chosen": -0.43062809109687805,
      "logits/rejected": -0.9160906672477722,
      "logps/chosen": -1.5956380367279053,
      "logps/rejected": -2.511876106262207,
      "loss": 1.6188,
      "nll_loss": 1.588538646697998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1595637947320938,
      "rewards/margins": 0.09162382781505585,
      "rewards/rejected": -0.25118762254714966,
      "step": 811
    },
    {
      "epoch": 1.2789598108747045,
      "grad_norm": 0.2194031924009323,
      "learning_rate": 1.3913840315585277e-06,
      "log_odds_chosen": 0.9953582286834717,
      "log_odds_ratio": -0.31567201018333435,
      "logits/chosen": -0.39321523904800415,
      "logits/rejected": -1.080066442489624,
      "logps/chosen": -1.5085080862045288,
      "logps/rejected": -2.353855609893799,
      "loss": 1.5483,
      "nll_loss": 1.5167378187179565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1508508026599884,
      "rewards/margins": 0.08453476428985596,
      "rewards/rejected": -0.23538556694984436,
      "step": 812
    },
    {
      "epoch": 1.28053585500394,
      "grad_norm": 0.23802867531776428,
      "learning_rate": 1.3861499441700337e-06,
      "log_odds_chosen": 1.257501482963562,
      "log_odds_ratio": -0.259028822183609,
      "logits/chosen": -0.51498943567276,
      "logits/rejected": -1.1061493158340454,
      "logps/chosen": -1.5957281589508057,
      "logps/rejected": -2.6983399391174316,
      "loss": 1.623,
      "nll_loss": 1.5970487594604492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1595728099346161,
      "rewards/margins": 0.11026117950677872,
      "rewards/rejected": -0.2698340117931366,
      "step": 813
    },
    {
      "epoch": 1.2821118991331757,
      "grad_norm": 0.2247147113084793,
      "learning_rate": 1.3809204941255144e-06,
      "log_odds_chosen": 1.2670109272003174,
      "log_odds_ratio": -0.2548001706600189,
      "logits/chosen": -0.4434301555156708,
      "logits/rejected": -1.1245754957199097,
      "logps/chosen": -1.4761030673980713,
      "logps/rejected": -2.563023567199707,
      "loss": 1.5115,
      "nll_loss": 1.4860399961471558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14761029183864594,
      "rewards/margins": 0.10869207233190536,
      "rewards/rejected": -0.2563023865222931,
      "step": 814
    },
    {
      "epoch": 1.2836879432624113,
      "grad_norm": 0.20461121201515198,
      "learning_rate": 1.3756957209309667e-06,
      "log_odds_chosen": 1.3737633228302002,
      "log_odds_ratio": -0.24145899713039398,
      "logits/chosen": -0.4477747678756714,
      "logits/rejected": -1.213658332824707,
      "logps/chosen": -1.513550877571106,
      "logps/rejected": -2.709103584289551,
      "loss": 1.5503,
      "nll_loss": 1.5261461734771729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1513551026582718,
      "rewards/margins": 0.1195552721619606,
      "rewards/rejected": -0.270910382270813,
      "step": 815
    },
    {
      "epoch": 1.285263987391647,
      "grad_norm": 0.23603622615337372,
      "learning_rate": 1.3704756640570575e-06,
      "log_odds_chosen": 1.4155566692352295,
      "log_odds_ratio": -0.23559989035129547,
      "logits/chosen": -0.46279430389404297,
      "logits/rejected": -1.0378425121307373,
      "logps/chosen": -1.4821847677230835,
      "logps/rejected": -2.7091140747070312,
      "loss": 1.5169,
      "nll_loss": 1.4933196306228638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14821848273277283,
      "rewards/margins": 0.12269290536642075,
      "rewards/rejected": -0.27091139554977417,
      "step": 816
    },
    {
      "epoch": 1.2868400315208826,
      "grad_norm": 0.21921786665916443,
      "learning_rate": 1.3652603629388224e-06,
      "log_odds_chosen": 1.2274490594863892,
      "log_odds_ratio": -0.27596333622932434,
      "logits/chosen": -0.3813299536705017,
      "logits/rejected": -1.0537049770355225,
      "logps/chosen": -1.4544984102249146,
      "logps/rejected": -2.4996795654296875,
      "loss": 1.4869,
      "nll_loss": 1.45933198928833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14544984698295593,
      "rewards/margins": 0.1045181155204773,
      "rewards/rejected": -0.24996794760227203,
      "step": 817
    },
    {
      "epoch": 1.2884160756501182,
      "grad_norm": 0.24252961575984955,
      "learning_rate": 1.3600498569753715e-06,
      "log_odds_chosen": 1.0021506547927856,
      "log_odds_ratio": -0.3139697015285492,
      "logits/chosen": -0.22751033306121826,
      "logits/rejected": -0.9177674055099487,
      "logps/chosen": -1.5088670253753662,
      "logps/rejected": -2.357330799102783,
      "loss": 1.5528,
      "nll_loss": 1.5214073657989502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15088672935962677,
      "rewards/margins": 0.0848463773727417,
      "rewards/rejected": -0.23573310673236847,
      "step": 818
    },
    {
      "epoch": 1.2899921197793538,
      "grad_norm": 0.235077366232872,
      "learning_rate": 1.3548441855295872e-06,
      "log_odds_chosen": 1.066481351852417,
      "log_odds_ratio": -0.300545871257782,
      "logits/chosen": -0.40409475564956665,
      "logits/rejected": -1.161527156829834,
      "logps/chosen": -1.5828237533569336,
      "logps/rejected": -2.5045242309570312,
      "loss": 1.6143,
      "nll_loss": 1.5842136144638062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15828239917755127,
      "rewards/margins": 0.09217005223035812,
      "rewards/rejected": -0.2504524290561676,
      "step": 819
    },
    {
      "epoch": 1.2915681639085894,
      "grad_norm": 0.235376238822937,
      "learning_rate": 1.3496433879278315e-06,
      "log_odds_chosen": 1.194473147392273,
      "log_odds_ratio": -0.2824646234512329,
      "logits/chosen": -0.3881264626979828,
      "logits/rejected": -0.9851077795028687,
      "logps/chosen": -1.6311043500900269,
      "logps/rejected": -2.6831674575805664,
      "loss": 1.6467,
      "nll_loss": 1.6184190511703491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1631104201078415,
      "rewards/margins": 0.10520632565021515,
      "rewards/rejected": -0.26831677556037903,
      "step": 820
    },
    {
      "epoch": 1.293144208037825,
      "grad_norm": 0.23449520766735077,
      "learning_rate": 1.3444475034596463e-06,
      "log_odds_chosen": 1.280067801475525,
      "log_odds_ratio": -0.25137269496917725,
      "logits/chosen": -0.4327819049358368,
      "logits/rejected": -1.101218342781067,
      "logps/chosen": -1.5026156902313232,
      "logps/rejected": -2.6103224754333496,
      "loss": 1.531,
      "nll_loss": 1.5058268308639526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15026158094406128,
      "rewards/margins": 0.11077067255973816,
      "rewards/rejected": -0.26103225350379944,
      "step": 821
    },
    {
      "epoch": 1.2947202521670607,
      "grad_norm": 0.22206051647663116,
      "learning_rate": 1.3392565713774575e-06,
      "log_odds_chosen": 1.382164478302002,
      "log_odds_ratio": -0.23260526359081268,
      "logits/chosen": -0.468174546957016,
      "logits/rejected": -1.1772918701171875,
      "logps/chosen": -1.509872555732727,
      "logps/rejected": -2.712132453918457,
      "loss": 1.5512,
      "nll_loss": 1.5279221534729004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15098726749420166,
      "rewards/margins": 0.12022601068019867,
      "rewards/rejected": -0.27121326327323914,
      "step": 822
    },
    {
      "epoch": 1.2962962962962963,
      "grad_norm": 0.22672586143016815,
      "learning_rate": 1.3340706308962763e-06,
      "log_odds_chosen": 1.1706663370132446,
      "log_odds_ratio": -0.28422343730926514,
      "logits/chosen": -0.41489338874816895,
      "logits/rejected": -0.9049216508865356,
      "logps/chosen": -1.4770910739898682,
      "logps/rejected": -2.475165843963623,
      "loss": 1.5165,
      "nll_loss": 1.4881079196929932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14770913124084473,
      "rewards/margins": 0.09980747103691101,
      "rewards/rejected": -0.24751658737659454,
      "step": 823
    },
    {
      "epoch": 1.297872340425532,
      "grad_norm": 0.2270732969045639,
      "learning_rate": 1.3288897211934066e-06,
      "log_odds_chosen": 1.1724331378936768,
      "log_odds_ratio": -0.2876794934272766,
      "logits/chosen": -0.41414085030555725,
      "logits/rejected": -0.8459904789924622,
      "logps/chosen": -1.4750962257385254,
      "logps/rejected": -2.4780097007751465,
      "loss": 1.5077,
      "nll_loss": 1.478973627090454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1475096344947815,
      "rewards/margins": 0.10029137134552002,
      "rewards/rejected": -0.24780097603797913,
      "step": 824
    },
    {
      "epoch": 1.2994483845547675,
      "grad_norm": 0.21012206375598907,
      "learning_rate": 1.323713881408147e-06,
      "log_odds_chosen": 1.0094988346099854,
      "log_odds_ratio": -0.3195574879646301,
      "logits/chosen": -0.4181651175022125,
      "logits/rejected": -1.1058521270751953,
      "logps/chosen": -1.5204685926437378,
      "logps/rejected": -2.3831543922424316,
      "loss": 1.552,
      "nll_loss": 1.5200499296188354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15204684436321259,
      "rewards/margins": 0.0862685814499855,
      "rewards/rejected": -0.2383154332637787,
      "step": 825
    },
    {
      "epoch": 1.3010244286840031,
      "grad_norm": 0.2056722640991211,
      "learning_rate": 1.3185431506414943e-06,
      "log_odds_chosen": 1.1373863220214844,
      "log_odds_ratio": -0.2863887548446655,
      "logits/chosen": -0.4336894452571869,
      "logits/rejected": -1.1000032424926758,
      "logps/chosen": -1.4164533615112305,
      "logps/rejected": -2.3781182765960693,
      "loss": 1.4541,
      "nll_loss": 1.4254745244979858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14164532721042633,
      "rewards/margins": 0.09616649150848389,
      "rewards/rejected": -0.23781181871891022,
      "step": 826
    },
    {
      "epoch": 1.3026004728132388,
      "grad_norm": 0.22690770030021667,
      "learning_rate": 1.313377567955851e-06,
      "log_odds_chosen": 0.9601044654846191,
      "log_odds_ratio": -0.33979982137680054,
      "logits/chosen": -0.3924036920070648,
      "logits/rejected": -0.9113993048667908,
      "logps/chosen": -1.5542742013931274,
      "logps/rejected": -2.3793892860412598,
      "loss": 1.5823,
      "nll_loss": 1.5483341217041016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15542742609977722,
      "rewards/margins": 0.08251151442527771,
      "rewards/rejected": -0.23793894052505493,
      "step": 827
    },
    {
      "epoch": 1.3041765169424744,
      "grad_norm": 0.22955073416233063,
      "learning_rate": 1.3082171723747257e-06,
      "log_odds_chosen": 1.0225118398666382,
      "log_odds_ratio": -0.31783902645111084,
      "logits/chosen": -0.5294691920280457,
      "logits/rejected": -1.0333658456802368,
      "logps/chosen": -1.5037142038345337,
      "logps/rejected": -2.3728489875793457,
      "loss": 1.539,
      "nll_loss": 1.5072402954101562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15037143230438232,
      "rewards/margins": 0.08691349625587463,
      "rewards/rejected": -0.23728492856025696,
      "step": 828
    },
    {
      "epoch": 1.30575256107171,
      "grad_norm": 0.2512214183807373,
      "learning_rate": 1.3030620028824424e-06,
      "log_odds_chosen": 0.9782799482345581,
      "log_odds_ratio": -0.3257397711277008,
      "logits/chosen": -0.4220568835735321,
      "logits/rejected": -0.9340043663978577,
      "logps/chosen": -1.5149894952774048,
      "logps/rejected": -2.3435518741607666,
      "loss": 1.5582,
      "nll_loss": 1.5256484746932983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1514989584684372,
      "rewards/margins": 0.08285626024007797,
      "rewards/rejected": -0.23435522615909576,
      "step": 829
    },
    {
      "epoch": 1.3073286052009456,
      "grad_norm": 0.22766518592834473,
      "learning_rate": 1.2979120984238449e-06,
      "log_odds_chosen": 1.0736923217773438,
      "log_odds_ratio": -0.3030780851840973,
      "logits/chosen": -0.43962764739990234,
      "logits/rejected": -0.9667648673057556,
      "logps/chosen": -1.5611544847488403,
      "logps/rejected": -2.4860024452209473,
      "loss": 1.593,
      "nll_loss": 1.5626908540725708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15611545741558075,
      "rewards/margins": 0.09248476475477219,
      "rewards/rejected": -0.24860021471977234,
      "step": 830
    },
    {
      "epoch": 1.3089046493301812,
      "grad_norm": 0.23188571631908417,
      "learning_rate": 1.2927674979040009e-06,
      "log_odds_chosen": 1.0847249031066895,
      "log_odds_ratio": -0.2950751483440399,
      "logits/chosen": -0.327899307012558,
      "logits/rejected": -0.9707051515579224,
      "logps/chosen": -1.4424419403076172,
      "logps/rejected": -2.3552637100219727,
      "loss": 1.4992,
      "nll_loss": 1.4697351455688477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14424419403076172,
      "rewards/margins": 0.0912821888923645,
      "rewards/rejected": -0.23552636802196503,
      "step": 831
    },
    {
      "epoch": 1.3104806934594169,
      "grad_norm": 0.2029716968536377,
      "learning_rate": 1.2876282401879106e-06,
      "log_odds_chosen": 1.2439287900924683,
      "log_odds_ratio": -0.25725361704826355,
      "logits/chosen": -0.36858928203582764,
      "logits/rejected": -1.029232144355774,
      "logps/chosen": -1.4818058013916016,
      "logps/rejected": -2.550248622894287,
      "loss": 1.5219,
      "nll_loss": 1.4962177276611328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14818057417869568,
      "rewards/margins": 0.10684427618980408,
      "rewards/rejected": -0.25502485036849976,
      "step": 832
    },
    {
      "epoch": 1.3120567375886525,
      "grad_norm": 0.22811907529830933,
      "learning_rate": 1.2824943641002115e-06,
      "log_odds_chosen": 1.3333407640457153,
      "log_odds_ratio": -0.2438172996044159,
      "logits/chosen": -0.31036245822906494,
      "logits/rejected": -1.1004436016082764,
      "logps/chosen": -1.475118637084961,
      "logps/rejected": -2.6273040771484375,
      "loss": 1.5133,
      "nll_loss": 1.4889659881591797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14751188457012177,
      "rewards/margins": 0.11521852761507034,
      "rewards/rejected": -0.2627303898334503,
      "step": 833
    },
    {
      "epoch": 1.313632781717888,
      "grad_norm": 0.26685869693756104,
      "learning_rate": 1.2773659084248845e-06,
      "log_odds_chosen": 1.4163463115692139,
      "log_odds_ratio": -0.24490469694137573,
      "logits/chosen": -0.5068438649177551,
      "logits/rejected": -1.210258960723877,
      "logps/chosen": -1.4376381635665894,
      "logps/rejected": -2.665928602218628,
      "loss": 1.4701,
      "nll_loss": 1.4456465244293213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14376381039619446,
      "rewards/margins": 0.12282902747392654,
      "rewards/rejected": -0.2665928602218628,
      "step": 834
    },
    {
      "epoch": 1.3152088258471237,
      "grad_norm": 0.22141002118587494,
      "learning_rate": 1.2722429119049632e-06,
      "log_odds_chosen": 0.9516381025314331,
      "log_odds_ratio": -0.34376731514930725,
      "logits/chosen": -0.33848103880882263,
      "logits/rejected": -0.936203122138977,
      "logps/chosen": -1.5605013370513916,
      "logps/rejected": -2.378781318664551,
      "loss": 1.6029,
      "nll_loss": 1.568530797958374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15605014562606812,
      "rewards/margins": 0.08182797580957413,
      "rewards/rejected": -0.23787811398506165,
      "step": 835
    },
    {
      "epoch": 1.3167848699763594,
      "grad_norm": 0.22151143848896027,
      "learning_rate": 1.267125413242239e-06,
      "log_odds_chosen": 1.201178789138794,
      "log_odds_ratio": -0.27666616439819336,
      "logits/chosen": -0.44035422801971436,
      "logits/rejected": -0.9765951633453369,
      "logps/chosen": -1.4348206520080566,
      "logps/rejected": -2.4565224647521973,
      "loss": 1.4819,
      "nll_loss": 1.4542245864868164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14348207414150238,
      "rewards/margins": 0.10217016935348511,
      "rewards/rejected": -0.2456522434949875,
      "step": 836
    },
    {
      "epoch": 1.318360914105595,
      "grad_norm": 0.23120108246803284,
      "learning_rate": 1.2620134510969719e-06,
      "log_odds_chosen": 1.0868710279464722,
      "log_odds_ratio": -0.30326172709465027,
      "logits/chosen": -0.47523611783981323,
      "logits/rejected": -0.8828234076499939,
      "logps/chosen": -1.4276272058486938,
      "logps/rejected": -2.3450591564178467,
      "loss": 1.4767,
      "nll_loss": 1.4463266134262085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14276273548603058,
      "rewards/margins": 0.09174320846796036,
      "rewards/rejected": -0.23450592160224915,
      "step": 837
    },
    {
      "epoch": 1.3199369582348306,
      "grad_norm": 0.2290244996547699,
      "learning_rate": 1.2569070640875912e-06,
      "log_odds_chosen": 1.353071928024292,
      "log_odds_ratio": -0.2724730670452118,
      "logits/chosen": -0.33452534675598145,
      "logits/rejected": -1.0474516153335571,
      "logps/chosen": -1.4005942344665527,
      "logps/rejected": -2.528109550476074,
      "loss": 1.4535,
      "nll_loss": 1.4262772798538208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1400594264268875,
      "rewards/margins": 0.11275152117013931,
      "rewards/rejected": -0.2528109550476074,
      "step": 838
    },
    {
      "epoch": 1.3215130023640662,
      "grad_norm": 0.25099191069602966,
      "learning_rate": 1.2518062907904136e-06,
      "log_odds_chosen": 1.3832988739013672,
      "log_odds_ratio": -0.24166999757289886,
      "logits/chosen": -0.47754788398742676,
      "logits/rejected": -1.1093342304229736,
      "logps/chosen": -1.4722689390182495,
      "logps/rejected": -2.671079397201538,
      "loss": 1.5105,
      "nll_loss": 1.4863805770874023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14722689986228943,
      "rewards/margins": 0.1198810264468193,
      "rewards/rejected": -0.26710793375968933,
      "step": 839
    },
    {
      "epoch": 1.3230890464933018,
      "grad_norm": 0.2254980355501175,
      "learning_rate": 1.2467111697393446e-06,
      "log_odds_chosen": 1.1377276182174683,
      "log_odds_ratio": -0.28483107686042786,
      "logits/chosen": -0.4024809002876282,
      "logits/rejected": -1.1928318738937378,
      "logps/chosen": -1.578923225402832,
      "logps/rejected": -2.565640926361084,
      "loss": 1.6091,
      "nll_loss": 1.580634593963623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1578923463821411,
      "rewards/margins": 0.09867175668478012,
      "rewards/rejected": -0.25656411051750183,
      "step": 840
    },
    {
      "epoch": 1.3246650906225375,
      "grad_norm": 0.22856196761131287,
      "learning_rate": 1.2416217394255905e-06,
      "log_odds_chosen": 0.9944614171981812,
      "log_odds_ratio": -0.319040447473526,
      "logits/chosen": -0.3357866406440735,
      "logits/rejected": -0.8649716377258301,
      "logps/chosen": -1.4208266735076904,
      "logps/rejected": -2.2502598762512207,
      "loss": 1.4912,
      "nll_loss": 1.459309697151184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14208266139030457,
      "rewards/margins": 0.08294330537319183,
      "rewards/rejected": -0.2250259667634964,
      "step": 841
    },
    {
      "epoch": 1.326241134751773,
      "grad_norm": 0.24705147743225098,
      "learning_rate": 1.2365380382973669e-06,
      "log_odds_chosen": 0.9611343741416931,
      "log_odds_ratio": -0.33399468660354614,
      "logits/chosen": -0.41096287965774536,
      "logits/rejected": -0.8459650278091431,
      "logps/chosen": -1.582237720489502,
      "logps/rejected": -2.4111733436584473,
      "loss": 1.6043,
      "nll_loss": 1.5708799362182617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15822376310825348,
      "rewards/margins": 0.08289356529712677,
      "rewards/rejected": -0.24111732840538025,
      "step": 842
    },
    {
      "epoch": 1.3278171788810087,
      "grad_norm": 0.22322946786880493,
      "learning_rate": 1.2314601047596061e-06,
      "log_odds_chosen": 1.3764740228652954,
      "log_odds_ratio": -0.23093223571777344,
      "logits/chosen": -0.3635350465774536,
      "logits/rejected": -1.0827128887176514,
      "logps/chosen": -1.4961938858032227,
      "logps/rejected": -2.6908254623413086,
      "loss": 1.5198,
      "nll_loss": 1.496692180633545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14961938560009003,
      "rewards/margins": 0.11946314573287964,
      "rewards/rejected": -0.26908254623413086,
      "step": 843
    },
    {
      "epoch": 1.3293932230102443,
      "grad_norm": 0.2095765769481659,
      "learning_rate": 1.2263879771736713e-06,
      "log_odds_chosen": 1.032625436782837,
      "log_odds_ratio": -0.3143763542175293,
      "logits/chosen": -0.4602211117744446,
      "logits/rejected": -1.1181039810180664,
      "logps/chosen": -1.5142139196395874,
      "logps/rejected": -2.393423557281494,
      "loss": 1.5523,
      "nll_loss": 1.5208513736724854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1514214128255844,
      "rewards/margins": 0.08792095631361008,
      "rewards/rejected": -0.23934237658977509,
      "step": 844
    },
    {
      "epoch": 1.33096926713948,
      "grad_norm": 0.2087593972682953,
      "learning_rate": 1.2213216938570642e-06,
      "log_odds_chosen": 1.1832407712936401,
      "log_odds_ratio": -0.2945471704006195,
      "logits/chosen": -0.42383629083633423,
      "logits/rejected": -1.0590530633926392,
      "logps/chosen": -1.4388800859451294,
      "logps/rejected": -2.439924716949463,
      "loss": 1.4916,
      "nll_loss": 1.462174654006958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14388801157474518,
      "rewards/margins": 0.10010445863008499,
      "rewards/rejected": -0.24399246275424957,
      "step": 845
    },
    {
      "epoch": 1.3325453112687156,
      "grad_norm": 0.2337147295475006,
      "learning_rate": 1.2162612930831354e-06,
      "log_odds_chosen": 1.4092597961425781,
      "log_odds_ratio": -0.23199397325515747,
      "logits/chosen": -0.468430757522583,
      "logits/rejected": -1.1354173421859741,
      "logps/chosen": -1.3423163890838623,
      "logps/rejected": -2.5311381816864014,
      "loss": 1.3907,
      "nll_loss": 1.3674795627593994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13423164188861847,
      "rewards/margins": 0.11888215690851212,
      "rewards/rejected": -0.2531138062477112,
      "step": 846
    },
    {
      "epoch": 1.3341213553979512,
      "grad_norm": 0.221159890294075,
      "learning_rate": 1.2112068130807949e-06,
      "log_odds_chosen": 1.5634915828704834,
      "log_odds_ratio": -0.20362898707389832,
      "logits/chosen": -0.43519529700279236,
      "logits/rejected": -1.1275871992111206,
      "logps/chosen": -1.4879201650619507,
      "logps/rejected": -2.8606157302856445,
      "loss": 1.5047,
      "nll_loss": 1.4843816757202148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14879199862480164,
      "rewards/margins": 0.13726957142353058,
      "rewards/rejected": -0.2860615849494934,
      "step": 847
    },
    {
      "epoch": 1.3356973995271868,
      "grad_norm": 0.2349996566772461,
      "learning_rate": 1.206158292034226e-06,
      "log_odds_chosen": 1.2384583950042725,
      "log_odds_ratio": -0.2609674334526062,
      "logits/chosen": -0.43850237131118774,
      "logits/rejected": -1.076735019683838,
      "logps/chosen": -1.5157259702682495,
      "logps/rejected": -2.5852601528167725,
      "loss": 1.5477,
      "nll_loss": 1.5216010808944702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1515726000070572,
      "rewards/margins": 0.10695342719554901,
      "rewards/rejected": -0.2585260272026062,
      "step": 848
    },
    {
      "epoch": 1.3372734436564224,
      "grad_norm": 0.21206647157669067,
      "learning_rate": 1.2011157680825928e-06,
      "log_odds_chosen": 1.3839961290359497,
      "log_odds_ratio": -0.23855844140052795,
      "logits/chosen": -0.36000242829322815,
      "logits/rejected": -1.1573578119277954,
      "logps/chosen": -1.429657220840454,
      "logps/rejected": -2.6215391159057617,
      "loss": 1.4702,
      "nll_loss": 1.4463777542114258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14296571910381317,
      "rewards/margins": 0.11918817460536957,
      "rewards/rejected": -0.2621539235115051,
      "step": 849
    },
    {
      "epoch": 1.338849487785658,
      "grad_norm": 0.24714502692222595,
      "learning_rate": 1.1960792793197553e-06,
      "log_odds_chosen": 1.248303771018982,
      "log_odds_ratio": -0.2710753381252289,
      "logits/chosen": -0.5240070819854736,
      "logits/rejected": -1.1066398620605469,
      "logps/chosen": -1.4856799840927124,
      "logps/rejected": -2.5599308013916016,
      "loss": 1.5207,
      "nll_loss": 1.4935446977615356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14856800436973572,
      "rewards/margins": 0.10742507129907608,
      "rewards/rejected": -0.2559930682182312,
      "step": 850
    },
    {
      "epoch": 1.3404255319148937,
      "grad_norm": 0.22980041801929474,
      "learning_rate": 1.1910488637939824e-06,
      "log_odds_chosen": 1.4624735116958618,
      "log_odds_ratio": -0.21082666516304016,
      "logits/chosen": -0.5417642593383789,
      "logits/rejected": -1.1412882804870605,
      "logps/chosen": -1.5338287353515625,
      "logps/rejected": -2.814779281616211,
      "loss": 1.5423,
      "nll_loss": 1.5212583541870117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15338288247585297,
      "rewards/margins": 0.12809503078460693,
      "rewards/rejected": -0.2814779281616211,
      "step": 851
    },
    {
      "epoch": 1.3420015760441293,
      "grad_norm": 0.22727517783641815,
      "learning_rate": 1.1860245595076582e-06,
      "log_odds_chosen": 0.9794008731842041,
      "log_odds_ratio": -0.3246573507785797,
      "logits/chosen": -0.33602237701416016,
      "logits/rejected": -1.0489847660064697,
      "logps/chosen": -1.568975567817688,
      "logps/rejected": -2.4110031127929688,
      "loss": 1.5821,
      "nll_loss": 1.5496397018432617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15689755976200104,
      "rewards/margins": 0.08420275151729584,
      "rewards/rejected": -0.24110031127929688,
      "step": 852
    },
    {
      "epoch": 1.343577620173365,
      "grad_norm": 0.22607745230197906,
      "learning_rate": 1.1810064044170027e-06,
      "log_odds_chosen": 1.1746796369552612,
      "log_odds_ratio": -0.28140440583229065,
      "logits/chosen": -0.49444812536239624,
      "logits/rejected": -1.139293909072876,
      "logps/chosen": -1.4488189220428467,
      "logps/rejected": -2.448302984237671,
      "loss": 1.4901,
      "nll_loss": 1.4619985818862915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14488190412521362,
      "rewards/margins": 0.09994840621948242,
      "rewards/rejected": -0.24483029544353485,
      "step": 853
    },
    {
      "epoch": 1.3451536643026005,
      "grad_norm": 0.22622895240783691,
      "learning_rate": 1.1759944364317812e-06,
      "log_odds_chosen": 1.234906792640686,
      "log_odds_ratio": -0.26593419909477234,
      "logits/chosen": -0.5045130848884583,
      "logits/rejected": -1.0543159246444702,
      "logps/chosen": -1.50751531124115,
      "logps/rejected": -2.5730347633361816,
      "loss": 1.5336,
      "nll_loss": 1.507010579109192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.150751531124115,
      "rewards/margins": 0.10655193775892258,
      "rewards/rejected": -0.25730347633361816,
      "step": 854
    },
    {
      "epoch": 1.3467297084318361,
      "grad_norm": 0.23035307228565216,
      "learning_rate": 1.1709886934150172e-06,
      "log_odds_chosen": 1.0615102052688599,
      "log_odds_ratio": -0.3074616491794586,
      "logits/chosen": -0.3765428960323334,
      "logits/rejected": -1.0091395378112793,
      "logps/chosen": -1.5286229848861694,
      "logps/rejected": -2.4355273246765137,
      "loss": 1.5729,
      "nll_loss": 1.5421828031539917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1528623104095459,
      "rewards/margins": 0.09069044142961502,
      "rewards/rejected": -0.24355272948741913,
      "step": 855
    },
    {
      "epoch": 1.3483057525610718,
      "grad_norm": 0.25148847699165344,
      "learning_rate": 1.1659892131827097e-06,
      "log_odds_chosen": 1.3056787252426147,
      "log_odds_ratio": -0.26284077763557434,
      "logits/chosen": -0.4943777322769165,
      "logits/rejected": -1.2130732536315918,
      "logps/chosen": -1.5549124479293823,
      "logps/rejected": -2.6997969150543213,
      "loss": 1.5794,
      "nll_loss": 1.5531346797943115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15549126267433167,
      "rewards/margins": 0.11448842287063599,
      "rewards/rejected": -0.26997965574264526,
      "step": 856
    },
    {
      "epoch": 1.3498817966903074,
      "grad_norm": 0.21482735872268677,
      "learning_rate": 1.1609960335035423e-06,
      "log_odds_chosen": 1.7083791494369507,
      "log_odds_ratio": -0.17967192828655243,
      "logits/chosen": -0.4335390627384186,
      "logits/rejected": -1.1564877033233643,
      "logps/chosen": -1.5436441898345947,
      "logps/rejected": -3.056908369064331,
      "loss": 1.5673,
      "nll_loss": 1.5493606328964233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543644219636917,
      "rewards/margins": 0.15132641792297363,
      "rewards/rejected": -0.30569085478782654,
      "step": 857
    },
    {
      "epoch": 1.351457840819543,
      "grad_norm": 0.23238582909107208,
      "learning_rate": 1.1560091920986028e-06,
      "log_odds_chosen": 1.353506088256836,
      "log_odds_ratio": -0.2372596710920334,
      "logits/chosen": -0.448645681142807,
      "logits/rejected": -1.1417397260665894,
      "logps/chosen": -1.4620473384857178,
      "logps/rejected": -2.62900447845459,
      "loss": 1.5016,
      "nll_loss": 1.4779021739959717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14620472490787506,
      "rewards/margins": 0.11669572442770004,
      "rewards/rejected": -0.2629004418849945,
      "step": 858
    },
    {
      "epoch": 1.3530338849487786,
      "grad_norm": 0.23076988756656647,
      "learning_rate": 1.1510287266410967e-06,
      "log_odds_chosen": 1.0504767894744873,
      "log_odds_ratio": -0.3067944645881653,
      "logits/chosen": -0.4650724530220032,
      "logits/rejected": -1.0073715448379517,
      "logps/chosen": -1.571841835975647,
      "logps/rejected": -2.479578971862793,
      "loss": 1.5995,
      "nll_loss": 1.5688267946243286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1571841835975647,
      "rewards/margins": 0.09077370166778564,
      "rewards/rejected": -0.24795788526535034,
      "step": 859
    },
    {
      "epoch": 1.3546099290780143,
      "grad_norm": 0.22231945395469666,
      "learning_rate": 1.1460546747560616e-06,
      "log_odds_chosen": 1.449920892715454,
      "log_odds_ratio": -0.22251494228839874,
      "logits/chosen": -0.472192645072937,
      "logits/rejected": -1.1997166872024536,
      "logps/chosen": -1.5049824714660645,
      "logps/rejected": -2.7728986740112305,
      "loss": 1.5228,
      "nll_loss": 1.5005621910095215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15049825608730316,
      "rewards/margins": 0.12679165601730347,
      "rewards/rejected": -0.27728989720344543,
      "step": 860
    },
    {
      "epoch": 1.3561859732072499,
      "grad_norm": 0.22649304568767548,
      "learning_rate": 1.1410870740200839e-06,
      "log_odds_chosen": 1.2354484796524048,
      "log_odds_ratio": -0.27476581931114197,
      "logits/chosen": -0.4375569522380829,
      "logits/rejected": -1.0863518714904785,
      "logps/chosen": -1.5507245063781738,
      "logps/rejected": -2.6268959045410156,
      "loss": 1.5759,
      "nll_loss": 1.5483767986297607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15507246553897858,
      "rewards/margins": 0.10761713236570358,
      "rewards/rejected": -0.2626895606517792,
      "step": 861
    },
    {
      "epoch": 1.3577620173364855,
      "grad_norm": 0.22282390296459198,
      "learning_rate": 1.1361259619610138e-06,
      "log_odds_chosen": 1.368697166442871,
      "log_odds_ratio": -0.24308288097381592,
      "logits/chosen": -0.4598146080970764,
      "logits/rejected": -1.1848602294921875,
      "logps/chosen": -1.4812456369400024,
      "logps/rejected": -2.6622142791748047,
      "loss": 1.5198,
      "nll_loss": 1.4955295324325562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1481245756149292,
      "rewards/margins": 0.11809685826301575,
      "rewards/rejected": -0.26622143387794495,
      "step": 862
    },
    {
      "epoch": 1.3593380614657211,
      "grad_norm": 0.24183566868305206,
      "learning_rate": 1.1311713760576834e-06,
      "log_odds_chosen": 1.305025339126587,
      "log_odds_ratio": -0.2511361241340637,
      "logits/chosen": -0.5733106732368469,
      "logits/rejected": -0.9739482402801514,
      "logps/chosen": -1.4690845012664795,
      "logps/rejected": -2.5891542434692383,
      "loss": 1.4891,
      "nll_loss": 1.4639630317687988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1469084471464157,
      "rewards/margins": 0.11200698465108871,
      "rewards/rejected": -0.25891542434692383,
      "step": 863
    },
    {
      "epoch": 1.3609141055949567,
      "grad_norm": 0.22981640696525574,
      "learning_rate": 1.1262233537396228e-06,
      "log_odds_chosen": 1.5467019081115723,
      "log_odds_ratio": -0.22023329138755798,
      "logits/chosen": -0.4064531922340393,
      "logits/rejected": -1.054137110710144,
      "logps/chosen": -1.462540864944458,
      "logps/rejected": -2.809943675994873,
      "loss": 1.4786,
      "nll_loss": 1.4565598964691162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14625407755374908,
      "rewards/margins": 0.13474030792713165,
      "rewards/rejected": -0.28099438548088074,
      "step": 864
    },
    {
      "epoch": 1.3624901497241924,
      "grad_norm": 0.22347261011600494,
      "learning_rate": 1.1212819323867778e-06,
      "log_odds_chosen": 1.2683062553405762,
      "log_odds_ratio": -0.2686724066734314,
      "logits/chosen": -0.4792177379131317,
      "logits/rejected": -1.075474500656128,
      "logps/chosen": -1.4263142347335815,
      "logps/rejected": -2.515336513519287,
      "loss": 1.477,
      "nll_loss": 1.4501179456710815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1426314413547516,
      "rewards/margins": 0.10890225321054459,
      "rewards/rejected": -0.2515336871147156,
      "step": 865
    },
    {
      "epoch": 1.364066193853428,
      "grad_norm": 0.2258315533399582,
      "learning_rate": 1.1163471493292267e-06,
      "log_odds_chosen": 1.2943799495697021,
      "log_odds_ratio": -0.25588259100914,
      "logits/chosen": -0.46966806054115295,
      "logits/rejected": -1.070439100265503,
      "logps/chosen": -1.507627248764038,
      "logps/rejected": -2.62831449508667,
      "loss": 1.5326,
      "nll_loss": 1.5069795846939087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15076273679733276,
      "rewards/margins": 0.11206871271133423,
      "rewards/rejected": -0.262831449508667,
      "step": 866
    },
    {
      "epoch": 1.3656422379826636,
      "grad_norm": 0.21010157465934753,
      "learning_rate": 1.1114190418468972e-06,
      "log_odds_chosen": 1.1957231760025024,
      "log_odds_ratio": -0.2829042673110962,
      "logits/chosen": -0.3893083333969116,
      "logits/rejected": -1.0369346141815186,
      "logps/chosen": -1.5242562294006348,
      "logps/rejected": -2.563066244125366,
      "loss": 1.5699,
      "nll_loss": 1.5416488647460938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1524256318807602,
      "rewards/margins": 0.10388099402189255,
      "rewards/rejected": -0.25630664825439453,
      "step": 867
    },
    {
      "epoch": 1.367218282111899,
      "grad_norm": 0.22691573202610016,
      "learning_rate": 1.106497647169288e-06,
      "log_odds_chosen": 1.3435533046722412,
      "log_odds_ratio": -0.2359973043203354,
      "logits/chosen": -0.3924179673194885,
      "logits/rejected": -1.1431825160980225,
      "logps/chosen": -1.4559916257858276,
      "logps/rejected": -2.6107699871063232,
      "loss": 1.4802,
      "nll_loss": 1.4565989971160889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1455991566181183,
      "rewards/margins": 0.11547785252332687,
      "rewards/rejected": -0.26107701659202576,
      "step": 868
    },
    {
      "epoch": 1.3687943262411348,
      "grad_norm": 0.24031208455562592,
      "learning_rate": 1.1015830024751854e-06,
      "log_odds_chosen": 1.257957935333252,
      "log_odds_ratio": -0.26588165760040283,
      "logits/chosen": -0.5487285852432251,
      "logits/rejected": -0.8780529499053955,
      "logps/chosen": -1.4152727127075195,
      "logps/rejected": -2.4892725944519043,
      "loss": 1.4558,
      "nll_loss": 1.4291722774505615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14152728021144867,
      "rewards/margins": 0.10739997774362564,
      "rewards/rejected": -0.2489272654056549,
      "step": 869
    },
    {
      "epoch": 1.3703703703703702,
      "grad_norm": 0.2295382022857666,
      "learning_rate": 1.0966751448923834e-06,
      "log_odds_chosen": 1.5923051834106445,
      "log_odds_ratio": -0.19828805327415466,
      "logits/chosen": -0.39850732684135437,
      "logits/rejected": -0.9948802590370178,
      "logps/chosen": -1.4130712747573853,
      "logps/rejected": -2.794156312942505,
      "loss": 1.4485,
      "nll_loss": 1.428704023361206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14130714535713196,
      "rewards/margins": 0.138108491897583,
      "rewards/rejected": -0.27941563725471497,
      "step": 870
    },
    {
      "epoch": 1.371946414499606,
      "grad_norm": 0.23377178609371185,
      "learning_rate": 1.0917741114974007e-06,
      "log_odds_chosen": 1.1595382690429688,
      "log_odds_ratio": -0.27767980098724365,
      "logits/chosen": -0.37238627672195435,
      "logits/rejected": -1.121927261352539,
      "logps/chosen": -1.5514824390411377,
      "logps/rejected": -2.5510687828063965,
      "loss": 1.5823,
      "nll_loss": 1.5545299053192139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15514825284481049,
      "rewards/margins": 0.09995865821838379,
      "rewards/rejected": -0.2551068961620331,
      "step": 871
    },
    {
      "epoch": 1.3735224586288415,
      "grad_norm": 0.22718104720115662,
      "learning_rate": 1.0868799393152035e-06,
      "log_odds_chosen": 1.486511468887329,
      "log_odds_ratio": -0.22496815025806427,
      "logits/chosen": -0.5031551122665405,
      "logits/rejected": -1.0961716175079346,
      "logps/chosen": -1.4375802278518677,
      "logps/rejected": -2.7251243591308594,
      "loss": 1.4721,
      "nll_loss": 1.4496312141418457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14375804364681244,
      "rewards/margins": 0.1287544071674347,
      "rewards/rejected": -0.27251243591308594,
      "step": 872
    },
    {
      "epoch": 1.3750985027580773,
      "grad_norm": 0.24282120168209076,
      "learning_rate": 1.0819926653189271e-06,
      "log_odds_chosen": 1.2221357822418213,
      "log_odds_ratio": -0.27214235067367554,
      "logits/chosen": -0.4541354775428772,
      "logits/rejected": -0.9479659199714661,
      "logps/chosen": -1.524904489517212,
      "logps/rejected": -2.5836267471313477,
      "loss": 1.559,
      "nll_loss": 1.5317652225494385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15249046683311462,
      "rewards/margins": 0.10587219893932343,
      "rewards/rejected": -0.25836265087127686,
      "step": 873
    },
    {
      "epoch": 1.3766745468873127,
      "grad_norm": 0.2146005630493164,
      "learning_rate": 1.0771123264295895e-06,
      "log_odds_chosen": 1.261728048324585,
      "log_odds_ratio": -0.25936099886894226,
      "logits/chosen": -0.48896950483322144,
      "logits/rejected": -1.197722315788269,
      "logps/chosen": -1.4136037826538086,
      "logps/rejected": -2.483643054962158,
      "loss": 1.4511,
      "nll_loss": 1.4251786470413208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14136040210723877,
      "rewards/margins": 0.10700392723083496,
      "rewards/rejected": -0.24836431443691254,
      "step": 874
    },
    {
      "epoch": 1.3782505910165486,
      "grad_norm": 0.20781339704990387,
      "learning_rate": 1.0722389595158215e-06,
      "log_odds_chosen": 1.42768132686615,
      "log_odds_ratio": -0.23379269242286682,
      "logits/chosen": -0.37707096338272095,
      "logits/rejected": -1.1037777662277222,
      "logps/chosen": -1.3957927227020264,
      "logps/rejected": -2.622485637664795,
      "loss": 1.44,
      "nll_loss": 1.4165735244750977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13957926630973816,
      "rewards/margins": 0.1226692795753479,
      "rewards/rejected": -0.26224857568740845,
      "step": 875
    },
    {
      "epoch": 1.379826635145784,
      "grad_norm": 0.22928835451602936,
      "learning_rate": 1.0673726013935827e-06,
      "log_odds_chosen": 1.3522485494613647,
      "log_odds_ratio": -0.26458612084388733,
      "logits/chosen": -0.4016760289669037,
      "logits/rejected": -1.0482399463653564,
      "logps/chosen": -1.4148075580596924,
      "logps/rejected": -2.5738525390625,
      "loss": 1.459,
      "nll_loss": 1.4325066804885864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14148077368736267,
      "rewards/margins": 0.11590449512004852,
      "rewards/rejected": -0.25738525390625,
      "step": 876
    },
    {
      "epoch": 1.3814026792750198,
      "grad_norm": 0.2242683470249176,
      "learning_rate": 1.0625132888258833e-06,
      "log_odds_chosen": 1.1951643228530884,
      "log_odds_ratio": -0.2789776623249054,
      "logits/chosen": -0.5197619199752808,
      "logits/rejected": -1.0357139110565186,
      "logps/chosen": -1.4870091676712036,
      "logps/rejected": -2.517120361328125,
      "loss": 1.52,
      "nll_loss": 1.4920597076416016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14870092272758484,
      "rewards/margins": 0.1030111163854599,
      "rewards/rejected": -0.25171202421188354,
      "step": 877
    },
    {
      "epoch": 1.3829787234042552,
      "grad_norm": 0.23171353340148926,
      "learning_rate": 1.057661058522509e-06,
      "log_odds_chosen": 1.3456848859786987,
      "log_odds_ratio": -0.24196185171604156,
      "logits/chosen": -0.48408859968185425,
      "logits/rejected": -1.20857572555542,
      "logps/chosen": -1.5075433254241943,
      "logps/rejected": -2.6741647720336914,
      "loss": 1.5439,
      "nll_loss": 1.519722580909729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15075434744358063,
      "rewards/margins": 0.11666211485862732,
      "rewards/rejected": -0.26741647720336914,
      "step": 878
    },
    {
      "epoch": 1.384554767533491,
      "grad_norm": 0.2430916130542755,
      "learning_rate": 1.0528159471397425e-06,
      "log_odds_chosen": 1.2328135967254639,
      "log_odds_ratio": -0.2663717269897461,
      "logits/chosen": -0.4501136243343353,
      "logits/rejected": -1.1262433528900146,
      "logps/chosen": -1.4531660079956055,
      "logps/rejected": -2.507840633392334,
      "loss": 1.4966,
      "nll_loss": 1.4699150323867798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14531661570072174,
      "rewards/margins": 0.10546744614839554,
      "rewards/rejected": -0.2507840692996979,
      "step": 879
    },
    {
      "epoch": 1.3861308116627264,
      "grad_norm": 0.22656792402267456,
      "learning_rate": 1.0479779912800868e-06,
      "log_odds_chosen": 1.1305993795394897,
      "log_odds_ratio": -0.28715604543685913,
      "logits/chosen": -0.5629742741584778,
      "logits/rejected": -1.1117656230926514,
      "logps/chosen": -1.4217956066131592,
      "logps/rejected": -2.373842239379883,
      "loss": 1.4677,
      "nll_loss": 1.438956379890442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14217956364154816,
      "rewards/margins": 0.0952046811580658,
      "rewards/rejected": -0.23738422989845276,
      "step": 880
    },
    {
      "epoch": 1.3877068557919623,
      "grad_norm": 0.22105728089809418,
      "learning_rate": 1.0431472274919863e-06,
      "log_odds_chosen": 1.4608548879623413,
      "log_odds_ratio": -0.22333656251430511,
      "logits/chosen": -0.3856619596481323,
      "logits/rejected": -1.1437593698501587,
      "logps/chosen": -1.4390586614608765,
      "logps/rejected": -2.7048139572143555,
      "loss": 1.4711,
      "nll_loss": 1.448738932609558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1439058631658554,
      "rewards/margins": 0.1265755295753479,
      "rewards/rejected": -0.2704813778400421,
      "step": 881
    },
    {
      "epoch": 1.3892828999211977,
      "grad_norm": 0.20596656203269958,
      "learning_rate": 1.0383236922695543e-06,
      "log_odds_chosen": 1.4967354536056519,
      "log_odds_ratio": -0.21175050735473633,
      "logits/chosen": -0.32488030195236206,
      "logits/rejected": -1.1130867004394531,
      "logps/chosen": -1.2785288095474243,
      "logps/rejected": -2.5162811279296875,
      "loss": 1.3347,
      "nll_loss": 1.3135534524917603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12785287201404572,
      "rewards/margins": 0.12377522885799408,
      "rewards/rejected": -0.2516281306743622,
      "step": 882
    },
    {
      "epoch": 1.3908589440504335,
      "grad_norm": 0.25251349806785583,
      "learning_rate": 1.0335074220522962e-06,
      "log_odds_chosen": 1.4807095527648926,
      "log_odds_ratio": -0.22595277428627014,
      "logits/chosen": -0.4294978678226471,
      "logits/rejected": -0.8683784604072571,
      "logps/chosen": -1.5218424797058105,
      "logps/rejected": -2.8242669105529785,
      "loss": 1.5548,
      "nll_loss": 1.5322010517120361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15218424797058105,
      "rewards/margins": 0.13024243712425232,
      "rewards/rejected": -0.28242671489715576,
      "step": 883
    },
    {
      "epoch": 1.392434988179669,
      "grad_norm": 0.25605475902557373,
      "learning_rate": 1.0286984532248326e-06,
      "log_odds_chosen": 1.0042290687561035,
      "log_odds_ratio": -0.3234960436820984,
      "logits/chosen": -0.3713316023349762,
      "logits/rejected": -1.1176303625106812,
      "logps/chosen": -1.5509390830993652,
      "logps/rejected": -2.410088300704956,
      "loss": 1.583,
      "nll_loss": 1.5506459474563599,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15509390830993652,
      "rewards/margins": 0.08591492474079132,
      "rewards/rejected": -0.24100883305072784,
      "step": 884
    },
    {
      "epoch": 1.3940110323089048,
      "grad_norm": 0.24771814048290253,
      "learning_rate": 1.0238968221166269e-06,
      "log_odds_chosen": 1.1720547676086426,
      "log_odds_ratio": -0.2909175753593445,
      "logits/chosen": -0.44203412532806396,
      "logits/rejected": -1.1457455158233643,
      "logps/chosen": -1.5098693370819092,
      "logps/rejected": -2.5236783027648926,
      "loss": 1.5523,
      "nll_loss": 1.5231854915618896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1509869247674942,
      "rewards/margins": 0.10138089209794998,
      "rewards/rejected": -0.2523678243160248,
      "step": 885
    },
    {
      "epoch": 1.3955870764381402,
      "grad_norm": 0.22360509634017944,
      "learning_rate": 1.019102565001707e-06,
      "log_odds_chosen": 1.188091516494751,
      "log_odds_ratio": -0.28113996982574463,
      "logits/chosen": -0.4140903055667877,
      "logits/rejected": -0.8816472887992859,
      "logps/chosen": -1.5419634580612183,
      "logps/rejected": -2.5729141235351562,
      "loss": 1.5723,
      "nll_loss": 1.5441887378692627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1541963517665863,
      "rewards/margins": 0.10309503227472305,
      "rewards/rejected": -0.25729137659072876,
      "step": 886
    },
    {
      "epoch": 1.397163120567376,
      "grad_norm": 0.27624866366386414,
      "learning_rate": 1.0143157180983965e-06,
      "log_odds_chosen": 1.1843692064285278,
      "log_odds_ratio": -0.28151312470436096,
      "logits/chosen": -0.47460079193115234,
      "logits/rejected": -1.069139838218689,
      "logps/chosen": -1.526496171951294,
      "logps/rejected": -2.5510191917419434,
      "loss": 1.5323,
      "nll_loss": 1.5041186809539795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526496261358261,
      "rewards/margins": 0.10245228558778763,
      "rewards/rejected": -0.25510191917419434,
      "step": 887
    },
    {
      "epoch": 1.3987391646966114,
      "grad_norm": 0.2208983153104782,
      "learning_rate": 1.0095363175690375e-06,
      "log_odds_chosen": 1.213573932647705,
      "log_odds_ratio": -0.2710980772972107,
      "logits/chosen": -0.49292194843292236,
      "logits/rejected": -0.921308696269989,
      "logps/chosen": -1.522839903831482,
      "logps/rejected": -2.571528911590576,
      "loss": 1.5414,
      "nll_loss": 1.5142542123794556,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15228401124477386,
      "rewards/margins": 0.10486888885498047,
      "rewards/rejected": -0.2571529150009155,
      "step": 888
    },
    {
      "epoch": 1.4003152088258473,
      "grad_norm": 0.22700989246368408,
      "learning_rate": 1.0047643995197178e-06,
      "log_odds_chosen": 1.3063966035842896,
      "log_odds_ratio": -0.250802606344223,
      "logits/chosen": -0.39796099066734314,
      "logits/rejected": -1.0632853507995605,
      "logps/chosen": -1.4333686828613281,
      "logps/rejected": -2.546491861343384,
      "loss": 1.4654,
      "nll_loss": 1.4402967691421509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14333687722682953,
      "rewards/margins": 0.11131229996681213,
      "rewards/rejected": -0.25464919209480286,
      "step": 889
    },
    {
      "epoch": 1.4018912529550827,
      "grad_norm": 0.2291909009218216,
      "learning_rate": 1.0000000000000004e-06,
      "log_odds_chosen": 1.1181808710098267,
      "log_odds_ratio": -0.2893332242965698,
      "logits/chosen": -0.3906664252281189,
      "logits/rejected": -1.0411070585250854,
      "logps/chosen": -1.4803695678710938,
      "logps/rejected": -2.427476644515991,
      "loss": 1.5287,
      "nll_loss": 1.4998010396957397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14803695678710938,
      "rewards/margins": 0.09471072256565094,
      "rewards/rejected": -0.24274767935276031,
      "step": 890
    },
    {
      "epoch": 1.4034672970843185,
      "grad_norm": 0.229339599609375,
      "learning_rate": 9.952431550026459e-07,
      "log_odds_chosen": 1.259032130241394,
      "log_odds_ratio": -0.2600708305835724,
      "logits/chosen": -0.5161879658699036,
      "logits/rejected": -1.0354653596878052,
      "logps/chosen": -1.5077309608459473,
      "logps/rejected": -2.598358154296875,
      "loss": 1.5433,
      "nll_loss": 1.5172942876815796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1507730782032013,
      "rewards/margins": 0.10906271636486053,
      "rewards/rejected": -0.2598358392715454,
      "step": 891
    },
    {
      "epoch": 1.405043341213554,
      "grad_norm": 0.2422095388174057,
      "learning_rate": 9.904939004633471e-07,
      "log_odds_chosen": 1.3811091184616089,
      "log_odds_ratio": -0.22861932218074799,
      "logits/chosen": -0.5773292779922485,
      "logits/rejected": -1.1708014011383057,
      "logps/chosen": -1.5717049837112427,
      "logps/rejected": -2.7810256481170654,
      "loss": 1.5819,
      "nll_loss": 1.559085488319397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15717048943042755,
      "rewards/margins": 0.12093205749988556,
      "rewards/rejected": -0.2781025469303131,
      "step": 892
    },
    {
      "epoch": 1.4066193853427895,
      "grad_norm": 0.24241942167282104,
      "learning_rate": 9.857522722604536e-07,
      "log_odds_chosen": 1.1048952341079712,
      "log_odds_ratio": -0.3112320601940155,
      "logits/chosen": -0.39980101585388184,
      "logits/rejected": -0.7909796237945557,
      "logps/chosen": -1.4434469938278198,
      "logps/rejected": -2.381676435470581,
      "loss": 1.4976,
      "nll_loss": 1.4664485454559326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14434470236301422,
      "rewards/margins": 0.09382294863462448,
      "rewards/rejected": -0.2381676584482193,
      "step": 893
    },
    {
      "epoch": 1.4081954294720251,
      "grad_norm": 0.23750005662441254,
      "learning_rate": 9.81018306214702e-07,
      "log_odds_chosen": 1.1862845420837402,
      "log_odds_ratio": -0.2730846107006073,
      "logits/chosen": -0.46723073720932007,
      "logits/rejected": -0.9479306936264038,
      "logps/chosen": -1.446435570716858,
      "logps/rejected": -2.4523262977600098,
      "loss": 1.502,
      "nll_loss": 1.4747289419174194,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14464356005191803,
      "rewards/margins": 0.10058906674385071,
      "rewards/rejected": -0.24523264169692993,
      "step": 894
    },
    {
      "epoch": 1.4097714736012608,
      "grad_norm": 0.2432183176279068,
      "learning_rate": 9.76292038088945e-07,
      "log_odds_chosen": 1.1407073736190796,
      "log_odds_ratio": -0.28316619992256165,
      "logits/chosen": -0.3931063115596771,
      "logits/rejected": -0.9391051530838013,
      "logps/chosen": -1.5297164916992188,
      "logps/rejected": -2.5124104022979736,
      "loss": 1.5557,
      "nll_loss": 1.5273460149765015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15297167003154755,
      "rewards/margins": 0.09826937317848206,
      "rewards/rejected": -0.2512410283088684,
      "step": 895
    },
    {
      "epoch": 1.4113475177304964,
      "grad_norm": 0.2184869796037674,
      "learning_rate": 9.715735035878799e-07,
      "log_odds_chosen": 1.3640148639678955,
      "log_odds_ratio": -0.2388024628162384,
      "logits/chosen": -0.4883621335029602,
      "logits/rejected": -1.0482102632522583,
      "logps/chosen": -1.4162169694900513,
      "logps/rejected": -2.577279806137085,
      "loss": 1.4488,
      "nll_loss": 1.4249364137649536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14162170886993408,
      "rewards/margins": 0.11610625684261322,
      "rewards/rejected": -0.2577279806137085,
      "step": 896
    },
    {
      "epoch": 1.412923561859732,
      "grad_norm": 0.20820306241512299,
      "learning_rate": 9.668627383577812e-07,
      "log_odds_chosen": 1.2566540241241455,
      "log_odds_ratio": -0.2861393988132477,
      "logits/chosen": -0.46984946727752686,
      "logits/rejected": -1.1305538415908813,
      "logps/chosen": -1.4988892078399658,
      "logps/rejected": -2.592288017272949,
      "loss": 1.5378,
      "nll_loss": 1.5091944932937622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14988893270492554,
      "rewards/margins": 0.1093398705124855,
      "rewards/rejected": -0.25922879576683044,
      "step": 897
    },
    {
      "epoch": 1.4144996059889676,
      "grad_norm": 0.23258733749389648,
      "learning_rate": 9.621597779862307e-07,
      "log_odds_chosen": 1.1137208938598633,
      "log_odds_ratio": -0.2901248037815094,
      "logits/chosen": -0.5670067667961121,
      "logits/rejected": -1.1112313270568848,
      "logps/chosen": -1.5049999952316284,
      "logps/rejected": -2.4555788040161133,
      "loss": 1.5363,
      "nll_loss": 1.5072691440582275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15050001442432404,
      "rewards/margins": 0.0950578972697258,
      "rewards/rejected": -0.24555790424346924,
      "step": 898
    },
    {
      "epoch": 1.4160756501182032,
      "grad_norm": 0.23953503370285034,
      "learning_rate": 9.57464658001848e-07,
      "log_odds_chosen": 1.1615594625473022,
      "log_odds_ratio": -0.2780800759792328,
      "logits/chosen": -0.4468817710876465,
      "logits/rejected": -1.0374969244003296,
      "logps/chosen": -1.5192381143569946,
      "logps/rejected": -2.5186662673950195,
      "loss": 1.5517,
      "nll_loss": 1.5239392518997192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15192382037639618,
      "rewards/margins": 0.09994282573461533,
      "rewards/rejected": -0.2518666386604309,
      "step": 899
    },
    {
      "epoch": 1.4176516942474389,
      "grad_norm": 0.2416912168264389,
      "learning_rate": 9.527774138740212e-07,
      "log_odds_chosen": 1.2771631479263306,
      "log_odds_ratio": -0.2532811462879181,
      "logits/chosen": -0.6083147525787354,
      "logits/rejected": -1.1319890022277832,
      "logps/chosen": -1.4746828079223633,
      "logps/rejected": -2.5635287761688232,
      "loss": 1.5129,
      "nll_loss": 1.4876124858856201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14746826887130737,
      "rewards/margins": 0.10888460278511047,
      "rewards/rejected": -0.25635284185409546,
      "step": 900
    },
    {
      "epoch": 1.4192277383766745,
      "grad_norm": 0.2291354387998581,
      "learning_rate": 9.480980810126411e-07,
      "log_odds_chosen": 1.3562443256378174,
      "log_odds_ratio": -0.26016074419021606,
      "logits/chosen": -0.5333552360534668,
      "logits/rejected": -1.0576164722442627,
      "logps/chosen": -1.373969554901123,
      "logps/rejected": -2.5386273860931396,
      "loss": 1.4252,
      "nll_loss": 1.3991349935531616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13739696145057678,
      "rewards/margins": 0.11646579205989838,
      "rewards/rejected": -0.25386273860931396,
      "step": 901
    },
    {
      "epoch": 1.42080378250591,
      "grad_norm": 0.243772491812706,
      "learning_rate": 9.434266947678324e-07,
      "log_odds_chosen": 1.2467873096466064,
      "log_odds_ratio": -0.26992088556289673,
      "logits/chosen": -0.5315362215042114,
      "logits/rejected": -1.0591105222702026,
      "logps/chosen": -1.467353343963623,
      "logps/rejected": -2.5401644706726074,
      "loss": 1.495,
      "nll_loss": 1.4680324792861938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14673534035682678,
      "rewards/margins": 0.10728108882904053,
      "rewards/rejected": -0.2540164291858673,
      "step": 902
    },
    {
      "epoch": 1.4223798266351457,
      "grad_norm": 0.22803054749965668,
      "learning_rate": 9.387632904296872e-07,
      "log_odds_chosen": 1.4677966833114624,
      "log_odds_ratio": -0.22388580441474915,
      "logits/chosen": -0.498870313167572,
      "logits/rejected": -1.1560969352722168,
      "logps/chosen": -1.490617275238037,
      "logps/rejected": -2.7693071365356445,
      "loss": 1.5136,
      "nll_loss": 1.4912127256393433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14906170964241028,
      "rewards/margins": 0.12786900997161865,
      "rewards/rejected": -0.2769307494163513,
      "step": 903
    },
    {
      "epoch": 1.4239558707643813,
      "grad_norm": 0.22817280888557434,
      "learning_rate": 9.341079032279986e-07,
      "log_odds_chosen": 1.4244046211242676,
      "log_odds_ratio": -0.23536738753318787,
      "logits/chosen": -0.4247323274612427,
      "logits/rejected": -1.0225040912628174,
      "logps/chosen": -1.4583475589752197,
      "logps/rejected": -2.689962148666382,
      "loss": 1.4955,
      "nll_loss": 1.4719250202178955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1458347588777542,
      "rewards/margins": 0.12316146492958069,
      "rewards/rejected": -0.2689962089061737,
      "step": 904
    },
    {
      "epoch": 1.425531914893617,
      "grad_norm": 0.21015721559524536,
      "learning_rate": 9.294605683319919e-07,
      "log_odds_chosen": 1.4184043407440186,
      "log_odds_ratio": -0.22777074575424194,
      "logits/chosen": -0.5354007482528687,
      "logits/rejected": -1.1131943464279175,
      "logps/chosen": -1.3411455154418945,
      "logps/rejected": -2.5405683517456055,
      "loss": 1.3792,
      "nll_loss": 1.3564625978469849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13411454856395721,
      "rewards/margins": 0.11994229257106781,
      "rewards/rejected": -0.254056841135025,
      "step": 905
    },
    {
      "epoch": 1.4271079590228526,
      "grad_norm": 0.23969052731990814,
      "learning_rate": 9.248213208500629e-07,
      "log_odds_chosen": 1.1282944679260254,
      "log_odds_ratio": -0.29622605443000793,
      "logits/chosen": -0.5672122836112976,
      "logits/rejected": -1.11539888381958,
      "logps/chosen": -1.5313770771026611,
      "logps/rejected": -2.503404140472412,
      "loss": 1.5519,
      "nll_loss": 1.5222562551498413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1531377136707306,
      "rewards/margins": 0.09720271825790405,
      "rewards/rejected": -0.25034043192863464,
      "step": 906
    },
    {
      "epoch": 1.4286840031520882,
      "grad_norm": 0.2438000589609146,
      "learning_rate": 9.201901958295115e-07,
      "log_odds_chosen": 1.3791134357452393,
      "log_odds_ratio": -0.24291890859603882,
      "logits/chosen": -0.49572768807411194,
      "logits/rejected": -1.0822786092758179,
      "logps/chosen": -1.4821187257766724,
      "logps/rejected": -2.6789534091949463,
      "loss": 1.5301,
      "nll_loss": 1.5058244466781616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14821186661720276,
      "rewards/margins": 0.11968345940113068,
      "rewards/rejected": -0.26789534091949463,
      "step": 907
    },
    {
      "epoch": 1.4302600472813238,
      "grad_norm": 0.21629725396633148,
      "learning_rate": 9.155672282562736e-07,
      "log_odds_chosen": 1.2493598461151123,
      "log_odds_ratio": -0.26273009181022644,
      "logits/chosen": -0.5882337093353271,
      "logits/rejected": -1.1518937349319458,
      "logps/chosen": -1.5013035535812378,
      "logps/rejected": -2.5799434185028076,
      "loss": 1.5404,
      "nll_loss": 1.514121413230896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15013036131858826,
      "rewards/margins": 0.10786399245262146,
      "rewards/rejected": -0.25799432396888733,
      "step": 908
    },
    {
      "epoch": 1.4318360914105595,
      "grad_norm": 0.2347356081008911,
      "learning_rate": 9.109524530546622e-07,
      "log_odds_chosen": 1.1326323747634888,
      "log_odds_ratio": -0.28518638014793396,
      "logits/chosen": -0.564480185508728,
      "logits/rejected": -1.1121978759765625,
      "logps/chosen": -1.5010037422180176,
      "logps/rejected": -2.4663476943969727,
      "loss": 1.542,
      "nll_loss": 1.5134718418121338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15010038018226624,
      "rewards/margins": 0.09653440117835999,
      "rewards/rejected": -0.24663476645946503,
      "step": 909
    },
    {
      "epoch": 1.433412135539795,
      "grad_norm": 0.26228731870651245,
      "learning_rate": 9.063459050871001e-07,
      "log_odds_chosen": 1.1044995784759521,
      "log_odds_ratio": -0.2949713468551636,
      "logits/chosen": -0.5899426341056824,
      "logits/rejected": -0.8904526233673096,
      "logps/chosen": -1.4717737436294556,
      "logps/rejected": -2.410245656967163,
      "loss": 1.5003,
      "nll_loss": 1.470845103263855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14717736840248108,
      "rewards/margins": 0.09384718537330627,
      "rewards/rejected": -0.24102455377578735,
      "step": 910
    },
    {
      "epoch": 1.4349881796690307,
      "grad_norm": 0.22420737147331238,
      "learning_rate": 9.017476191538555e-07,
      "log_odds_chosen": 1.340219497680664,
      "log_odds_ratio": -0.24607248604297638,
      "logits/chosen": -0.557191014289856,
      "logits/rejected": -0.9715001583099365,
      "logps/chosen": -1.420982003211975,
      "logps/rejected": -2.5603408813476562,
      "loss": 1.4681,
      "nll_loss": 1.4434876441955566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14209820330142975,
      "rewards/margins": 0.1139359176158905,
      "rewards/rejected": -0.25603410601615906,
      "step": 911
    },
    {
      "epoch": 1.4365642237982663,
      "grad_norm": 0.2309519499540329,
      "learning_rate": 8.971576299927832e-07,
      "log_odds_chosen": 1.4135029315948486,
      "log_odds_ratio": -0.23116764426231384,
      "logits/chosen": -0.519967794418335,
      "logits/rejected": -1.2319024801254272,
      "logps/chosen": -1.5085625648498535,
      "logps/rejected": -2.739762544631958,
      "loss": 1.5412,
      "nll_loss": 1.5180702209472656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15085627138614655,
      "rewards/margins": 0.12311998009681702,
      "rewards/rejected": -0.27397623658180237,
      "step": 912
    },
    {
      "epoch": 1.438140267927502,
      "grad_norm": 0.2276877611875534,
      "learning_rate": 8.925759722790591e-07,
      "log_odds_chosen": 1.23707115650177,
      "log_odds_ratio": -0.2620353102684021,
      "logits/chosen": -0.4842718243598938,
      "logits/rejected": -1.124516248703003,
      "logps/chosen": -1.4825104475021362,
      "logps/rejected": -2.544254779815674,
      "loss": 1.519,
      "nll_loss": 1.4927771091461182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14825105667114258,
      "rewards/margins": 0.10617443919181824,
      "rewards/rejected": -0.2544254958629608,
      "step": 913
    },
    {
      "epoch": 1.4397163120567376,
      "grad_norm": 0.24656160175800323,
      "learning_rate": 8.880026806249192e-07,
      "log_odds_chosen": 1.5142099857330322,
      "log_odds_ratio": -0.20852155983448029,
      "logits/chosen": -0.40701812505722046,
      "logits/rejected": -1.1775416135787964,
      "logps/chosen": -1.5421316623687744,
      "logps/rejected": -2.872865915298462,
      "loss": 1.5661,
      "nll_loss": 1.5452061891555786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15421316027641296,
      "rewards/margins": 0.13307344913482666,
      "rewards/rejected": -0.287286639213562,
      "step": 914
    },
    {
      "epoch": 1.4412923561859732,
      "grad_norm": 0.24856220185756683,
      "learning_rate": 8.834377895793965e-07,
      "log_odds_chosen": 1.1950743198394775,
      "log_odds_ratio": -0.2843016982078552,
      "logits/chosen": -0.4448228180408478,
      "logits/rejected": -1.0445208549499512,
      "logps/chosen": -1.6146800518035889,
      "logps/rejected": -2.6612002849578857,
      "loss": 1.642,
      "nll_loss": 1.6135315895080566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1614679992198944,
      "rewards/margins": 0.104652039706707,
      "rewards/rejected": -0.2661200165748596,
      "step": 915
    },
    {
      "epoch": 1.4428684003152088,
      "grad_norm": 0.23053759336471558,
      "learning_rate": 8.78881333628063e-07,
      "log_odds_chosen": 1.2762019634246826,
      "log_odds_ratio": -0.24987497925758362,
      "logits/chosen": -0.5229912996292114,
      "logits/rejected": -1.2520302534103394,
      "logps/chosen": -1.52783203125,
      "logps/rejected": -2.6330416202545166,
      "loss": 1.5491,
      "nll_loss": 1.5240962505340576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15278321504592896,
      "rewards/margins": 0.11052095890045166,
      "rewards/rejected": -0.2633041739463806,
      "step": 916
    },
    {
      "epoch": 1.4444444444444444,
      "grad_norm": 0.2274601012468338,
      "learning_rate": 8.743333471927672e-07,
      "log_odds_chosen": 1.3678957223892212,
      "log_odds_ratio": -0.23064440488815308,
      "logits/chosen": -0.5603076219558716,
      "logits/rejected": -1.1934438943862915,
      "logps/chosen": -1.5381139516830444,
      "logps/rejected": -2.731173515319824,
      "loss": 1.569,
      "nll_loss": 1.5459635257720947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15381139516830444,
      "rewards/margins": 0.11930596828460693,
      "rewards/rejected": -0.2731173634529114,
      "step": 917
    },
    {
      "epoch": 1.44602048857368,
      "grad_norm": 0.23243410885334015,
      "learning_rate": 8.697938646313742e-07,
      "log_odds_chosen": 1.2985018491744995,
      "log_odds_ratio": -0.24338120222091675,
      "logits/chosen": -0.5740074515342712,
      "logits/rejected": -1.1005221605300903,
      "logps/chosen": -1.443261742591858,
      "logps/rejected": -2.5511584281921387,
      "loss": 1.4739,
      "nll_loss": 1.449582815170288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14432616531848907,
      "rewards/margins": 0.11078968644142151,
      "rewards/rejected": -0.2551158666610718,
      "step": 918
    },
    {
      "epoch": 1.4475965327029157,
      "grad_norm": 0.23867270350456238,
      "learning_rate": 8.652629202375075e-07,
      "log_odds_chosen": 1.2081325054168701,
      "log_odds_ratio": -0.2761199474334717,
      "logits/chosen": -0.3565526008605957,
      "logits/rejected": -1.0872251987457275,
      "logps/chosen": -1.4505397081375122,
      "logps/rejected": -2.4810431003570557,
      "loss": 1.4991,
      "nll_loss": 1.4714428186416626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14505398273468018,
      "rewards/margins": 0.10305032879114151,
      "rewards/rejected": -0.2481042891740799,
      "step": 919
    },
    {
      "epoch": 1.4491725768321513,
      "grad_norm": 0.22519585490226746,
      "learning_rate": 8.607405482402861e-07,
      "log_odds_chosen": 1.5054830312728882,
      "log_odds_ratio": -0.22368517518043518,
      "logits/chosen": -0.5637195110321045,
      "logits/rejected": -1.155135989189148,
      "logps/chosen": -1.3372365236282349,
      "logps/rejected": -2.622675895690918,
      "loss": 1.38,
      "nll_loss": 1.3576359748840332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.133723646402359,
      "rewards/margins": 0.1285439431667328,
      "rewards/rejected": -0.2622675895690918,
      "step": 920
    },
    {
      "epoch": 1.450748620961387,
      "grad_norm": 0.22806285321712494,
      "learning_rate": 8.562267828040712e-07,
      "log_odds_chosen": 1.425764560699463,
      "log_odds_ratio": -0.22667354345321655,
      "logits/chosen": -0.5632104873657227,
      "logits/rejected": -1.316353678703308,
      "logps/chosen": -1.4671934843063354,
      "logps/rejected": -2.70287823677063,
      "loss": 1.4935,
      "nll_loss": 1.4708629846572876,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14671935141086578,
      "rewards/margins": 0.12356846034526825,
      "rewards/rejected": -0.27028781175613403,
      "step": 921
    },
    {
      "epoch": 1.4523246650906225,
      "grad_norm": 0.23085395991802216,
      "learning_rate": 8.517216580282048e-07,
      "log_odds_chosen": 1.1708112955093384,
      "log_odds_ratio": -0.2806692123413086,
      "logits/chosen": -0.42356687784194946,
      "logits/rejected": -1.1739113330841064,
      "logps/chosen": -1.5396034717559814,
      "logps/rejected": -2.5486061573028564,
      "loss": 1.5899,
      "nll_loss": 1.5618412494659424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15396034717559814,
      "rewards/margins": 0.10090027004480362,
      "rewards/rejected": -0.25486063957214355,
      "step": 922
    },
    {
      "epoch": 1.4539007092198581,
      "grad_norm": 0.25347068905830383,
      "learning_rate": 8.472252079467535e-07,
      "log_odds_chosen": 1.221853256225586,
      "log_odds_ratio": -0.2641017735004425,
      "logits/chosen": -0.5206030607223511,
      "logits/rejected": -1.1709809303283691,
      "logps/chosen": -1.517340898513794,
      "logps/rejected": -2.5704567432403564,
      "loss": 1.5512,
      "nll_loss": 1.5247725248336792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15173408389091492,
      "rewards/margins": 0.10531160980463028,
      "rewards/rejected": -0.2570456862449646,
      "step": 923
    },
    {
      "epoch": 1.4554767533490938,
      "grad_norm": 0.21951721608638763,
      "learning_rate": 8.427374665282488e-07,
      "log_odds_chosen": 1.0286931991577148,
      "log_odds_ratio": -0.30910027027130127,
      "logits/chosen": -0.38548630475997925,
      "logits/rejected": -0.9503060579299927,
      "logps/chosen": -1.4296391010284424,
      "logps/rejected": -2.2901575565338135,
      "loss": 1.4773,
      "nll_loss": 1.4463722705841064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1429639309644699,
      "rewards/margins": 0.08605185151100159,
      "rewards/rejected": -0.2290157675743103,
      "step": 924
    },
    {
      "epoch": 1.4570527974783294,
      "grad_norm": 0.2927302122116089,
      "learning_rate": 8.382584676754336e-07,
      "log_odds_chosen": 1.097365140914917,
      "log_odds_ratio": -0.29348045587539673,
      "logits/chosen": -0.547596275806427,
      "logits/rejected": -1.0367677211761475,
      "logps/chosen": -1.5122588872909546,
      "logps/rejected": -2.4530653953552246,
      "loss": 1.5491,
      "nll_loss": 1.5197961330413818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15122587978839874,
      "rewards/margins": 0.09408067166805267,
      "rewards/rejected": -0.2453065663576126,
      "step": 925
    },
    {
      "epoch": 1.458628841607565,
      "grad_norm": 0.22215890884399414,
      "learning_rate": 8.337882452250058e-07,
      "log_odds_chosen": 1.5822429656982422,
      "log_odds_ratio": -0.20570620894432068,
      "logits/chosen": -0.5610384345054626,
      "logits/rejected": -1.3084702491760254,
      "logps/chosen": -1.5319766998291016,
      "logps/rejected": -2.9337334632873535,
      "loss": 1.5552,
      "nll_loss": 1.534663438796997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15319769084453583,
      "rewards/margins": 0.14017567038536072,
      "rewards/rejected": -0.29337334632873535,
      "step": 926
    },
    {
      "epoch": 1.4602048857368006,
      "grad_norm": 0.23373226821422577,
      "learning_rate": 8.293268329473602e-07,
      "log_odds_chosen": 1.602201223373413,
      "log_odds_ratio": -0.20163081586360931,
      "logits/chosen": -0.588670015335083,
      "logits/rejected": -1.1862667798995972,
      "logps/chosen": -1.4707884788513184,
      "logps/rejected": -2.8774189949035645,
      "loss": 1.5009,
      "nll_loss": 1.4807301759719849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14707887172698975,
      "rewards/margins": 0.1406630128622055,
      "rewards/rejected": -0.28774186968803406,
      "step": 927
    },
    {
      "epoch": 1.4617809298660362,
      "grad_norm": 0.23032058775424957,
      "learning_rate": 8.248742645463367e-07,
      "log_odds_chosen": 1.3913533687591553,
      "log_odds_ratio": -0.23348787426948547,
      "logits/chosen": -0.5104522109031677,
      "logits/rejected": -1.1148470640182495,
      "logps/chosen": -1.541141390800476,
      "logps/rejected": -2.7589306831359863,
      "loss": 1.5568,
      "nll_loss": 1.533469557762146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15411412715911865,
      "rewards/margins": 0.1217789500951767,
      "rewards/rejected": -0.27589309215545654,
      "step": 928
    },
    {
      "epoch": 1.4633569739952719,
      "grad_norm": 0.2321007400751114,
      "learning_rate": 8.204305736589612e-07,
      "log_odds_chosen": 1.3415515422821045,
      "log_odds_ratio": -0.2517376244068146,
      "logits/chosen": -0.4968351721763611,
      "logits/rejected": -1.0054467916488647,
      "logps/chosen": -1.4843194484710693,
      "logps/rejected": -2.6470062732696533,
      "loss": 1.5124,
      "nll_loss": 1.4872469902038574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1484319418668747,
      "rewards/margins": 0.11626870930194855,
      "rewards/rejected": -0.26470068097114563,
      "step": 929
    },
    {
      "epoch": 1.4649330181245075,
      "grad_norm": 0.21468254923820496,
      "learning_rate": 8.159957938551966e-07,
      "log_odds_chosen": 1.2326208353042603,
      "log_odds_ratio": -0.2712433934211731,
      "logits/chosen": -0.46227264404296875,
      "logits/rejected": -1.0725722312927246,
      "logps/chosen": -1.4921149015426636,
      "logps/rejected": -2.547154426574707,
      "loss": 1.5305,
      "nll_loss": 1.5033750534057617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14921149611473083,
      "rewards/margins": 0.10550396144390106,
      "rewards/rejected": -0.2547154426574707,
      "step": 930
    },
    {
      "epoch": 1.4665090622537431,
      "grad_norm": 0.24245744943618774,
      "learning_rate": 8.115699586376865e-07,
      "log_odds_chosen": 1.218868374824524,
      "log_odds_ratio": -0.2658065855503082,
      "logits/chosen": -0.5657477974891663,
      "logits/rejected": -1.065493106842041,
      "logps/chosen": -1.5351643562316895,
      "logps/rejected": -2.5921943187713623,
      "loss": 1.5624,
      "nll_loss": 1.5357915163040161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15351644158363342,
      "rewards/margins": 0.10570300370454788,
      "rewards/rejected": -0.2592194378376007,
      "step": 931
    },
    {
      "epoch": 1.4680851063829787,
      "grad_norm": 0.2725280523300171,
      "learning_rate": 8.071531014415018e-07,
      "log_odds_chosen": 1.4912704229354858,
      "log_odds_ratio": -0.21387484669685364,
      "logits/chosen": -0.52660071849823,
      "logits/rejected": -1.0814998149871826,
      "logps/chosen": -1.4952937364578247,
      "logps/rejected": -2.799556255340576,
      "loss": 1.5059,
      "nll_loss": 1.484555959701538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14952939748764038,
      "rewards/margins": 0.13042624294757843,
      "rewards/rejected": -0.2799556255340576,
      "step": 932
    },
    {
      "epoch": 1.4696611505122144,
      "grad_norm": 0.22247099876403809,
      "learning_rate": 8.027452556338894e-07,
      "log_odds_chosen": 1.37318754196167,
      "log_odds_ratio": -0.2307034730911255,
      "logits/chosen": -0.5145717263221741,
      "logits/rejected": -1.3214309215545654,
      "logps/chosen": -1.4936453104019165,
      "logps/rejected": -2.6784701347351074,
      "loss": 1.5236,
      "nll_loss": 1.5005756616592407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14936454594135284,
      "rewards/margins": 0.11848248541355133,
      "rewards/rejected": -0.2678470015525818,
      "step": 933
    },
    {
      "epoch": 1.47123719464145,
      "grad_norm": 0.25509294867515564,
      "learning_rate": 7.983464545140179e-07,
      "log_odds_chosen": 1.5380477905273438,
      "log_odds_ratio": -0.199909508228302,
      "logits/chosen": -0.5718511939048767,
      "logits/rejected": -1.0699493885040283,
      "logps/chosen": -1.4320309162139893,
      "logps/rejected": -2.763169527053833,
      "loss": 1.4544,
      "nll_loss": 1.4343845844268799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14320310950279236,
      "rewards/margins": 0.13311387598514557,
      "rewards/rejected": -0.27631697058677673,
      "step": 934
    },
    {
      "epoch": 1.4728132387706856,
      "grad_norm": 0.24963222444057465,
      "learning_rate": 7.939567313127295e-07,
      "log_odds_chosen": 1.4071202278137207,
      "log_odds_ratio": -0.22343285381793976,
      "logits/chosen": -0.5277109146118164,
      "logits/rejected": -1.0694714784622192,
      "logps/chosen": -1.5046491622924805,
      "logps/rejected": -2.72735857963562,
      "loss": 1.5364,
      "nll_loss": 1.514044165611267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15046493709087372,
      "rewards/margins": 0.12227092683315277,
      "rewards/rejected": -0.2727358639240265,
      "step": 935
    },
    {
      "epoch": 1.4743892828999212,
      "grad_norm": 0.23046410083770752,
      "learning_rate": 7.89576119192286e-07,
      "log_odds_chosen": 1.4321949481964111,
      "log_odds_ratio": -0.22496697306632996,
      "logits/chosen": -0.4957532286643982,
      "logits/rejected": -1.1616541147232056,
      "logps/chosen": -1.4462517499923706,
      "logps/rejected": -2.6830947399139404,
      "loss": 1.4669,
      "nll_loss": 1.4444458484649658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14462518692016602,
      "rewards/margins": 0.12368427217006683,
      "rewards/rejected": -0.26830947399139404,
      "step": 936
    },
    {
      "epoch": 1.4759653270291568,
      "grad_norm": 0.23694641888141632,
      "learning_rate": 7.852046512461201e-07,
      "log_odds_chosen": 1.4917337894439697,
      "log_odds_ratio": -0.2158287912607193,
      "logits/chosen": -0.555746853351593,
      "logits/rejected": -1.184554100036621,
      "logps/chosen": -1.4619816541671753,
      "logps/rejected": -2.755159854888916,
      "loss": 1.4903,
      "nll_loss": 1.4687634706497192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14619815349578857,
      "rewards/margins": 0.12931782007217407,
      "rewards/rejected": -0.27551597356796265,
      "step": 937
    },
    {
      "epoch": 1.4775413711583925,
      "grad_norm": 0.2220073789358139,
      "learning_rate": 7.808423604985843e-07,
      "log_odds_chosen": 1.4883636236190796,
      "log_odds_ratio": -0.22011616826057434,
      "logits/chosen": -0.44736918807029724,
      "logits/rejected": -1.0429682731628418,
      "logps/chosen": -1.452591061592102,
      "logps/rejected": -2.7403714656829834,
      "loss": 1.5098,
      "nll_loss": 1.4877458810806274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14525911211967468,
      "rewards/margins": 0.12877802550792694,
      "rewards/rejected": -0.27403712272644043,
      "step": 938
    },
    {
      "epoch": 1.479117415287628,
      "grad_norm": 0.22835201025009155,
      "learning_rate": 7.764892799047005e-07,
      "log_odds_chosen": 1.5656852722167969,
      "log_odds_ratio": -0.2053748071193695,
      "logits/chosen": -0.5150584578514099,
      "logits/rejected": -1.2160230875015259,
      "logps/chosen": -1.5014896392822266,
      "logps/rejected": -2.8747475147247314,
      "loss": 1.5276,
      "nll_loss": 1.5070182085037231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15014897286891937,
      "rewards/margins": 0.13732576370239258,
      "rewards/rejected": -0.28747475147247314,
      "step": 939
    },
    {
      "epoch": 1.4806934594168637,
      "grad_norm": 0.23358474671840668,
      "learning_rate": 7.721454423499143e-07,
      "log_odds_chosen": 1.5624244213104248,
      "log_odds_ratio": -0.20779985189437866,
      "logits/chosen": -0.5545346140861511,
      "logits/rejected": -1.2257802486419678,
      "logps/chosen": -1.4814645051956177,
      "logps/rejected": -2.8450005054473877,
      "loss": 1.5103,
      "nll_loss": 1.4895291328430176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14814645051956177,
      "rewards/margins": 0.13635361194610596,
      "rewards/rejected": -0.2845000624656677,
      "step": 940
    },
    {
      "epoch": 1.4822695035460993,
      "grad_norm": 0.2509925365447998,
      "learning_rate": 7.678108806498441e-07,
      "log_odds_chosen": 1.486884593963623,
      "log_odds_ratio": -0.22265397012233734,
      "logits/chosen": -0.4839404225349426,
      "logits/rejected": -1.0287790298461914,
      "logps/chosen": -1.4549399614334106,
      "logps/rejected": -2.750178575515747,
      "loss": 1.4951,
      "nll_loss": 1.4728068113327026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1454939842224121,
      "rewards/margins": 0.1295238584280014,
      "rewards/rejected": -0.2750178575515747,
      "step": 941
    },
    {
      "epoch": 1.483845547675335,
      "grad_norm": 0.27367255091667175,
      "learning_rate": 7.634856275500315e-07,
      "log_odds_chosen": 1.154842495918274,
      "log_odds_ratio": -0.2872955799102783,
      "logits/chosen": -0.4990924894809723,
      "logits/rejected": -0.9579907655715942,
      "logps/chosen": -1.5973601341247559,
      "logps/rejected": -2.6065640449523926,
      "loss": 1.626,
      "nll_loss": 1.597232699394226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1597360223531723,
      "rewards/margins": 0.10092037916183472,
      "rewards/rejected": -0.2606564164161682,
      "step": 942
    },
    {
      "epoch": 1.4854215918045706,
      "grad_norm": 0.21209317445755005,
      "learning_rate": 7.591697157256991e-07,
      "log_odds_chosen": 1.409385085105896,
      "log_odds_ratio": -0.23340846598148346,
      "logits/chosen": -0.49798765778541565,
      "logits/rejected": -1.1521496772766113,
      "logps/chosen": -1.4210861921310425,
      "logps/rejected": -2.6285037994384766,
      "loss": 1.4535,
      "nll_loss": 1.4301966428756714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14210861921310425,
      "rewards/margins": 0.12074176967144012,
      "rewards/rejected": -0.26285040378570557,
      "step": 943
    },
    {
      "epoch": 1.4869976359338062,
      "grad_norm": 0.2431243360042572,
      "learning_rate": 7.548631777814995e-07,
      "log_odds_chosen": 1.5920264720916748,
      "log_odds_ratio": -0.20379099249839783,
      "logits/chosen": -0.4855715334415436,
      "logits/rejected": -1.055014729499817,
      "logps/chosen": -1.47369384765625,
      "logps/rejected": -2.8693430423736572,
      "loss": 1.5086,
      "nll_loss": 1.4882118701934814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.147369384765625,
      "rewards/margins": 0.1395649015903473,
      "rewards/rejected": -0.2869343161582947,
      "step": 944
    },
    {
      "epoch": 1.4885736800630418,
      "grad_norm": 0.226885586977005,
      "learning_rate": 7.505660462512689e-07,
      "log_odds_chosen": 1.303566336631775,
      "log_odds_ratio": -0.24759596586227417,
      "logits/chosen": -0.566405713558197,
      "logits/rejected": -1.0458842515945435,
      "logps/chosen": -1.4012614488601685,
      "logps/rejected": -2.503734588623047,
      "loss": 1.452,
      "nll_loss": 1.4272035360336304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14012613892555237,
      "rewards/margins": 0.11024729907512665,
      "rewards/rejected": -0.2503734230995178,
      "step": 945
    },
    {
      "epoch": 1.4901497241922774,
      "grad_norm": 0.23939156532287598,
      "learning_rate": 7.462783535977842e-07,
      "log_odds_chosen": 1.482673168182373,
      "log_odds_ratio": -0.20977656543254852,
      "logits/chosen": -0.5286505818367004,
      "logits/rejected": -1.1272447109222412,
      "logps/chosen": -1.4555305242538452,
      "logps/rejected": -2.738517999649048,
      "loss": 1.4866,
      "nll_loss": 1.4656636714935303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14555305242538452,
      "rewards/margins": 0.12829875946044922,
      "rewards/rejected": -0.27385181188583374,
      "step": 946
    },
    {
      "epoch": 1.491725768321513,
      "grad_norm": 0.22249917685985565,
      "learning_rate": 7.420001322125156e-07,
      "log_odds_chosen": 0.9814853072166443,
      "log_odds_ratio": -0.33113470673561096,
      "logits/chosen": -0.6564266085624695,
      "logits/rejected": -1.0469541549682617,
      "logps/chosen": -1.4330018758773804,
      "logps/rejected": -2.260852098464966,
      "loss": 1.4735,
      "nll_loss": 1.4403642416000366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14330019056797028,
      "rewards/margins": 0.08278501778841019,
      "rewards/rejected": -0.22608521580696106,
      "step": 947
    },
    {
      "epoch": 1.4933018124507487,
      "grad_norm": 0.23276977241039276,
      "learning_rate": 7.377314144153814e-07,
      "log_odds_chosen": 1.107121229171753,
      "log_odds_ratio": -0.290560781955719,
      "logits/chosen": -0.583237886428833,
      "logits/rejected": -1.1113476753234863,
      "logps/chosen": -1.5114789009094238,
      "logps/rejected": -2.4587900638580322,
      "loss": 1.5522,
      "nll_loss": 1.5231281518936157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15114790201187134,
      "rewards/margins": 0.09473110735416412,
      "rewards/rejected": -0.24587899446487427,
      "step": 948
    },
    {
      "epoch": 1.4948778565799843,
      "grad_norm": 0.23013441264629364,
      "learning_rate": 7.334722324545064e-07,
      "log_odds_chosen": 1.457214117050171,
      "log_odds_ratio": -0.2198820412158966,
      "logits/chosen": -0.47068580985069275,
      "logits/rejected": -1.2165051698684692,
      "logps/chosen": -1.465359091758728,
      "logps/rejected": -2.7277474403381348,
      "loss": 1.4796,
      "nll_loss": 1.4576427936553955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14653591811656952,
      "rewards/margins": 0.1262388378381729,
      "rewards/rejected": -0.27277475595474243,
      "step": 949
    },
    {
      "epoch": 1.49645390070922,
      "grad_norm": 0.2346450388431549,
      "learning_rate": 7.292226185059756e-07,
      "log_odds_chosen": 1.280535340309143,
      "log_odds_ratio": -0.25280797481536865,
      "logits/chosen": -0.4806886911392212,
      "logits/rejected": -1.0456727743148804,
      "logps/chosen": -1.5331767797470093,
      "logps/rejected": -2.6470608711242676,
      "loss": 1.5735,
      "nll_loss": 1.5482414960861206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1533176749944687,
      "rewards/margins": 0.1113884299993515,
      "rewards/rejected": -0.2647061049938202,
      "step": 950
    },
    {
      "epoch": 1.4980299448384555,
      "grad_norm": 0.2279590666294098,
      "learning_rate": 7.249826046735927e-07,
      "log_odds_chosen": 1.385414481163025,
      "log_odds_ratio": -0.23294499516487122,
      "logits/chosen": -0.5691102743148804,
      "logits/rejected": -1.2125122547149658,
      "logps/chosen": -1.4664280414581299,
      "logps/rejected": -2.664513111114502,
      "loss": 1.4963,
      "nll_loss": 1.4730455875396729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14664281904697418,
      "rewards/margins": 0.11980851739645004,
      "rewards/rejected": -0.26645132899284363,
      "step": 951
    },
    {
      "epoch": 1.4996059889676912,
      "grad_norm": 0.2652284801006317,
      "learning_rate": 7.207522229886379e-07,
      "log_odds_chosen": 1.4250870943069458,
      "log_odds_ratio": -0.22942163050174713,
      "logits/chosen": -0.5447030067443848,
      "logits/rejected": -0.9880243539810181,
      "logps/chosen": -1.3955967426300049,
      "logps/rejected": -2.6106948852539062,
      "loss": 1.4329,
      "nll_loss": 1.4099078178405762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13955968618392944,
      "rewards/margins": 0.12150982022285461,
      "rewards/rejected": -0.26106947660446167,
      "step": 952
    },
    {
      "epoch": 1.5011820330969265,
      "grad_norm": 0.24431435763835907,
      "learning_rate": 7.165315054096228e-07,
      "log_odds_chosen": 1.3399044275283813,
      "log_odds_ratio": -0.24496197700500488,
      "logits/chosen": -0.5608055591583252,
      "logits/rejected": -1.1914477348327637,
      "logps/chosen": -1.540886640548706,
      "logps/rejected": -2.7111942768096924,
      "loss": 1.5571,
      "nll_loss": 1.5326396226882935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15408867597579956,
      "rewards/margins": 0.11703076958656311,
      "rewards/rejected": -0.27111944556236267,
      "step": 953
    },
    {
      "epoch": 1.5027580772261624,
      "grad_norm": 0.2183518260717392,
      "learning_rate": 7.123204838220533e-07,
      "log_odds_chosen": 1.5956528186798096,
      "log_odds_ratio": -0.2071644812822342,
      "logits/chosen": -0.6269218325614929,
      "logits/rejected": -1.2425068616867065,
      "logps/chosen": -1.4194064140319824,
      "logps/rejected": -2.810275077819824,
      "loss": 1.448,
      "nll_loss": 1.4272733926773071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.141940638422966,
      "rewards/margins": 0.13908687233924866,
      "rewards/rejected": -0.28102749586105347,
      "step": 954
    },
    {
      "epoch": 1.5043341213553978,
      "grad_norm": 0.24050156772136688,
      "learning_rate": 7.081191900381862e-07,
      "log_odds_chosen": 1.3933483362197876,
      "log_odds_ratio": -0.22965580224990845,
      "logits/chosen": -0.5546469688415527,
      "logits/rejected": -1.1882423162460327,
      "logps/chosen": -1.536947250366211,
      "logps/rejected": -2.75390362739563,
      "loss": 1.5695,
      "nll_loss": 1.546506643295288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15369471907615662,
      "rewards/margins": 0.12169566005468369,
      "rewards/rejected": -0.2753903865814209,
      "step": 955
    },
    {
      "epoch": 1.5059101654846336,
      "grad_norm": 0.22683848440647125,
      "learning_rate": 7.039276557967895e-07,
      "log_odds_chosen": 1.767035961151123,
      "log_odds_ratio": -0.16716304421424866,
      "logits/chosen": -0.5107672810554504,
      "logits/rejected": -1.200775146484375,
      "logps/chosen": -1.4198424816131592,
      "logps/rejected": -2.9595937728881836,
      "loss": 1.4573,
      "nll_loss": 1.4405633211135864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419842541217804,
      "rewards/margins": 0.15397511422634125,
      "rewards/rejected": -0.29595935344696045,
      "step": 956
    },
    {
      "epoch": 1.507486209613869,
      "grad_norm": 0.22145600616931915,
      "learning_rate": 6.997459127629028e-07,
      "log_odds_chosen": 1.156578540802002,
      "log_odds_ratio": -0.29022669792175293,
      "logits/chosen": -0.5374658107757568,
      "logits/rejected": -1.09673011302948,
      "logps/chosen": -1.4964768886566162,
      "logps/rejected": -2.4902737140655518,
      "loss": 1.5464,
      "nll_loss": 1.5173897743225098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14964769780635834,
      "rewards/margins": 0.09937969595193863,
      "rewards/rejected": -0.24902740120887756,
      "step": 957
    },
    {
      "epoch": 1.5090622537431049,
      "grad_norm": 0.23631010949611664,
      "learning_rate": 6.955739925275963e-07,
      "log_odds_chosen": 1.7793992757797241,
      "log_odds_ratio": -0.1729913204908371,
      "logits/chosen": -0.5843755602836609,
      "logits/rejected": -1.1253644227981567,
      "logps/chosen": -1.4159923791885376,
      "logps/rejected": -2.9696507453918457,
      "loss": 1.4513,
      "nll_loss": 1.4339982271194458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14159925282001495,
      "rewards/margins": 0.15536582469940186,
      "rewards/rejected": -0.296965092420578,
      "step": 958
    },
    {
      "epoch": 1.5106382978723403,
      "grad_norm": 0.23230597376823425,
      "learning_rate": 6.914119266077354e-07,
      "log_odds_chosen": 1.493029236793518,
      "log_odds_ratio": -0.20502763986587524,
      "logits/chosen": -0.45821213722229004,
      "logits/rejected": -1.2774142026901245,
      "logps/chosen": -1.5468357801437378,
      "logps/rejected": -2.859137535095215,
      "loss": 1.5794,
      "nll_loss": 1.5589423179626465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15468357503414154,
      "rewards/margins": 0.1312301605939865,
      "rewards/rejected": -0.28591373562812805,
      "step": 959
    },
    {
      "epoch": 1.5122143420015761,
      "grad_norm": 0.24414998292922974,
      "learning_rate": 6.872597464457397e-07,
      "log_odds_chosen": 1.2731674909591675,
      "log_odds_ratio": -0.2607763707637787,
      "logits/chosen": -0.5055373311042786,
      "logits/rejected": -1.0789213180541992,
      "logps/chosen": -1.4229998588562012,
      "logps/rejected": -2.5055572986602783,
      "loss": 1.451,
      "nll_loss": 1.424925684928894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14229997992515564,
      "rewards/margins": 0.10825575143098831,
      "rewards/rejected": -0.25055575370788574,
      "step": 960
    },
    {
      "epoch": 1.5137903861308115,
      "grad_norm": 0.23284637928009033,
      "learning_rate": 6.831174834093476e-07,
      "log_odds_chosen": 1.324260950088501,
      "log_odds_ratio": -0.24578021466732025,
      "logits/chosen": -0.5531517267227173,
      "logits/rejected": -1.094421625137329,
      "logps/chosen": -1.4508875608444214,
      "logps/rejected": -2.588124990463257,
      "loss": 1.4792,
      "nll_loss": 1.4546388387680054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14508874714374542,
      "rewards/margins": 0.1137237474322319,
      "rewards/rejected": -0.25881248712539673,
      "step": 961
    },
    {
      "epoch": 1.5153664302600474,
      "grad_norm": 0.24024493992328644,
      "learning_rate": 6.789851687913784e-07,
      "log_odds_chosen": 1.2035846710205078,
      "log_odds_ratio": -0.28860709071159363,
      "logits/chosen": -0.5882235765457153,
      "logits/rejected": -1.1736358404159546,
      "logps/chosen": -1.3984804153442383,
      "logps/rejected": -2.423901319503784,
      "loss": 1.4369,
      "nll_loss": 1.4080098867416382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13984805345535278,
      "rewards/margins": 0.10254207253456116,
      "rewards/rejected": -0.24239011108875275,
      "step": 962
    },
    {
      "epoch": 1.5169424743892828,
      "grad_norm": 0.23437634110450745,
      "learning_rate": 6.748628338094937e-07,
      "log_odds_chosen": 1.5645782947540283,
      "log_odds_ratio": -0.20110386610031128,
      "logits/chosen": -0.512070894241333,
      "logits/rejected": -1.0736764669418335,
      "logps/chosen": -1.414776086807251,
      "logps/rejected": -2.7635304927825928,
      "loss": 1.4482,
      "nll_loss": 1.4281259775161743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14147762954235077,
      "rewards/margins": 0.13487544655799866,
      "rewards/rejected": -0.27635306119918823,
      "step": 963
    },
    {
      "epoch": 1.5185185185185186,
      "grad_norm": 0.21932587027549744,
      "learning_rate": 6.707505096059662e-07,
      "log_odds_chosen": 1.4649726152420044,
      "log_odds_ratio": -0.21752658486366272,
      "logits/chosen": -0.5394257307052612,
      "logits/rejected": -1.186219573020935,
      "logps/chosen": -1.4192025661468506,
      "logps/rejected": -2.6823384761810303,
      "loss": 1.4628,
      "nll_loss": 1.44109046459198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14192025363445282,
      "rewards/margins": 0.12631359696388245,
      "rewards/rejected": -0.2682338356971741,
      "step": 964
    },
    {
      "epoch": 1.520094562647754,
      "grad_norm": 0.23025809228420258,
      "learning_rate": 6.666482272474412e-07,
      "log_odds_chosen": 1.3805735111236572,
      "log_odds_ratio": -0.22953107953071594,
      "logits/chosen": -0.532515823841095,
      "logits/rejected": -1.1705037355422974,
      "logps/chosen": -1.4748280048370361,
      "logps/rejected": -2.6676025390625,
      "loss": 1.5066,
      "nll_loss": 1.4836927652359009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14748279750347137,
      "rewards/margins": 0.11927744746208191,
      "rewards/rejected": -0.2667602598667145,
      "step": 965
    },
    {
      "epoch": 1.5216706067769898,
      "grad_norm": 0.23901532590389252,
      "learning_rate": 6.625560177247023e-07,
      "log_odds_chosen": 1.4248603582382202,
      "log_odds_ratio": -0.22349603474140167,
      "logits/chosen": -0.5546669363975525,
      "logits/rejected": -1.2032575607299805,
      "logps/chosen": -1.572448492050171,
      "logps/rejected": -2.8211164474487305,
      "loss": 1.5959,
      "nll_loss": 1.573559284210205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15724486112594604,
      "rewards/margins": 0.1248667910695076,
      "rewards/rejected": -0.28211167454719543,
      "step": 966
    },
    {
      "epoch": 1.5232466509062252,
      "grad_norm": 0.25810325145721436,
      "learning_rate": 6.584739119524383e-07,
      "log_odds_chosen": 1.5695710182189941,
      "log_odds_ratio": -0.2057805061340332,
      "logits/chosen": -0.4918912351131439,
      "logits/rejected": -1.1777918338775635,
      "logps/chosen": -1.440580129623413,
      "logps/rejected": -2.805605888366699,
      "loss": 1.4563,
      "nll_loss": 1.4357250928878784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1440580189228058,
      "rewards/margins": 0.13650256395339966,
      "rewards/rejected": -0.28056058287620544,
      "step": 967
    },
    {
      "epoch": 1.524822695035461,
      "grad_norm": 0.23448987305164337,
      "learning_rate": 6.544019407690077e-07,
      "log_odds_chosen": 1.304165244102478,
      "log_odds_ratio": -0.252460777759552,
      "logits/chosen": -0.5383502244949341,
      "logits/rejected": -1.0561842918395996,
      "logps/chosen": -1.5359160900115967,
      "logps/rejected": -2.6730666160583496,
      "loss": 1.5574,
      "nll_loss": 1.5321393013000488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1535916030406952,
      "rewards/margins": 0.11371506750583649,
      "rewards/rejected": -0.26730671525001526,
      "step": 968
    },
    {
      "epoch": 1.5263987391646965,
      "grad_norm": 0.23763923346996307,
      "learning_rate": 6.503401349362083e-07,
      "log_odds_chosen": 1.3596715927124023,
      "log_odds_ratio": -0.2394452840089798,
      "logits/chosen": -0.675857424736023,
      "logits/rejected": -1.2328929901123047,
      "logps/chosen": -1.5621181726455688,
      "logps/rejected": -2.7546143531799316,
      "loss": 1.5883,
      "nll_loss": 1.5643137693405151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15621183812618256,
      "rewards/margins": 0.11924959719181061,
      "rewards/rejected": -0.27546143531799316,
      "step": 969
    },
    {
      "epoch": 1.5279747832939323,
      "grad_norm": 0.23846010863780975,
      "learning_rate": 6.462885251390433e-07,
      "log_odds_chosen": 1.2473676204681396,
      "log_odds_ratio": -0.2593209147453308,
      "logits/chosen": -0.6061593294143677,
      "logits/rejected": -1.0851634740829468,
      "logps/chosen": -1.5588171482086182,
      "logps/rejected": -2.643918514251709,
      "loss": 1.5738,
      "nll_loss": 1.5478794574737549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15588173270225525,
      "rewards/margins": 0.10851012170314789,
      "rewards/rejected": -0.26439183950424194,
      "step": 970
    },
    {
      "epoch": 1.5295508274231677,
      "grad_norm": 0.2130812108516693,
      "learning_rate": 6.422471419854898e-07,
      "log_odds_chosen": 1.4767725467681885,
      "log_odds_ratio": -0.21253429353237152,
      "logits/chosen": -0.5180621147155762,
      "logits/rejected": -1.144821286201477,
      "logps/chosen": -1.3797147274017334,
      "logps/rejected": -2.6386866569519043,
      "loss": 1.4151,
      "nll_loss": 1.393884301185608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13797146081924438,
      "rewards/margins": 0.12589721381664276,
      "rewards/rejected": -0.26386868953704834,
      "step": 971
    },
    {
      "epoch": 1.5311268715524036,
      "grad_norm": 0.2518880367279053,
      "learning_rate": 6.382160160062662e-07,
      "log_odds_chosen": 1.7233786582946777,
      "log_odds_ratio": -0.18346856534481049,
      "logits/chosen": -0.4957371652126312,
      "logits/rejected": -1.1163181066513062,
      "logps/chosen": -1.4778578281402588,
      "logps/rejected": -2.997682809829712,
      "loss": 1.5083,
      "nll_loss": 1.4899775981903076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14778578281402588,
      "rewards/margins": 0.15198251605033875,
      "rewards/rejected": -0.2997682988643646,
      "step": 972
    },
    {
      "epoch": 1.532702915681639,
      "grad_norm": 0.24372602999210358,
      "learning_rate": 6.341951776546044e-07,
      "log_odds_chosen": 1.3304800987243652,
      "log_odds_ratio": -0.24658480286598206,
      "logits/chosen": -0.46883171796798706,
      "logits/rejected": -1.1417944431304932,
      "logps/chosen": -1.4798043966293335,
      "logps/rejected": -2.62838077545166,
      "loss": 1.508,
      "nll_loss": 1.483365774154663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479804515838623,
      "rewards/margins": 0.11485762894153595,
      "rewards/rejected": -0.26283809542655945,
      "step": 973
    },
    {
      "epoch": 1.5342789598108748,
      "grad_norm": 0.2315274477005005,
      "learning_rate": 6.301846573060177e-07,
      "log_odds_chosen": 1.1873555183410645,
      "log_odds_ratio": -0.2747114598751068,
      "logits/chosen": -0.5921677350997925,
      "logits/rejected": -1.0610358715057373,
      "logps/chosen": -1.429610252380371,
      "logps/rejected": -2.4378793239593506,
      "loss": 1.4733,
      "nll_loss": 1.445785403251648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1429610401391983,
      "rewards/margins": 0.10082691162824631,
      "rewards/rejected": -0.243787944316864,
      "step": 974
    },
    {
      "epoch": 1.5358550039401102,
      "grad_norm": 0.23147934675216675,
      "learning_rate": 6.261844852580722e-07,
      "log_odds_chosen": 1.4448254108428955,
      "log_odds_ratio": -0.23359642922878265,
      "logits/chosen": -0.5760229825973511,
      "logits/rejected": -0.9010260701179504,
      "logps/chosen": -1.3827587366104126,
      "logps/rejected": -2.624427556991577,
      "loss": 1.4224,
      "nll_loss": 1.3990039825439453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1382758766412735,
      "rewards/margins": 0.12416687607765198,
      "rewards/rejected": -0.26244276762008667,
      "step": 975
    },
    {
      "epoch": 1.537431048069346,
      "grad_norm": 0.22817595303058624,
      "learning_rate": 6.221946917301561e-07,
      "log_odds_chosen": 1.4606870412826538,
      "log_odds_ratio": -0.22092492878437042,
      "logits/chosen": -0.4953918755054474,
      "logits/rejected": -1.1805686950683594,
      "logps/chosen": -1.4747236967086792,
      "logps/rejected": -2.7418582439422607,
      "loss": 1.5057,
      "nll_loss": 1.4835586547851562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14747236669063568,
      "rewards/margins": 0.12671343982219696,
      "rewards/rejected": -0.27418580651283264,
      "step": 976
    },
    {
      "epoch": 1.5390070921985815,
      "grad_norm": 0.23945246636867523,
      "learning_rate": 6.182153068632545e-07,
      "log_odds_chosen": 1.4020558595657349,
      "log_odds_ratio": -0.23104116320610046,
      "logits/chosen": -0.5097079277038574,
      "logits/rejected": -1.1956994533538818,
      "logps/chosen": -1.490002155303955,
      "logps/rejected": -2.7096667289733887,
      "loss": 1.5332,
      "nll_loss": 1.510090708732605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14900021255016327,
      "rewards/margins": 0.12196645885705948,
      "rewards/rejected": -0.27096667885780334,
      "step": 977
    },
    {
      "epoch": 1.5405831363278173,
      "grad_norm": 0.24623700976371765,
      "learning_rate": 6.142463607197197e-07,
      "log_odds_chosen": 1.4083584547042847,
      "log_odds_ratio": -0.23126868903636932,
      "logits/chosen": -0.5768950581550598,
      "logits/rejected": -1.2269421815872192,
      "logps/chosen": -1.5024503469467163,
      "logps/rejected": -2.7304129600524902,
      "loss": 1.5184,
      "nll_loss": 1.4952857494354248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1502450406551361,
      "rewards/margins": 0.12279626727104187,
      "rewards/rejected": -0.273041307926178,
      "step": 978
    },
    {
      "epoch": 1.5421591804570527,
      "grad_norm": 0.22342562675476074,
      "learning_rate": 6.102878832830431e-07,
      "log_odds_chosen": 1.3728972673416138,
      "log_odds_ratio": -0.2313397377729416,
      "logits/chosen": -0.6124773621559143,
      "logits/rejected": -1.3108428716659546,
      "logps/chosen": -1.4976592063903809,
      "logps/rejected": -2.6899378299713135,
      "loss": 1.5366,
      "nll_loss": 1.5134751796722412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14976592361927032,
      "rewards/margins": 0.11922784149646759,
      "rewards/rejected": -0.2689937353134155,
      "step": 979
    },
    {
      "epoch": 1.5437352245862885,
      "grad_norm": 0.22539131343364716,
      "learning_rate": 6.063399044576316e-07,
      "log_odds_chosen": 1.492227554321289,
      "log_odds_ratio": -0.21251127123832703,
      "logits/chosen": -0.5483823418617249,
      "logits/rejected": -1.137655258178711,
      "logps/chosen": -1.4453543424606323,
      "logps/rejected": -2.736375570297241,
      "loss": 1.4795,
      "nll_loss": 1.4582099914550781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14453545212745667,
      "rewards/margins": 0.12910211086273193,
      "rewards/rejected": -0.2736375629901886,
      "step": 980
    },
    {
      "epoch": 1.545311268715524,
      "grad_norm": 0.2323814481496811,
      "learning_rate": 6.024024540685802e-07,
      "log_odds_chosen": 1.4258785247802734,
      "log_odds_ratio": -0.2345668524503708,
      "logits/chosen": -0.4949217438697815,
      "logits/rejected": -1.0727516412734985,
      "logps/chosen": -1.443427324295044,
      "logps/rejected": -2.6735379695892334,
      "loss": 1.4813,
      "nll_loss": 1.4578876495361328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14434273540973663,
      "rewards/margins": 0.12301107496023178,
      "rewards/rejected": -0.2673538029193878,
      "step": 981
    },
    {
      "epoch": 1.5468873128447598,
      "grad_norm": 0.23888812959194183,
      "learning_rate": 5.984755618614443e-07,
      "log_odds_chosen": 1.3218895196914673,
      "log_odds_ratio": -0.24549773335456848,
      "logits/chosen": -0.612534761428833,
      "logits/rejected": -1.2285454273223877,
      "logps/chosen": -1.4884923696517944,
      "logps/rejected": -2.6302218437194824,
      "loss": 1.5196,
      "nll_loss": 1.4950504302978516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1488492488861084,
      "rewards/margins": 0.11417294293642044,
      "rewards/rejected": -0.26302218437194824,
      "step": 982
    },
    {
      "epoch": 1.5484633569739952,
      "grad_norm": 0.219101220369339,
      "learning_rate": 5.945592575020199e-07,
      "log_odds_chosen": 1.400936484336853,
      "log_odds_ratio": -0.2356840819120407,
      "logits/chosen": -0.4243534207344055,
      "logits/rejected": -1.0595777034759521,
      "logps/chosen": -1.4558807611465454,
      "logps/rejected": -2.6673097610473633,
      "loss": 1.4984,
      "nll_loss": 1.4748611450195312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14558807015419006,
      "rewards/margins": 0.1211429089307785,
      "rewards/rejected": -0.26673099398612976,
      "step": 983
    },
    {
      "epoch": 1.550039401103231,
      "grad_norm": 0.25756168365478516,
      "learning_rate": 5.90653570576116e-07,
      "log_odds_chosen": 1.724707007408142,
      "log_odds_ratio": -0.18645404279232025,
      "logits/chosen": -0.6291805505752563,
      "logits/rejected": -1.1717040538787842,
      "logps/chosen": -1.5127263069152832,
      "logps/rejected": -3.0439517498016357,
      "loss": 1.5366,
      "nll_loss": 1.5179901123046875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15127262473106384,
      "rewards/margins": 0.15312254428863525,
      "rewards/rejected": -0.3043951988220215,
      "step": 984
    },
    {
      "epoch": 1.5516154452324664,
      "grad_norm": 0.2433800995349884,
      "learning_rate": 5.867585305893315e-07,
      "log_odds_chosen": 1.3656306266784668,
      "log_odds_ratio": -0.24259380996227264,
      "logits/chosen": -0.5985369682312012,
      "logits/rejected": -1.0483858585357666,
      "logps/chosen": -1.427693247795105,
      "logps/rejected": -2.602436065673828,
      "loss": 1.4734,
      "nll_loss": 1.4491654634475708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14276932179927826,
      "rewards/margins": 0.11747428774833679,
      "rewards/rejected": -0.26024362444877625,
      "step": 985
    },
    {
      "epoch": 1.5531914893617023,
      "grad_norm": 0.22279983758926392,
      "learning_rate": 5.828741669668337e-07,
      "log_odds_chosen": 1.692441463470459,
      "log_odds_ratio": -0.17523367702960968,
      "logits/chosen": -0.5229300856590271,
      "logits/rejected": -1.2200078964233398,
      "logps/chosen": -1.4838160276412964,
      "logps/rejected": -2.972411870956421,
      "loss": 1.5013,
      "nll_loss": 1.4838006496429443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14838162064552307,
      "rewards/margins": 0.14885959029197693,
      "rewards/rejected": -0.2972412109375,
      "step": 986
    },
    {
      "epoch": 1.5547675334909377,
      "grad_norm": 0.2178654819726944,
      "learning_rate": 5.790005090531333e-07,
      "log_odds_chosen": 1.6117709875106812,
      "log_odds_ratio": -0.20646710693836212,
      "logits/chosen": -0.5787883996963501,
      "logits/rejected": -1.2313177585601807,
      "logps/chosen": -1.492388129234314,
      "logps/rejected": -2.906677722930908,
      "loss": 1.5183,
      "nll_loss": 1.4976226091384888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14923880994319916,
      "rewards/margins": 0.14142896234989166,
      "rewards/rejected": -0.2906677722930908,
      "step": 987
    },
    {
      "epoch": 1.5563435776201735,
      "grad_norm": 0.23063760995864868,
      "learning_rate": 5.751375861118666e-07,
      "log_odds_chosen": 1.6522552967071533,
      "log_odds_ratio": -0.2178073227405548,
      "logits/chosen": -0.5732280611991882,
      "logits/rejected": -1.152604341506958,
      "logps/chosen": -1.506281852722168,
      "logps/rejected": -2.9734768867492676,
      "loss": 1.5207,
      "nll_loss": 1.4989233016967773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15062817931175232,
      "rewards/margins": 0.14671948552131653,
      "rewards/rejected": -0.29734766483306885,
      "step": 988
    },
    {
      "epoch": 1.557919621749409,
      "grad_norm": 0.23059920966625214,
      "learning_rate": 5.712854273255707e-07,
      "log_odds_chosen": 1.3739960193634033,
      "log_odds_ratio": -0.23299655318260193,
      "logits/chosen": -0.46076440811157227,
      "logits/rejected": -1.207507848739624,
      "logps/chosen": -1.4784526824951172,
      "logps/rejected": -2.6683952808380127,
      "loss": 1.5251,
      "nll_loss": 1.5017659664154053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14784526824951172,
      "rewards/margins": 0.11899429559707642,
      "rewards/rejected": -0.26683956384658813,
      "step": 989
    },
    {
      "epoch": 1.5594956658786447,
      "grad_norm": 0.2195482701063156,
      "learning_rate": 5.674440617954659e-07,
      "log_odds_chosen": 1.4465641975402832,
      "log_odds_ratio": -0.22163105010986328,
      "logits/chosen": -0.6010885834693909,
      "logits/rejected": -1.0619293451309204,
      "logps/chosen": -1.3853051662445068,
      "logps/rejected": -2.6200191974639893,
      "loss": 1.4148,
      "nll_loss": 1.3925957679748535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13853052258491516,
      "rewards/margins": 0.12347142398357391,
      "rewards/rejected": -0.2620019316673279,
      "step": 990
    },
    {
      "epoch": 1.5610717100078801,
      "grad_norm": 0.2296878546476364,
      "learning_rate": 5.63613518541234e-07,
      "log_odds_chosen": 1.358451247215271,
      "log_odds_ratio": -0.2611231207847595,
      "logits/chosen": -0.49619537591934204,
      "logits/rejected": -1.0923731327056885,
      "logps/chosen": -1.4310359954833984,
      "logps/rejected": -2.604966878890991,
      "loss": 1.481,
      "nll_loss": 1.4548733234405518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14310359954833984,
      "rewards/margins": 0.11739308387041092,
      "rewards/rejected": -0.26049667596817017,
      "step": 991
    },
    {
      "epoch": 1.562647754137116,
      "grad_norm": 0.24957799911499023,
      "learning_rate": 5.597938265007993e-07,
      "log_odds_chosen": 1.5398213863372803,
      "log_odds_ratio": -0.19846056401729584,
      "logits/chosen": -0.5488215088844299,
      "logits/rejected": -1.2726433277130127,
      "logps/chosen": -1.4298733472824097,
      "logps/rejected": -2.7580158710479736,
      "loss": 1.459,
      "nll_loss": 1.4391186237335205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14298734068870544,
      "rewards/margins": 0.13281424343585968,
      "rewards/rejected": -0.2758015990257263,
      "step": 992
    },
    {
      "epoch": 1.5642237982663514,
      "grad_norm": 0.25925877690315247,
      "learning_rate": 5.559850145301106e-07,
      "log_odds_chosen": 1.2498853206634521,
      "log_odds_ratio": -0.2625320255756378,
      "logits/chosen": -0.5191019773483276,
      "logits/rejected": -1.0686025619506836,
      "logps/chosen": -1.459303617477417,
      "logps/rejected": -2.530735969543457,
      "loss": 1.5038,
      "nll_loss": 1.4775936603546143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14593034982681274,
      "rewards/margins": 0.10714326053857803,
      "rewards/rejected": -0.2530736029148102,
      "step": 993
    },
    {
      "epoch": 1.5657998423955872,
      "grad_norm": 0.2253548502922058,
      "learning_rate": 5.521871114029233e-07,
      "log_odds_chosen": 1.485551118850708,
      "log_odds_ratio": -0.20778267085552216,
      "logits/chosen": -0.589963436126709,
      "logits/rejected": -1.221030831336975,
      "logps/chosen": -1.3782234191894531,
      "logps/rejected": -2.6475584506988525,
      "loss": 1.4313,
      "nll_loss": 1.410495400428772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13782235980033875,
      "rewards/margins": 0.1269334852695465,
      "rewards/rejected": -0.26475581526756287,
      "step": 994
    },
    {
      "epoch": 1.5673758865248226,
      "grad_norm": 0.26374179124832153,
      "learning_rate": 5.484001458105823e-07,
      "log_odds_chosen": 1.3245313167572021,
      "log_odds_ratio": -0.2562502324581146,
      "logits/chosen": -0.6059260368347168,
      "logits/rejected": -0.9131497740745544,
      "logps/chosen": -1.4769196510314941,
      "logps/rejected": -2.621497392654419,
      "loss": 1.5087,
      "nll_loss": 1.4831230640411377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476919800043106,
      "rewards/margins": 0.11445777118206024,
      "rewards/rejected": -0.26214978098869324,
      "step": 995
    },
    {
      "epoch": 1.5689519306540585,
      "grad_norm": 0.2326793670654297,
      "learning_rate": 5.446241463618027e-07,
      "log_odds_chosen": 1.7417064905166626,
      "log_odds_ratio": -0.21662843227386475,
      "logits/chosen": -0.5617032647132874,
      "logits/rejected": -1.110959768295288,
      "logps/chosen": -1.4931955337524414,
      "logps/rejected": -3.043185234069824,
      "loss": 1.5233,
      "nll_loss": 1.501662015914917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14931955933570862,
      "rewards/margins": 0.15499897301197052,
      "rewards/rejected": -0.30431854724884033,
      "step": 996
    },
    {
      "epoch": 1.5705279747832939,
      "grad_norm": 0.23864975571632385,
      "learning_rate": 5.40859141582457e-07,
      "log_odds_chosen": 1.6404131650924683,
      "log_odds_ratio": -0.1807386428117752,
      "logits/chosen": -0.6005215644836426,
      "logits/rejected": -1.1642825603485107,
      "logps/chosen": -1.3507462739944458,
      "logps/rejected": -2.75468111038208,
      "loss": 1.3827,
      "nll_loss": 1.3645787239074707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.135074645280838,
      "rewards/margins": 0.14039349555969238,
      "rewards/rejected": -0.2754681408405304,
      "step": 997
    },
    {
      "epoch": 1.5721040189125297,
      "grad_norm": 0.26277869939804077,
      "learning_rate": 5.371051599153582e-07,
      "log_odds_chosen": 1.1333339214324951,
      "log_odds_ratio": -0.30302396416664124,
      "logits/chosen": -0.48770660161972046,
      "logits/rejected": -0.9194856882095337,
      "logps/chosen": -1.422499179840088,
      "logps/rejected": -2.377899169921875,
      "loss": 1.4677,
      "nll_loss": 1.4373992681503296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1422499269247055,
      "rewards/margins": 0.09553999453783035,
      "rewards/rejected": -0.23778992891311646,
      "step": 998
    },
    {
      "epoch": 1.573680063041765,
      "grad_norm": 0.24202293157577515,
      "learning_rate": 5.333622297200449e-07,
      "log_odds_chosen": 1.324034333229065,
      "log_odds_ratio": -0.23905494809150696,
      "logits/chosen": -0.5503994226455688,
      "logits/rejected": -1.0024315118789673,
      "logps/chosen": -1.4111573696136475,
      "logps/rejected": -2.533215284347534,
      "loss": 1.4642,
      "nll_loss": 1.4402716159820557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411157250404358,
      "rewards/margins": 0.11220580339431763,
      "rewards/rejected": -0.2533215284347534,
      "step": 999
    },
    {
      "epoch": 1.575256107171001,
      "grad_norm": 0.2206570953130722,
      "learning_rate": 5.296303792725676e-07,
      "log_odds_chosen": 1.6299400329589844,
      "log_odds_ratio": -0.18306228518486023,
      "logits/chosen": -0.600836455821991,
      "logits/rejected": -1.307792067527771,
      "logps/chosen": -1.5206996202468872,
      "logps/rejected": -2.9547481536865234,
      "loss": 1.5392,
      "nll_loss": 1.5208828449249268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15206997096538544,
      "rewards/margins": 0.14340484142303467,
      "rewards/rejected": -0.2954748272895813,
      "step": 1000
    },
    {
      "epoch": 1.5768321513002364,
      "grad_norm": 0.2330697923898697,
      "learning_rate": 5.259096367652725e-07,
      "log_odds_chosen": 1.5772244930267334,
      "log_odds_ratio": -0.20086143910884857,
      "logits/chosen": -0.5763773918151855,
      "logits/rejected": -1.18201744556427,
      "logps/chosen": -1.4716806411743164,
      "logps/rejected": -2.849454164505005,
      "loss": 1.5044,
      "nll_loss": 1.484352946281433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1471680998802185,
      "rewards/margins": 0.13777735829353333,
      "rewards/rejected": -0.28494542837142944,
      "step": 1001
    },
    {
      "epoch": 1.578408195429472,
      "grad_norm": 0.22556854784488678,
      "learning_rate": 5.222000303065927e-07,
      "log_odds_chosen": 1.5036588907241821,
      "log_odds_ratio": -0.2185424119234085,
      "logits/chosen": -0.5098148584365845,
      "logits/rejected": -1.3263771533966064,
      "logps/chosen": -1.4602028131484985,
      "logps/rejected": -2.769603967666626,
      "loss": 1.4951,
      "nll_loss": 1.4732788801193237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14602027833461761,
      "rewards/margins": 0.13094010949134827,
      "rewards/rejected": -0.27696043252944946,
      "step": 1002
    },
    {
      "epoch": 1.5799842395587076,
      "grad_norm": 0.2382277250289917,
      "learning_rate": 5.185015879208335e-07,
      "log_odds_chosen": 1.4302736520767212,
      "log_odds_ratio": -0.21838317811489105,
      "logits/chosen": -0.571363627910614,
      "logits/rejected": -1.1598929166793823,
      "logps/chosen": -1.4546284675598145,
      "logps/rejected": -2.683072805404663,
      "loss": 1.4943,
      "nll_loss": 1.4725018739700317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14546287059783936,
      "rewards/margins": 0.12284442782402039,
      "rewards/rejected": -0.26830726861953735,
      "step": 1003
    },
    {
      "epoch": 1.5815602836879432,
      "grad_norm": 0.2373654991388321,
      "learning_rate": 5.148143375479601e-07,
      "log_odds_chosen": 1.470292568206787,
      "log_odds_ratio": -0.20819450914859772,
      "logits/chosen": -0.5738686323165894,
      "logits/rejected": -1.1935853958129883,
      "logps/chosen": -1.514944076538086,
      "logps/rejected": -2.7983388900756836,
      "loss": 1.5301,
      "nll_loss": 1.5092508792877197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15149442851543427,
      "rewards/margins": 0.1283394694328308,
      "rewards/rejected": -0.2798338830471039,
      "step": 1004
    },
    {
      "epoch": 1.5831363278171788,
      "grad_norm": 0.23390109837055206,
      "learning_rate": 5.111383070433887e-07,
      "log_odds_chosen": 1.7050981521606445,
      "log_odds_ratio": -0.1848384588956833,
      "logits/chosen": -0.5373984575271606,
      "logits/rejected": -1.1884740591049194,
      "logps/chosen": -1.5155117511749268,
      "logps/rejected": -3.0270256996154785,
      "loss": 1.5378,
      "nll_loss": 1.5193629264831543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15155118703842163,
      "rewards/margins": 0.1511513739824295,
      "rewards/rejected": -0.30270254611968994,
      "step": 1005
    },
    {
      "epoch": 1.5847123719464145,
      "grad_norm": 0.25701385736465454,
      "learning_rate": 5.074735241777733e-07,
      "log_odds_chosen": 1.159680724143982,
      "log_odds_ratio": -0.2847953140735626,
      "logits/chosen": -0.4990413188934326,
      "logits/rejected": -1.0648868083953857,
      "logps/chosen": -1.6135644912719727,
      "logps/rejected": -2.6295039653778076,
      "loss": 1.6195,
      "nll_loss": 1.5910569429397583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16135644912719727,
      "rewards/margins": 0.10159394890069962,
      "rewards/rejected": -0.2629503905773163,
      "step": 1006
    },
    {
      "epoch": 1.58628841607565,
      "grad_norm": 0.2401193529367447,
      "learning_rate": 5.038200166367978e-07,
      "log_odds_chosen": 1.5429149866104126,
      "log_odds_ratio": -0.2054755836725235,
      "logits/chosen": -0.6429669260978699,
      "logits/rejected": -1.1563552618026733,
      "logps/chosen": -1.4454388618469238,
      "logps/rejected": -2.7840726375579834,
      "loss": 1.4797,
      "nll_loss": 1.4591237306594849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14454390108585358,
      "rewards/margins": 0.13386335968971252,
      "rewards/rejected": -0.2784072756767273,
      "step": 1007
    },
    {
      "epoch": 1.5878644602048857,
      "grad_norm": 0.22525815665721893,
      "learning_rate": 5.00177812020967e-07,
      "log_odds_chosen": 1.4066658020019531,
      "log_odds_ratio": -0.22515088319778442,
      "logits/chosen": -0.559375524520874,
      "logits/rejected": -1.1886323690414429,
      "logps/chosen": -1.5336283445358276,
      "logps/rejected": -2.7634639739990234,
      "loss": 1.5715,
      "nll_loss": 1.548957347869873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15336282551288605,
      "rewards/margins": 0.12298360466957092,
      "rewards/rejected": -0.2763464152812958,
      "step": 1008
    },
    {
      "epoch": 1.5894405043341213,
      "grad_norm": 0.2581009268760681,
      "learning_rate": 4.965469378453979e-07,
      "log_odds_chosen": 1.3718085289001465,
      "log_odds_ratio": -0.26245445013046265,
      "logits/chosen": -0.6085140705108643,
      "logits/rejected": -0.9518263936042786,
      "logps/chosen": -1.450960397720337,
      "logps/rejected": -2.632451295852661,
      "loss": 1.4906,
      "nll_loss": 1.4643597602844238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1450960338115692,
      "rewards/margins": 0.11814911663532257,
      "rewards/rejected": -0.263245165348053,
      "step": 1009
    },
    {
      "epoch": 1.591016548463357,
      "grad_norm": 0.24249351024627686,
      "learning_rate": 4.929274215396091e-07,
      "log_odds_chosen": 1.4204492568969727,
      "log_odds_ratio": -0.22473208606243134,
      "logits/chosen": -0.6509701013565063,
      "logits/rejected": -1.1477832794189453,
      "logps/chosen": -1.4949126243591309,
      "logps/rejected": -2.7247629165649414,
      "loss": 1.5182,
      "nll_loss": 1.495713233947754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14949126541614532,
      "rewards/margins": 0.12298502027988434,
      "rewards/rejected": -0.27247628569602966,
      "step": 1010
    },
    {
      "epoch": 1.5925925925925926,
      "grad_norm": 0.24071195721626282,
      "learning_rate": 4.893192904473182e-07,
      "log_odds_chosen": 1.0365221500396729,
      "log_odds_ratio": -0.320484459400177,
      "logits/chosen": -0.6212356090545654,
      "logits/rejected": -0.8027983903884888,
      "logps/chosen": -1.5227209329605103,
      "logps/rejected": -2.4116804599761963,
      "loss": 1.5671,
      "nll_loss": 1.535049319267273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1522720903158188,
      "rewards/margins": 0.08889594674110413,
      "rewards/rejected": -0.2411680370569229,
      "step": 1011
    },
    {
      "epoch": 1.5941686367218282,
      "grad_norm": 0.22898022830486298,
      "learning_rate": 4.85722571826233e-07,
      "log_odds_chosen": 1.376934289932251,
      "log_odds_ratio": -0.2448529750108719,
      "logits/chosen": -0.6195841431617737,
      "logits/rejected": -1.1483511924743652,
      "logps/chosen": -1.4243050813674927,
      "logps/rejected": -2.6101884841918945,
      "loss": 1.4646,
      "nll_loss": 1.4400908946990967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14243051409721375,
      "rewards/margins": 0.11858832836151123,
      "rewards/rejected": -0.26101887226104736,
      "step": 1012
    },
    {
      "epoch": 1.5957446808510638,
      "grad_norm": 0.2573704123497009,
      "learning_rate": 4.821372928478433e-07,
      "log_odds_chosen": 1.2995766401290894,
      "log_odds_ratio": -0.2527857720851898,
      "logits/chosen": -0.5270690321922302,
      "logits/rejected": -1.0693113803863525,
      "logps/chosen": -1.5362205505371094,
      "logps/rejected": -2.663954973220825,
      "loss": 1.565,
      "nll_loss": 1.5397347211837769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15362204611301422,
      "rewards/margins": 0.11277343332767487,
      "rewards/rejected": -0.2663954794406891,
      "step": 1013
    },
    {
      "epoch": 1.5973207249802994,
      "grad_norm": 0.2260870486497879,
      "learning_rate": 4.785634805972201e-07,
      "log_odds_chosen": 1.2423927783966064,
      "log_odds_ratio": -0.2707635760307312,
      "logits/chosen": -0.5511917471885681,
      "logits/rejected": -1.0825436115264893,
      "logps/chosen": -1.4702643156051636,
      "logps/rejected": -2.536020517349243,
      "loss": 1.5061,
      "nll_loss": 1.4790399074554443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470264345407486,
      "rewards/margins": 0.1065756231546402,
      "rewards/rejected": -0.2536020576953888,
      "step": 1014
    },
    {
      "epoch": 1.598896769109535,
      "grad_norm": 0.2386903464794159,
      "learning_rate": 4.750011620728085e-07,
      "log_odds_chosen": 1.4323338270187378,
      "log_odds_ratio": -0.22130835056304932,
      "logits/chosen": -0.6016347408294678,
      "logits/rejected": -1.1095510721206665,
      "logps/chosen": -1.4916538000106812,
      "logps/rejected": -2.7366549968719482,
      "loss": 1.5194,
      "nll_loss": 1.497276782989502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14916540682315826,
      "rewards/margins": 0.12450012564659119,
      "rewards/rejected": -0.27366554737091064,
      "step": 1015
    },
    {
      "epoch": 1.6004728132387707,
      "grad_norm": 0.2451801300048828,
      "learning_rate": 4.714503641862224e-07,
      "log_odds_chosen": 1.3950867652893066,
      "log_odds_ratio": -0.2260989397764206,
      "logits/chosen": -0.546274721622467,
      "logits/rejected": -1.1583294868469238,
      "logps/chosen": -1.525007724761963,
      "logps/rejected": -2.737661600112915,
      "loss": 1.5497,
      "nll_loss": 1.5270910263061523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15250077843666077,
      "rewards/margins": 0.12126541882753372,
      "rewards/rejected": -0.2737661898136139,
      "step": 1016
    },
    {
      "epoch": 1.6020488573680063,
      "grad_norm": 0.25802090764045715,
      "learning_rate": 4.679111137620442e-07,
      "log_odds_chosen": 1.7816516160964966,
      "log_odds_ratio": -0.20580695569515228,
      "logits/chosen": -0.6786735653877258,
      "logits/rejected": -1.0724308490753174,
      "logps/chosen": -1.4887828826904297,
      "logps/rejected": -3.0731201171875,
      "loss": 1.5012,
      "nll_loss": 1.4806009531021118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.148878276348114,
      "rewards/margins": 0.158433735370636,
      "rewards/rejected": -0.30731201171875,
      "step": 1017
    },
    {
      "epoch": 1.603624901497242,
      "grad_norm": 0.2458200603723526,
      "learning_rate": 4.6438343753762055e-07,
      "log_odds_chosen": 1.288680911064148,
      "log_odds_ratio": -0.2676832377910614,
      "logits/chosen": -0.5883816480636597,
      "logits/rejected": -1.0468894243240356,
      "logps/chosen": -1.4738123416900635,
      "logps/rejected": -2.581355094909668,
      "loss": 1.4887,
      "nll_loss": 1.4619470834732056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1473812460899353,
      "rewards/margins": 0.11075427383184433,
      "rewards/rejected": -0.25813552737236023,
      "step": 1018
    },
    {
      "epoch": 1.6052009456264775,
      "grad_norm": 0.23983252048492432,
      "learning_rate": 4.608673621628609e-07,
      "log_odds_chosen": 1.4389286041259766,
      "log_odds_ratio": -0.21935194730758667,
      "logits/chosen": -0.5431928634643555,
      "logits/rejected": -1.1362862586975098,
      "logps/chosen": -1.4972708225250244,
      "logps/rejected": -2.747602939605713,
      "loss": 1.5121,
      "nll_loss": 1.4901388883590698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14972709119319916,
      "rewards/margins": 0.12503322958946228,
      "rewards/rejected": -0.27476030588150024,
      "step": 1019
    },
    {
      "epoch": 1.6067769897557131,
      "grad_norm": 0.24781209230422974,
      "learning_rate": 4.5736291420003413e-07,
      "log_odds_chosen": 1.4313749074935913,
      "log_odds_ratio": -0.22888794541358948,
      "logits/chosen": -0.5261708498001099,
      "logits/rejected": -1.0032241344451904,
      "logps/chosen": -1.4842643737792969,
      "logps/rejected": -2.7309203147888184,
      "loss": 1.5185,
      "nll_loss": 1.495652675628662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14842644333839417,
      "rewards/margins": 0.12466560304164886,
      "rewards/rejected": -0.2730920612812042,
      "step": 1020
    },
    {
      "epoch": 1.6083530338849488,
      "grad_norm": 0.25586873292922974,
      "learning_rate": 4.538701201235713e-07,
      "log_odds_chosen": 1.5266203880310059,
      "log_odds_ratio": -0.21408236026763916,
      "logits/chosen": -0.503991961479187,
      "logits/rejected": -1.1439121961593628,
      "logps/chosen": -1.4394030570983887,
      "logps/rejected": -2.7683298587799072,
      "loss": 1.472,
      "nll_loss": 1.4505736827850342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14394031465053558,
      "rewards/margins": 0.1328926980495453,
      "rewards/rejected": -0.27683302760124207,
      "step": 1021
    },
    {
      "epoch": 1.6099290780141844,
      "grad_norm": 0.2292323261499405,
      "learning_rate": 4.5038900631986367e-07,
      "log_odds_chosen": 1.3885390758514404,
      "log_odds_ratio": -0.23708733916282654,
      "logits/chosen": -0.5118030309677124,
      "logits/rejected": -1.0901174545288086,
      "logps/chosen": -1.4164763689041138,
      "logps/rejected": -2.6054136753082275,
      "loss": 1.4725,
      "nll_loss": 1.4488348960876465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14164763689041138,
      "rewards/margins": 0.11889374256134033,
      "rewards/rejected": -0.2605413794517517,
      "step": 1022
    },
    {
      "epoch": 1.61150512214342,
      "grad_norm": 0.24720241129398346,
      "learning_rate": 4.4691959908706287e-07,
      "log_odds_chosen": 1.2601948976516724,
      "log_odds_ratio": -0.274635374546051,
      "logits/chosen": -0.5365511775016785,
      "logits/rejected": -0.9877060055732727,
      "logps/chosen": -1.5388953685760498,
      "logps/rejected": -2.641653537750244,
      "loss": 1.5609,
      "nll_loss": 1.5334821939468384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15388953685760498,
      "rewards/margins": 0.11027583479881287,
      "rewards/rejected": -0.26416537165641785,
      "step": 1023
    },
    {
      "epoch": 1.6130811662726556,
      "grad_norm": 0.2170920968055725,
      "learning_rate": 4.434619246348843e-07,
      "log_odds_chosen": 1.446617603302002,
      "log_odds_ratio": -0.2382478415966034,
      "logits/chosen": -0.52433180809021,
      "logits/rejected": -1.2474020719528198,
      "logps/chosen": -1.4292290210723877,
      "logps/rejected": -2.6819615364074707,
      "loss": 1.4734,
      "nll_loss": 1.4495912790298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14292290806770325,
      "rewards/margins": 0.12527324259281158,
      "rewards/rejected": -0.268196165561676,
      "step": 1024
    },
    {
      "epoch": 1.6146572104018913,
      "grad_norm": 0.26740148663520813,
      "learning_rate": 4.4001600908440564e-07,
      "log_odds_chosen": 1.4140287637710571,
      "log_odds_ratio": -0.2243608981370926,
      "logits/chosen": -0.46684539318084717,
      "logits/rejected": -1.0661324262619019,
      "logps/chosen": -1.5254026651382446,
      "logps/rejected": -2.7596540451049805,
      "loss": 1.5449,
      "nll_loss": 1.5224339962005615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15254026651382446,
      "rewards/margins": 0.12342514097690582,
      "rewards/rejected": -0.2759653925895691,
      "step": 1025
    },
    {
      "epoch": 1.6162332545311269,
      "grad_norm": 0.2533715069293976,
      "learning_rate": 4.365818784678736e-07,
      "log_odds_chosen": 1.421607255935669,
      "log_odds_ratio": -0.21872574090957642,
      "logits/chosen": -0.5264071226119995,
      "logits/rejected": -1.1232389211654663,
      "logps/chosen": -1.370159387588501,
      "logps/rejected": -2.5749623775482178,
      "loss": 1.423,
      "nll_loss": 1.401151418685913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1370159387588501,
      "rewards/margins": 0.12048029154539108,
      "rewards/rejected": -0.2574962377548218,
      "step": 1026
    },
    {
      "epoch": 1.6178092986603625,
      "grad_norm": 0.2234547734260559,
      "learning_rate": 4.3315955872850464e-07,
      "log_odds_chosen": 1.4137414693832397,
      "log_odds_ratio": -0.2406781166791916,
      "logits/chosen": -0.6054703593254089,
      "logits/rejected": -1.101162075996399,
      "logps/chosen": -1.4922658205032349,
      "logps/rejected": -2.7233831882476807,
      "loss": 1.5229,
      "nll_loss": 1.4988764524459839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1492265909910202,
      "rewards/margins": 0.12311175465583801,
      "rewards/rejected": -0.272338330745697,
      "step": 1027
    },
    {
      "epoch": 1.6193853427895981,
      "grad_norm": 0.23411206901073456,
      "learning_rate": 4.2974907572029016e-07,
      "log_odds_chosen": 1.3719000816345215,
      "log_odds_ratio": -0.24718119204044342,
      "logits/chosen": -0.5696765780448914,
      "logits/rejected": -1.2292473316192627,
      "logps/chosen": -1.5276460647583008,
      "logps/rejected": -2.725832462310791,
      "loss": 1.5444,
      "nll_loss": 1.5196757316589355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15276460349559784,
      "rewards/margins": 0.11981862783432007,
      "rewards/rejected": -0.2725832462310791,
      "step": 1028
    },
    {
      "epoch": 1.6209613869188337,
      "grad_norm": 0.25170716643333435,
      "learning_rate": 4.2635045520780033e-07,
      "log_odds_chosen": 1.4888306856155396,
      "log_odds_ratio": -0.2285872846841812,
      "logits/chosen": -0.5751727819442749,
      "logits/rejected": -1.1347098350524902,
      "logps/chosen": -1.481864333152771,
      "logps/rejected": -2.784130096435547,
      "loss": 1.5162,
      "nll_loss": 1.493357539176941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14818644523620605,
      "rewards/margins": 0.13022656738758087,
      "rewards/rejected": -0.27841299772262573,
      "step": 1029
    },
    {
      "epoch": 1.6225374310480694,
      "grad_norm": 0.25657132267951965,
      "learning_rate": 4.2296372286598904e-07,
      "log_odds_chosen": 1.7655705213546753,
      "log_odds_ratio": -0.17192082107067108,
      "logits/chosen": -0.5926560163497925,
      "logits/rejected": -1.2589290142059326,
      "logps/chosen": -1.4576746225357056,
      "logps/rejected": -3.0116119384765625,
      "loss": 1.4727,
      "nll_loss": 1.4554781913757324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1457674503326416,
      "rewards/margins": 0.15539371967315674,
      "rewards/rejected": -0.3011612296104431,
      "step": 1030
    },
    {
      "epoch": 1.624113475177305,
      "grad_norm": 0.29955559968948364,
      "learning_rate": 4.1958890428000205e-07,
      "log_odds_chosen": 1.1283953189849854,
      "log_odds_ratio": -0.29256242513656616,
      "logits/chosen": -0.5805360674858093,
      "logits/rejected": -1.1152665615081787,
      "logps/chosen": -1.4892380237579346,
      "logps/rejected": -2.4578397274017334,
      "loss": 1.5301,
      "nll_loss": 1.5008854866027832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14892378449440002,
      "rewards/margins": 0.09686017781496048,
      "rewards/rejected": -0.2457839846611023,
      "step": 1031
    },
    {
      "epoch": 1.6256895193065406,
      "grad_norm": 0.24420811235904694,
      "learning_rate": 4.162260249449816e-07,
      "log_odds_chosen": 1.3352773189544678,
      "log_odds_ratio": -0.24342773854732513,
      "logits/chosen": -0.5465145111083984,
      "logits/rejected": -1.0243265628814697,
      "logps/chosen": -1.415776014328003,
      "logps/rejected": -2.5524866580963135,
      "loss": 1.4637,
      "nll_loss": 1.439316987991333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1415776014328003,
      "rewards/margins": 0.11367104947566986,
      "rewards/rejected": -0.25524866580963135,
      "step": 1032
    },
    {
      "epoch": 1.6272655634357762,
      "grad_norm": 0.2555002272129059,
      "learning_rate": 4.128751102658752e-07,
      "log_odds_chosen": 1.4492204189300537,
      "log_odds_ratio": -0.23177595436573029,
      "logits/chosen": -0.555109441280365,
      "logits/rejected": -1.065739393234253,
      "logps/chosen": -1.4694397449493408,
      "logps/rejected": -2.730774402618408,
      "loss": 1.4906,
      "nll_loss": 1.4674389362335205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14694397151470184,
      "rewards/margins": 0.12613347172737122,
      "rewards/rejected": -0.27307742834091187,
      "step": 1033
    },
    {
      "epoch": 1.6288416075650118,
      "grad_norm": 0.22693617641925812,
      "learning_rate": 4.09536185557243e-07,
      "log_odds_chosen": 1.7489532232284546,
      "log_odds_ratio": -0.18098562955856323,
      "logits/chosen": -0.4922034740447998,
      "logits/rejected": -1.234031319618225,
      "logps/chosen": -1.53195059299469,
      "logps/rejected": -3.089261770248413,
      "loss": 1.545,
      "nll_loss": 1.5268791913986206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1531950682401657,
      "rewards/margins": 0.15573115646839142,
      "rewards/rejected": -0.30892622470855713,
      "step": 1034
    },
    {
      "epoch": 1.6304176516942475,
      "grad_norm": 0.2317894995212555,
      "learning_rate": 4.0620927604306533e-07,
      "log_odds_chosen": 1.4553436040878296,
      "log_odds_ratio": -0.2158581018447876,
      "logits/chosen": -0.5696369409561157,
      "logits/rejected": -1.2792422771453857,
      "logps/chosen": -1.4979912042617798,
      "logps/rejected": -2.7616498470306396,
      "loss": 1.5155,
      "nll_loss": 1.4939019680023193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1497991383075714,
      "rewards/margins": 0.12636587023735046,
      "rewards/rejected": -0.2761650085449219,
      "step": 1035
    },
    {
      "epoch": 1.631993695823483,
      "grad_norm": 0.2383565902709961,
      "learning_rate": 4.0289440685655516e-07,
      "log_odds_chosen": 1.3822853565216064,
      "log_odds_ratio": -0.23834577202796936,
      "logits/chosen": -0.5993545651435852,
      "logits/rejected": -1.2289386987686157,
      "logps/chosen": -1.5039522647857666,
      "logps/rejected": -2.706685781478882,
      "loss": 1.5173,
      "nll_loss": 1.493487000465393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503952294588089,
      "rewards/margins": 0.1202733963727951,
      "rewards/rejected": -0.270668625831604,
      "step": 1036
    },
    {
      "epoch": 1.6335697399527187,
      "grad_norm": 0.29290854930877686,
      "learning_rate": 3.9959160303996575e-07,
      "log_odds_chosen": 1.4371747970581055,
      "log_odds_ratio": -0.25804412364959717,
      "logits/chosen": -0.4866321384906769,
      "logits/rejected": -0.9293813109397888,
      "logps/chosen": -1.4826276302337646,
      "logps/rejected": -2.7350714206695557,
      "loss": 1.5152,
      "nll_loss": 1.489388346672058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14826278388500214,
      "rewards/margins": 0.1252443641424179,
      "rewards/rejected": -0.27350714802742004,
      "step": 1037
    },
    {
      "epoch": 1.6351457840819543,
      "grad_norm": 0.2448466569185257,
      "learning_rate": 3.9630088954440244e-07,
      "log_odds_chosen": 1.4421818256378174,
      "log_odds_ratio": -0.22921352088451385,
      "logits/chosen": -0.523463785648346,
      "logits/rejected": -1.1926350593566895,
      "logps/chosen": -1.517987608909607,
      "logps/rejected": -2.7820348739624023,
      "loss": 1.5507,
      "nll_loss": 1.52776300907135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517987698316574,
      "rewards/margins": 0.12640470266342163,
      "rewards/rejected": -0.27820348739624023,
      "step": 1038
    },
    {
      "epoch": 1.63672182821119,
      "grad_norm": 0.2324703484773636,
      "learning_rate": 3.930222912296346e-07,
      "log_odds_chosen": 1.7867870330810547,
      "log_odds_ratio": -0.1732819378376007,
      "logits/chosen": -0.5439863801002502,
      "logits/rejected": -1.1891701221466064,
      "logps/chosen": -1.4344892501831055,
      "logps/rejected": -3.007711887359619,
      "loss": 1.4634,
      "nll_loss": 1.4460222721099854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14344890415668488,
      "rewards/margins": 0.1573222577571869,
      "rewards/rejected": -0.30077117681503296,
      "step": 1039
    },
    {
      "epoch": 1.6382978723404256,
      "grad_norm": 0.23288051784038544,
      "learning_rate": 3.8975583286390546e-07,
      "log_odds_chosen": 1.2234938144683838,
      "log_odds_ratio": -0.26092731952667236,
      "logits/chosen": -0.5586830377578735,
      "logits/rejected": -1.1848300695419312,
      "logps/chosen": -1.5218865871429443,
      "logps/rejected": -2.5783262252807617,
      "loss": 1.543,
      "nll_loss": 1.516922116279602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15218865871429443,
      "rewards/margins": 0.10564397275447845,
      "rewards/rejected": -0.2578326165676117,
      "step": 1040
    },
    {
      "epoch": 1.6398739164696612,
      "grad_norm": 0.23314298689365387,
      "learning_rate": 3.8650153912374807e-07,
      "log_odds_chosen": 1.7205265760421753,
      "log_odds_ratio": -0.1773129403591156,
      "logits/chosen": -0.5474420785903931,
      "logits/rejected": -1.1834218502044678,
      "logps/chosen": -1.4960788488388062,
      "logps/rejected": -3.010667324066162,
      "loss": 1.5096,
      "nll_loss": 1.4918667078018188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14960788190364838,
      "rewards/margins": 0.15145882964134216,
      "rewards/rejected": -0.30106672644615173,
      "step": 1041
    },
    {
      "epoch": 1.6414499605988968,
      "grad_norm": 0.2824546694755554,
      "learning_rate": 3.832594345937974e-07,
      "log_odds_chosen": 1.3036589622497559,
      "log_odds_ratio": -0.2520041763782501,
      "logits/chosen": -0.6231052875518799,
      "logits/rejected": -0.9753563404083252,
      "logps/chosen": -1.4195770025253296,
      "logps/rejected": -2.5295310020446777,
      "loss": 1.4742,
      "nll_loss": 1.4490244388580322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14195768535137177,
      "rewards/margins": 0.11099538952112198,
      "rewards/rejected": -0.25295308232307434,
      "step": 1042
    },
    {
      "epoch": 1.6430260047281324,
      "grad_norm": 0.253890722990036,
      "learning_rate": 3.8002954376660524e-07,
      "log_odds_chosen": 1.3481009006500244,
      "log_odds_ratio": -0.2336786836385727,
      "logits/chosen": -0.5074504017829895,
      "logits/rejected": -1.2498489618301392,
      "logps/chosen": -1.5538502931594849,
      "logps/rejected": -2.7319226264953613,
      "loss": 1.5758,
      "nll_loss": 1.552431583404541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15538501739501953,
      "rewards/margins": 0.11780723184347153,
      "rewards/rejected": -0.27319225668907166,
      "step": 1043
    },
    {
      "epoch": 1.644602048857368,
      "grad_norm": 0.240655779838562,
      "learning_rate": 3.7681189104245316e-07,
      "log_odds_chosen": 1.4556872844696045,
      "log_odds_ratio": -0.21800783276557922,
      "logits/chosen": -0.6808980703353882,
      "logits/rejected": -1.1310241222381592,
      "logps/chosen": -1.377492070198059,
      "logps/rejected": -2.6163370609283447,
      "loss": 1.4235,
      "nll_loss": 1.4016854763031006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13774922490119934,
      "rewards/margins": 0.12388448417186737,
      "rewards/rejected": -0.2616336941719055,
      "step": 1044
    },
    {
      "epoch": 1.6461780929866037,
      "grad_norm": 0.2670923173427582,
      "learning_rate": 3.7360650072917067e-07,
      "log_odds_chosen": 1.4392362833023071,
      "log_odds_ratio": -0.23462611436843872,
      "logits/chosen": -0.42758339643478394,
      "logits/rejected": -0.995101809501648,
      "logps/chosen": -1.4439878463745117,
      "logps/rejected": -2.68536114692688,
      "loss": 1.4683,
      "nll_loss": 1.4448206424713135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14439880847930908,
      "rewards/margins": 0.12413733452558517,
      "rewards/rejected": -0.26853615045547485,
      "step": 1045
    },
    {
      "epoch": 1.6477541371158393,
      "grad_norm": 0.2281276285648346,
      "learning_rate": 3.704133970419514e-07,
      "log_odds_chosen": 1.361617922782898,
      "log_odds_ratio": -0.24288137257099152,
      "logits/chosen": -0.6320463418960571,
      "logits/rejected": -1.1901187896728516,
      "logps/chosen": -1.4430246353149414,
      "logps/rejected": -2.6145782470703125,
      "loss": 1.4902,
      "nll_loss": 1.4658656120300293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14430245757102966,
      "rewards/margins": 0.11715538799762726,
      "rewards/rejected": -0.2614578604698181,
      "step": 1046
    },
    {
      "epoch": 1.649330181245075,
      "grad_norm": 0.2727247178554535,
      "learning_rate": 3.672326041031677e-07,
      "log_odds_chosen": 1.2072465419769287,
      "log_odds_ratio": -0.27017131447792053,
      "logits/chosen": -0.4498339295387268,
      "logits/rejected": -1.039910078048706,
      "logps/chosen": -1.4983584880828857,
      "logps/rejected": -2.5374128818511963,
      "loss": 1.5245,
      "nll_loss": 1.4974457025527954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14983585476875305,
      "rewards/margins": 0.10390545427799225,
      "rewards/rejected": -0.2537413239479065,
      "step": 1047
    },
    {
      "epoch": 1.6509062253743105,
      "grad_norm": 0.22254766523838043,
      "learning_rate": 3.6406414594219116e-07,
      "log_odds_chosen": 1.7156623601913452,
      "log_odds_ratio": -0.1777404397726059,
      "logits/chosen": -0.5613861680030823,
      "logits/rejected": -1.3160078525543213,
      "logps/chosen": -1.511025309562683,
      "logps/rejected": -3.0267491340637207,
      "loss": 1.526,
      "nll_loss": 1.508190631866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15110255777835846,
      "rewards/margins": 0.1515723466873169,
      "rewards/rejected": -0.30267491936683655,
      "step": 1048
    },
    {
      "epoch": 1.6524822695035462,
      "grad_norm": 0.233059361577034,
      "learning_rate": 3.609080464952103e-07,
      "log_odds_chosen": 1.569747805595398,
      "log_odds_ratio": -0.20914728939533234,
      "logits/chosen": -0.6616379022598267,
      "logits/rejected": -1.2277605533599854,
      "logps/chosen": -1.5039784908294678,
      "logps/rejected": -2.878020763397217,
      "loss": 1.5265,
      "nll_loss": 1.505600094795227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503978669643402,
      "rewards/margins": 0.13740423321723938,
      "rewards/rejected": -0.2878020703792572,
      "step": 1049
    },
    {
      "epoch": 1.6540583136327816,
      "grad_norm": 0.2632991671562195,
      "learning_rate": 3.577643296050483e-07,
      "log_odds_chosen": 1.7292767763137817,
      "log_odds_ratio": -0.20081210136413574,
      "logits/chosen": -0.5559372305870056,
      "logits/rejected": -1.2755073308944702,
      "logps/chosen": -1.6162062883377075,
      "logps/rejected": -3.1716952323913574,
      "loss": 1.6116,
      "nll_loss": 1.5915621519088745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16162064671516418,
      "rewards/margins": 0.15554890036582947,
      "rewards/rejected": -0.31716951727867126,
      "step": 1050
    },
    {
      "epoch": 1.6556343577620174,
      "grad_norm": 0.2421002984046936,
      "learning_rate": 3.5463301902098496e-07,
      "log_odds_chosen": 1.5112746953964233,
      "log_odds_ratio": -0.227679044008255,
      "logits/chosen": -0.5569344758987427,
      "logits/rejected": -1.0021055936813354,
      "logps/chosen": -1.3936378955841064,
      "logps/rejected": -2.706923723220825,
      "loss": 1.4316,
      "nll_loss": 1.4088218212127686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13936378061771393,
      "rewards/margins": 0.13132859766483307,
      "rewards/rejected": -0.270692378282547,
      "step": 1051
    },
    {
      "epoch": 1.6572104018912528,
      "grad_norm": 0.23249223828315735,
      "learning_rate": 3.5151413839857625e-07,
      "log_odds_chosen": 1.4170948266983032,
      "log_odds_ratio": -0.23029619455337524,
      "logits/chosen": -0.5116890072822571,
      "logits/rejected": -1.104848861694336,
      "logps/chosen": -1.5236592292785645,
      "logps/rejected": -2.7633934020996094,
      "loss": 1.541,
      "nll_loss": 1.51799738407135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15236592292785645,
      "rewards/margins": 0.12397340685129166,
      "rewards/rejected": -0.2763393223285675,
      "step": 1052
    },
    {
      "epoch": 1.6587864460204886,
      "grad_norm": 0.2527998387813568,
      "learning_rate": 3.484077112994759e-07,
      "log_odds_chosen": 1.6636877059936523,
      "log_odds_ratio": -0.18316401541233063,
      "logits/chosen": -0.6709176301956177,
      "logits/rejected": -1.3103519678115845,
      "logps/chosen": -1.5224910974502563,
      "logps/rejected": -2.992649555206299,
      "loss": 1.5431,
      "nll_loss": 1.524802327156067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15224911272525787,
      "rewards/margins": 0.14701583981513977,
      "rewards/rejected": -0.29926493763923645,
      "step": 1053
    },
    {
      "epoch": 1.660362490149724,
      "grad_norm": 0.2615368068218231,
      "learning_rate": 3.45313761191256e-07,
      "log_odds_chosen": 1.1876877546310425,
      "log_odds_ratio": -0.2874801754951477,
      "logits/chosen": -0.6191905736923218,
      "logits/rejected": -0.983271598815918,
      "logps/chosen": -1.4506546258926392,
      "logps/rejected": -2.467876672744751,
      "loss": 1.4911,
      "nll_loss": 1.4623490571975708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14506545662879944,
      "rewards/margins": 0.10172222554683685,
      "rewards/rejected": -0.2467876821756363,
      "step": 1054
    },
    {
      "epoch": 1.6619385342789599,
      "grad_norm": 0.23760876059532166,
      "learning_rate": 3.4223231144723253e-07,
      "log_odds_chosen": 1.3772153854370117,
      "log_odds_ratio": -0.24057058990001678,
      "logits/chosen": -0.5682808756828308,
      "logits/rejected": -1.1523633003234863,
      "logps/chosen": -1.548622965812683,
      "logps/rejected": -2.7544891834259033,
      "loss": 1.5902,
      "nll_loss": 1.5661405324935913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15486228466033936,
      "rewards/margins": 0.12058664858341217,
      "rewards/rejected": -0.27544891834259033,
      "step": 1055
    },
    {
      "epoch": 1.6635145784081953,
      "grad_norm": 0.28083619475364685,
      "learning_rate": 3.391633853462861e-07,
      "log_odds_chosen": 1.600939154624939,
      "log_odds_ratio": -0.18783672153949738,
      "logits/chosen": -0.5874448418617249,
      "logits/rejected": -1.1939144134521484,
      "logps/chosen": -1.4916601181030273,
      "logps/rejected": -2.8930625915527344,
      "loss": 1.5353,
      "nll_loss": 1.5165108442306519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14916600286960602,
      "rewards/margins": 0.14014025032520294,
      "rewards/rejected": -0.28930628299713135,
      "step": 1056
    },
    {
      "epoch": 1.6650906225374311,
      "grad_norm": 0.24082481861114502,
      "learning_rate": 3.3610700607268803e-07,
      "log_odds_chosen": 1.3335075378417969,
      "log_odds_ratio": -0.23781822621822357,
      "logits/chosen": -0.5040817856788635,
      "logits/rejected": -1.1055271625518799,
      "logps/chosen": -1.473649263381958,
      "logps/rejected": -2.6233253479003906,
      "loss": 1.5121,
      "nll_loss": 1.4883246421813965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14736491441726685,
      "rewards/margins": 0.11496759951114655,
      "rewards/rejected": -0.2623325288295746,
      "step": 1057
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 0.23849353194236755,
      "learning_rate": 3.330631967159239e-07,
      "log_odds_chosen": 1.3722360134124756,
      "log_odds_ratio": -0.22970950603485107,
      "logits/chosen": -0.5206635594367981,
      "logits/rejected": -1.207780361175537,
      "logps/chosen": -1.4683030843734741,
      "logps/rejected": -2.652791976928711,
      "loss": 1.4991,
      "nll_loss": 1.4761226177215576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14683032035827637,
      "rewards/margins": 0.118448905646801,
      "rewards/rejected": -0.26527920365333557,
      "step": 1058
    },
    {
      "epoch": 1.6682427107959024,
      "grad_norm": 0.22701510787010193,
      "learning_rate": 3.300319802705189e-07,
      "log_odds_chosen": 1.7900515794754028,
      "log_odds_ratio": -0.1908876895904541,
      "logits/chosen": -0.4572758376598358,
      "logits/rejected": -1.2248320579528809,
      "logps/chosen": -1.5278782844543457,
      "logps/rejected": -3.128338575363159,
      "loss": 1.5608,
      "nll_loss": 1.5417128801345825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15278781950473785,
      "rewards/margins": 0.1600460410118103,
      "rewards/rejected": -0.31283384561538696,
      "step": 1059
    },
    {
      "epoch": 1.6698187549251378,
      "grad_norm": 0.22186289727687836,
      "learning_rate": 3.2701337963586604e-07,
      "log_odds_chosen": 1.4773789644241333,
      "log_odds_ratio": -0.21626219153404236,
      "logits/chosen": -0.5267972350120544,
      "logits/rejected": -1.1005977392196655,
      "logps/chosen": -1.3346282243728638,
      "logps/rejected": -2.5860729217529297,
      "loss": 1.3837,
      "nll_loss": 1.3620424270629883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1334628164768219,
      "rewards/margins": 0.12514445185661316,
      "rewards/rejected": -0.25860732793807983,
      "step": 1060
    },
    {
      "epoch": 1.6713947990543736,
      "grad_norm": 0.23126469552516937,
      "learning_rate": 3.24007417616051e-07,
      "log_odds_chosen": 1.4688457250595093,
      "log_odds_ratio": -0.22335568070411682,
      "logits/chosen": -0.6574072241783142,
      "logits/rejected": -1.150550127029419,
      "logps/chosen": -1.4284827709197998,
      "logps/rejected": -2.7017009258270264,
      "loss": 1.4524,
      "nll_loss": 1.4300440549850464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14284828305244446,
      "rewards/margins": 0.12732183933258057,
      "rewards/rejected": -0.27017009258270264,
      "step": 1061
    },
    {
      "epoch": 1.672970843183609,
      "grad_norm": 0.45742619037628174,
      "learning_rate": 3.2101411691968115e-07,
      "log_odds_chosen": 1.4833433628082275,
      "log_odds_ratio": -0.21074333786964417,
      "logits/chosen": -0.5551731586456299,
      "logits/rejected": -1.2144925594329834,
      "logps/chosen": -1.430672526359558,
      "logps/rejected": -2.707599639892578,
      "loss": 1.4694,
      "nll_loss": 1.4483249187469482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14306725561618805,
      "rewards/margins": 0.12769271433353424,
      "rewards/rejected": -0.2707599699497223,
      "step": 1062
    },
    {
      "epoch": 1.6745468873128448,
      "grad_norm": 0.2510012090206146,
      "learning_rate": 3.180335001597143e-07,
      "log_odds_chosen": 1.642303705215454,
      "log_odds_ratio": -0.1978750079870224,
      "logits/chosen": -0.6389750838279724,
      "logits/rejected": -1.1751081943511963,
      "logps/chosen": -1.4891475439071655,
      "logps/rejected": -2.9355459213256836,
      "loss": 1.5104,
      "nll_loss": 1.4906222820281982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14891476929187775,
      "rewards/margins": 0.14463984966278076,
      "rewards/rejected": -0.2935546338558197,
      "step": 1063
    },
    {
      "epoch": 1.6761229314420802,
      "grad_norm": 0.2376561164855957,
      "learning_rate": 3.1506558985328523e-07,
      "log_odds_chosen": 1.521850347518921,
      "log_odds_ratio": -0.21236465871334076,
      "logits/chosen": -0.5390481352806091,
      "logits/rejected": -1.322774887084961,
      "logps/chosen": -1.4321036338806152,
      "logps/rejected": -2.747835159301758,
      "loss": 1.4603,
      "nll_loss": 1.439075231552124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14321036636829376,
      "rewards/margins": 0.1315731704235077,
      "rewards/rejected": -0.27478355169296265,
      "step": 1064
    },
    {
      "epoch": 1.677698975571316,
      "grad_norm": 0.2653907835483551,
      "learning_rate": 3.1211040842153934e-07,
      "log_odds_chosen": 1.362098217010498,
      "log_odds_ratio": -0.23629456758499146,
      "logits/chosen": -0.5150231122970581,
      "logits/rejected": -1.1769152879714966,
      "logps/chosen": -1.6363221406936646,
      "logps/rejected": -2.8428685665130615,
      "loss": 1.6424,
      "nll_loss": 1.6187540292739868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16363221406936646,
      "rewards/margins": 0.1206546351313591,
      "rewards/rejected": -0.28428685665130615,
      "step": 1065
    },
    {
      "epoch": 1.6792750197005515,
      "grad_norm": 0.24473443627357483,
      "learning_rate": 3.091679781894605e-07,
      "log_odds_chosen": 1.5158252716064453,
      "log_odds_ratio": -0.20809954404830933,
      "logits/chosen": -0.5453354120254517,
      "logits/rejected": -1.0543676614761353,
      "logps/chosen": -1.4767189025878906,
      "logps/rejected": -2.796322822570801,
      "loss": 1.495,
      "nll_loss": 1.4742385149002075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1476718932390213,
      "rewards/margins": 0.1319604068994522,
      "rewards/rejected": -0.2796323299407959,
      "step": 1066
    },
    {
      "epoch": 1.6808510638297873,
      "grad_norm": 0.23107919096946716,
      "learning_rate": 3.0623832138570473e-07,
      "log_odds_chosen": 1.6840357780456543,
      "log_odds_ratio": -0.17377886176109314,
      "logits/chosen": -0.5350216031074524,
      "logits/rejected": -1.1876095533370972,
      "logps/chosen": -1.3697733879089355,
      "logps/rejected": -2.8162453174591064,
      "loss": 1.3958,
      "nll_loss": 1.3783882856369019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13697734475135803,
      "rewards/margins": 0.14464718103408813,
      "rewards/rejected": -0.28162452578544617,
      "step": 1067
    },
    {
      "epoch": 1.6824271079590227,
      "grad_norm": 0.24077357351779938,
      "learning_rate": 3.033214601424283e-07,
      "log_odds_chosen": 1.4439938068389893,
      "log_odds_ratio": -0.2295798361301422,
      "logits/chosen": -0.48163264989852905,
      "logits/rejected": -0.9928042888641357,
      "logps/chosen": -1.505954384803772,
      "logps/rejected": -2.7707200050354004,
      "loss": 1.5295,
      "nll_loss": 1.5065245628356934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15059544146060944,
      "rewards/margins": 0.12647655606269836,
      "rewards/rejected": -0.277072012424469,
      "step": 1068
    },
    {
      "epoch": 1.6840031520882586,
      "grad_norm": 0.24111145734786987,
      "learning_rate": 3.0041741649512584e-07,
      "log_odds_chosen": 1.5701847076416016,
      "log_odds_ratio": -0.1972634494304657,
      "logits/chosen": -0.4729172885417938,
      "logits/rejected": -1.2404983043670654,
      "logps/chosen": -1.4998829364776611,
      "logps/rejected": -2.87569522857666,
      "loss": 1.5194,
      "nll_loss": 1.4996436834335327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1499882936477661,
      "rewards/margins": 0.1375812590122223,
      "rewards/rejected": -0.2875695526599884,
      "step": 1069
    },
    {
      "epoch": 1.685579196217494,
      "grad_norm": 0.2563757002353668,
      "learning_rate": 2.975262123824596e-07,
      "log_odds_chosen": 1.4139864444732666,
      "log_odds_ratio": -0.2330787181854248,
      "logits/chosen": -0.6340278387069702,
      "logits/rejected": -1.1499080657958984,
      "logps/chosen": -1.4550116062164307,
      "logps/rejected": -2.6778602600097656,
      "loss": 1.4889,
      "nll_loss": 1.4655683040618896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14550116658210754,
      "rewards/margins": 0.12228487432003021,
      "rewards/rejected": -0.26778602600097656,
      "step": 1070
    },
    {
      "epoch": 1.6871552403467298,
      "grad_norm": 0.23501606285572052,
      "learning_rate": 2.9464786964609567e-07,
      "log_odds_chosen": 1.449416995048523,
      "log_odds_ratio": -0.22743137180805206,
      "logits/chosen": -0.5684559941291809,
      "logits/rejected": -1.015746831893921,
      "logps/chosen": -1.512995958328247,
      "logps/rejected": -2.7820746898651123,
      "loss": 1.532,
      "nll_loss": 1.509217381477356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1512995958328247,
      "rewards/margins": 0.12690787017345428,
      "rewards/rejected": -0.2782074809074402,
      "step": 1071
    },
    {
      "epoch": 1.6887312844759652,
      "grad_norm": 0.22857315838336945,
      "learning_rate": 2.917824100305393e-07,
      "log_odds_chosen": 1.5437496900558472,
      "log_odds_ratio": -0.20127391815185547,
      "logits/chosen": -0.48678553104400635,
      "logits/rejected": -1.2556241750717163,
      "logps/chosen": -1.4958239793777466,
      "logps/rejected": -2.848050355911255,
      "loss": 1.5241,
      "nll_loss": 1.5040056705474854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495824009180069,
      "rewards/margins": 0.1352226436138153,
      "rewards/rejected": -0.2848050594329834,
      "step": 1072
    },
    {
      "epoch": 1.690307328605201,
      "grad_norm": 0.22445034980773926,
      "learning_rate": 2.889298551829682e-07,
      "log_odds_chosen": 1.3881555795669556,
      "log_odds_ratio": -0.2381008118391037,
      "logits/chosen": -0.5410254001617432,
      "logits/rejected": -1.0424563884735107,
      "logps/chosen": -1.4065275192260742,
      "logps/rejected": -2.5901050567626953,
      "loss": 1.444,
      "nll_loss": 1.4202224016189575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14065276086330414,
      "rewards/margins": 0.11835776269435883,
      "rewards/rejected": -0.25901052355766296,
      "step": 1073
    },
    {
      "epoch": 1.6918833727344365,
      "grad_norm": 0.22629152238368988,
      "learning_rate": 2.8609022665307226e-07,
      "log_odds_chosen": 1.4165868759155273,
      "log_odds_ratio": -0.22475402057170868,
      "logits/chosen": -0.5748792886734009,
      "logits/rejected": -1.0941567420959473,
      "logps/chosen": -1.44132661819458,
      "logps/rejected": -2.6615536212921143,
      "loss": 1.4783,
      "nll_loss": 1.455823302268982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14413267374038696,
      "rewards/margins": 0.12202268093824387,
      "rewards/rejected": -0.26615533232688904,
      "step": 1074
    },
    {
      "epoch": 1.6934594168636723,
      "grad_norm": 0.22893467545509338,
      "learning_rate": 2.832635458928887e-07,
      "log_odds_chosen": 1.44675612449646,
      "log_odds_ratio": -0.21547579765319824,
      "logits/chosen": -0.5010592937469482,
      "logits/rejected": -1.1085104942321777,
      "logps/chosen": -1.3950062990188599,
      "logps/rejected": -2.6264936923980713,
      "loss": 1.4455,
      "nll_loss": 1.4239811897277832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13950063288211823,
      "rewards/margins": 0.12314873188734055,
      "rewards/rejected": -0.26264938712120056,
      "step": 1075
    },
    {
      "epoch": 1.6950354609929077,
      "grad_norm": 0.2273607701063156,
      "learning_rate": 2.804498342566406e-07,
      "log_odds_chosen": 1.7893620729446411,
      "log_odds_ratio": -0.1783633977174759,
      "logits/chosen": -0.5607014298439026,
      "logits/rejected": -1.116290807723999,
      "logps/chosen": -1.3538328409194946,
      "logps/rejected": -2.9056737422943115,
      "loss": 1.4031,
      "nll_loss": 1.385244369506836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13538329303264618,
      "rewards/margins": 0.1551840901374817,
      "rewards/rejected": -0.2905673682689667,
      "step": 1076
    },
    {
      "epoch": 1.6966115051221435,
      "grad_norm": 0.24715925753116608,
      "learning_rate": 2.776491130005758e-07,
      "log_odds_chosen": 1.3739885091781616,
      "log_odds_ratio": -0.24058020114898682,
      "logits/chosen": -0.48696237802505493,
      "logits/rejected": -1.1175223588943481,
      "logps/chosen": -1.4451289176940918,
      "logps/rejected": -2.6200382709503174,
      "loss": 1.5014,
      "nll_loss": 1.477297067642212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14451290667057037,
      "rewards/margins": 0.11749093979597092,
      "rewards/rejected": -0.2620038390159607,
      "step": 1077
    },
    {
      "epoch": 1.698187549251379,
      "grad_norm": 0.23278237879276276,
      "learning_rate": 2.7486140328280473e-07,
      "log_odds_chosen": 1.4528337717056274,
      "log_odds_ratio": -0.2165869176387787,
      "logits/chosen": -0.640472412109375,
      "logits/rejected": -1.3511531352996826,
      "logps/chosen": -1.5099921226501465,
      "logps/rejected": -2.7752208709716797,
      "loss": 1.5306,
      "nll_loss": 1.508980631828308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15099921822547913,
      "rewards/margins": 0.12652288377285004,
      "rewards/rejected": -0.27752211689949036,
      "step": 1078
    },
    {
      "epoch": 1.6997635933806148,
      "grad_norm": 0.24571862816810608,
      "learning_rate": 2.7208672616314345e-07,
      "log_odds_chosen": 1.5212634801864624,
      "log_odds_ratio": -0.20545443892478943,
      "logits/chosen": -0.6022442579269409,
      "logits/rejected": -1.1761116981506348,
      "logps/chosen": -1.5129997730255127,
      "logps/rejected": -2.8458454608917236,
      "loss": 1.5394,
      "nll_loss": 1.5188080072402954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15129998326301575,
      "rewards/margins": 0.1332845538854599,
      "rewards/rejected": -0.28458455204963684,
      "step": 1079
    },
    {
      "epoch": 1.7013396375098502,
      "grad_norm": 0.23747242987155914,
      "learning_rate": 2.6932510260295237e-07,
      "log_odds_chosen": 1.4921329021453857,
      "log_odds_ratio": -0.22186747193336487,
      "logits/chosen": -0.5433292388916016,
      "logits/rejected": -1.0109025239944458,
      "logps/chosen": -1.3942593336105347,
      "logps/rejected": -2.6738498210906982,
      "loss": 1.4295,
      "nll_loss": 1.407320261001587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13942593336105347,
      "rewards/margins": 0.12795905768871307,
      "rewards/rejected": -0.26738500595092773,
      "step": 1080
    },
    {
      "epoch": 1.702915681639086,
      "grad_norm": 0.251139372587204,
      "learning_rate": 2.66576553464978e-07,
      "log_odds_chosen": 1.3601816892623901,
      "log_odds_ratio": -0.23202723264694214,
      "logits/chosen": -0.5780200362205505,
      "logits/rejected": -1.2449634075164795,
      "logps/chosen": -1.5806702375411987,
      "logps/rejected": -2.7753427028656006,
      "loss": 1.5986,
      "nll_loss": 1.5753669738769531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1580670177936554,
      "rewards/margins": 0.11946723610162735,
      "rewards/rejected": -0.27753427624702454,
      "step": 1081
    },
    {
      "epoch": 1.7044917257683214,
      "grad_norm": 0.23873770236968994,
      "learning_rate": 2.6384109951319654e-07,
      "log_odds_chosen": 1.3129750490188599,
      "log_odds_ratio": -0.2519412636756897,
      "logits/chosen": -0.533735990524292,
      "logits/rejected": -1.1656168699264526,
      "logps/chosen": -1.4920392036437988,
      "logps/rejected": -2.6273510456085205,
      "loss": 1.5201,
      "nll_loss": 1.4949027299880981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14920392632484436,
      "rewards/margins": 0.1135311871767044,
      "rewards/rejected": -0.2627350986003876,
      "step": 1082
    },
    {
      "epoch": 1.7060677698975573,
      "grad_norm": 0.23456136882305145,
      "learning_rate": 2.6111876141265646e-07,
      "log_odds_chosen": 1.3457024097442627,
      "log_odds_ratio": -0.24244683980941772,
      "logits/chosen": -0.5344383120536804,
      "logits/rejected": -1.1451764106750488,
      "logps/chosen": -1.456725835800171,
      "logps/rejected": -2.619199275970459,
      "loss": 1.491,
      "nll_loss": 1.4667109251022339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14567258954048157,
      "rewards/margins": 0.11624733358621597,
      "rewards/rejected": -0.26191991567611694,
      "step": 1083
    },
    {
      "epoch": 1.7076438140267927,
      "grad_norm": 0.23178094625473022,
      "learning_rate": 2.58409559729321e-07,
      "log_odds_chosen": 1.3686121702194214,
      "log_odds_ratio": -0.2414955049753189,
      "logits/chosen": -0.5990633964538574,
      "logits/rejected": -1.1458981037139893,
      "logps/chosen": -1.4336516857147217,
      "logps/rejected": -2.6138854026794434,
      "loss": 1.4724,
      "nll_loss": 1.4482686519622803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14336517453193665,
      "rewards/margins": 0.1180233582854271,
      "rewards/rejected": -0.26138854026794434,
      "step": 1084
    },
    {
      "epoch": 1.7092198581560285,
      "grad_norm": 0.22593750059604645,
      "learning_rate": 2.55713514929915e-07,
      "log_odds_chosen": 1.5214976072311401,
      "log_odds_ratio": -0.21560263633728027,
      "logits/chosen": -0.6354103684425354,
      "logits/rejected": -1.1834520101547241,
      "logps/chosen": -1.4260567426681519,
      "logps/rejected": -2.742741107940674,
      "loss": 1.4634,
      "nll_loss": 1.4418387413024902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14260567724704742,
      "rewards/margins": 0.13166844844818115,
      "rewards/rejected": -0.2742741107940674,
      "step": 1085
    },
    {
      "epoch": 1.710795902285264,
      "grad_norm": 0.25965067744255066,
      "learning_rate": 2.530306473817696e-07,
      "log_odds_chosen": 1.444462537765503,
      "log_odds_ratio": -0.22052349150180817,
      "logits/chosen": -0.4930550158023834,
      "logits/rejected": -1.2061861753463745,
      "logps/chosen": -1.5523051023483276,
      "logps/rejected": -2.8222475051879883,
      "loss": 1.5726,
      "nll_loss": 1.550567388534546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15523050725460052,
      "rewards/margins": 0.12699423730373383,
      "rewards/rejected": -0.28222474455833435,
      "step": 1086
    },
    {
      "epoch": 1.7123719464144997,
      "grad_norm": 0.2601432800292969,
      "learning_rate": 2.50360977352668e-07,
      "log_odds_chosen": 1.5093591213226318,
      "log_odds_ratio": -0.20860955119132996,
      "logits/chosen": -0.5795674920082092,
      "logits/rejected": -1.0373741388320923,
      "logps/chosen": -1.5181925296783447,
      "logps/rejected": -2.83870267868042,
      "loss": 1.5492,
      "nll_loss": 1.5283558368682861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15181924402713776,
      "rewards/margins": 0.13205105066299438,
      "rewards/rejected": -0.28387027978897095,
      "step": 1087
    },
    {
      "epoch": 1.7139479905437351,
      "grad_norm": 0.24424780905246735,
      "learning_rate": 2.477045250106913e-07,
      "log_odds_chosen": 1.458065152168274,
      "log_odds_ratio": -0.23266252875328064,
      "logits/chosen": -0.4854944348335266,
      "logits/rejected": -1.1634329557418823,
      "logps/chosen": -1.5861568450927734,
      "logps/rejected": -2.8769891262054443,
      "loss": 1.6008,
      "nll_loss": 1.5775679349899292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15861569344997406,
      "rewards/margins": 0.12908321619033813,
      "rewards/rejected": -0.287698894739151,
      "step": 1088
    },
    {
      "epoch": 1.715524034672971,
      "grad_norm": 0.2365056276321411,
      "learning_rate": 2.450613104240684e-07,
      "log_odds_chosen": 1.9529310464859009,
      "log_odds_ratio": -0.1459507942199707,
      "logits/chosen": -0.5606799721717834,
      "logits/rejected": -1.2288486957550049,
      "logps/chosen": -1.449303150177002,
      "logps/rejected": -3.1704654693603516,
      "loss": 1.464,
      "nll_loss": 1.4493612051010132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14493031799793243,
      "rewards/margins": 0.1721162497997284,
      "rewards/rejected": -0.31704655289649963,
      "step": 1089
    },
    {
      "epoch": 1.7171000788022064,
      "grad_norm": 0.2658349275588989,
      "learning_rate": 2.42431353561023e-07,
      "log_odds_chosen": 1.5651302337646484,
      "log_odds_ratio": -0.2014286369085312,
      "logits/chosen": -0.5602660179138184,
      "logits/rejected": -1.1105549335479736,
      "logps/chosen": -1.5207159519195557,
      "logps/rejected": -2.897491693496704,
      "loss": 1.5524,
      "nll_loss": 1.532224416732788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15207159519195557,
      "rewards/margins": 0.13767756521701813,
      "rewards/rejected": -0.2897491753101349,
      "step": 1090
    },
    {
      "epoch": 1.7186761229314422,
      "grad_norm": 0.24004048109054565,
      "learning_rate": 2.3981467428962364e-07,
      "log_odds_chosen": 1.6278833150863647,
      "log_odds_ratio": -0.2189147025346756,
      "logits/chosen": -0.51349276304245,
      "logits/rejected": -1.2104847431182861,
      "logps/chosen": -1.433386206626892,
      "logps/rejected": -2.8616557121276855,
      "loss": 1.4652,
      "nll_loss": 1.4432955980300903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433386206626892,
      "rewards/margins": 0.14282691478729248,
      "rewards/rejected": -0.2861655652523041,
      "step": 1091
    },
    {
      "epoch": 1.7202521670606776,
      "grad_norm": 0.2524133324623108,
      "learning_rate": 2.372112923776315e-07,
      "log_odds_chosen": 1.4475654363632202,
      "log_odds_ratio": -0.2231072634458542,
      "logits/chosen": -0.5050214529037476,
      "logits/rejected": -1.057558298110962,
      "logps/chosen": -1.5443834066390991,
      "logps/rejected": -2.813723564147949,
      "loss": 1.5724,
      "nll_loss": 1.5500787496566772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544383317232132,
      "rewards/margins": 0.1269340217113495,
      "rewards/rejected": -0.2813723385334015,
      "step": 1092
    },
    {
      "epoch": 1.7218282111899135,
      "grad_norm": 0.23853714764118195,
      "learning_rate": 2.346212274923538e-07,
      "log_odds_chosen": 1.2487622499465942,
      "log_odds_ratio": -0.2640945613384247,
      "logits/chosen": -0.6171495318412781,
      "logits/rejected": -1.1145663261413574,
      "logps/chosen": -1.4421777725219727,
      "logps/rejected": -2.511570453643799,
      "loss": 1.4862,
      "nll_loss": 1.4597663879394531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14421778917312622,
      "rewards/margins": 0.10693926364183426,
      "rewards/rejected": -0.2511570453643799,
      "step": 1093
    },
    {
      "epoch": 1.7234042553191489,
      "grad_norm": 0.2417575865983963,
      "learning_rate": 2.3204449920049372e-07,
      "log_odds_chosen": 1.4656773805618286,
      "log_odds_ratio": -0.21700911223888397,
      "logits/chosen": -0.584980845451355,
      "logits/rejected": -1.1715086698532104,
      "logps/chosen": -1.606001377105713,
      "logps/rejected": -2.9031922817230225,
      "loss": 1.6103,
      "nll_loss": 1.5885722637176514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16060014069080353,
      "rewards/margins": 0.12971912324428558,
      "rewards/rejected": -0.2903192639350891,
      "step": 1094
    },
    {
      "epoch": 1.7249802994483847,
      "grad_norm": 0.26134467124938965,
      "learning_rate": 2.2948112696800283e-07,
      "log_odds_chosen": 1.3611866235733032,
      "log_odds_ratio": -0.23141160607337952,
      "logits/chosen": -0.5193547606468201,
      "logits/rejected": -1.2015901803970337,
      "logps/chosen": -1.5611448287963867,
      "logps/rejected": -2.751809597015381,
      "loss": 1.581,
      "nll_loss": 1.557887077331543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15611448884010315,
      "rewards/margins": 0.11906647682189941,
      "rewards/rejected": -0.27518096566200256,
      "step": 1095
    },
    {
      "epoch": 1.7265563435776201,
      "grad_norm": 0.24378441274166107,
      "learning_rate": 2.2693113015993438e-07,
      "log_odds_chosen": 1.3584837913513184,
      "log_odds_ratio": -0.2515845000743866,
      "logits/chosen": -0.5921484231948853,
      "logits/rejected": -1.0930372476577759,
      "logps/chosen": -1.56572687625885,
      "logps/rejected": -2.761518955230713,
      "loss": 1.5869,
      "nll_loss": 1.5617401599884033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15657269954681396,
      "rewards/margins": 0.11957921087741852,
      "rewards/rejected": -0.2761518955230713,
      "step": 1096
    },
    {
      "epoch": 1.728132387706856,
      "grad_norm": 0.23917409777641296,
      "learning_rate": 2.2439452804029524e-07,
      "log_odds_chosen": 1.531793236732483,
      "log_odds_ratio": -0.20326431095600128,
      "logits/chosen": -0.42496171593666077,
      "logits/rejected": -1.1399880647659302,
      "logps/chosen": -1.541717767715454,
      "logps/rejected": -2.8890419006347656,
      "loss": 1.5563,
      "nll_loss": 1.5359447002410889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15417177975177765,
      "rewards/margins": 0.1347324252128601,
      "rewards/rejected": -0.28890419006347656,
      "step": 1097
    },
    {
      "epoch": 1.7297084318360914,
      "grad_norm": 0.24650467932224274,
      "learning_rate": 2.2187133977190343e-07,
      "log_odds_chosen": 1.5100090503692627,
      "log_odds_ratio": -0.21313849091529846,
      "logits/chosen": -0.573370099067688,
      "logits/rejected": -1.1048282384872437,
      "logps/chosen": -1.4752497673034668,
      "logps/rejected": -2.7901291847229004,
      "loss": 1.4897,
      "nll_loss": 1.4683949947357178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14752499759197235,
      "rewards/margins": 0.1314879208803177,
      "rewards/rejected": -0.27901291847229004,
      "step": 1098
    },
    {
      "epoch": 1.731284475965327,
      "grad_norm": 0.2658592164516449,
      "learning_rate": 2.193615844162411e-07,
      "log_odds_chosen": 1.4578858613967896,
      "log_odds_ratio": -0.2233109325170517,
      "logits/chosen": -0.558879554271698,
      "logits/rejected": -1.1920090913772583,
      "logps/chosen": -1.498852014541626,
      "logps/rejected": -2.769965171813965,
      "loss": 1.518,
      "nll_loss": 1.4956549406051636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14988519251346588,
      "rewards/margins": 0.1271113008260727,
      "rewards/rejected": -0.27699652314186096,
      "step": 1099
    },
    {
      "epoch": 1.7328605200945626,
      "grad_norm": 0.2357098013162613,
      "learning_rate": 2.1686528093331113e-07,
      "log_odds_chosen": 1.7384421825408936,
      "log_odds_ratio": -0.17331789433956146,
      "logits/chosen": -0.6772224307060242,
      "logits/rejected": -1.233218789100647,
      "logps/chosen": -1.4087449312210083,
      "logps/rejected": -2.9215545654296875,
      "loss": 1.4286,
      "nll_loss": 1.4112974405288696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1408744901418686,
      "rewards/margins": 0.1512809842824936,
      "rewards/rejected": -0.2921554744243622,
      "step": 1100
    },
    {
      "epoch": 1.7344365642237982,
      "grad_norm": 0.24360127747058868,
      "learning_rate": 2.1438244818149465e-07,
      "log_odds_chosen": 1.5256110429763794,
      "log_odds_ratio": -0.20060251653194427,
      "logits/chosen": -0.6647403240203857,
      "logits/rejected": -1.2393889427185059,
      "logps/chosen": -1.5186569690704346,
      "logps/rejected": -2.8553578853607178,
      "loss": 1.5382,
      "nll_loss": 1.5181422233581543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15186570584774017,
      "rewards/margins": 0.13367009162902832,
      "rewards/rejected": -0.2855357825756073,
      "step": 1101
    },
    {
      "epoch": 1.7360126083530338,
      "grad_norm": 0.3265460431575775,
      "learning_rate": 2.1191310491740633e-07,
      "log_odds_chosen": 1.1639649868011475,
      "log_odds_ratio": -0.2778826951980591,
      "logits/chosen": -0.49773356318473816,
      "logits/rejected": -0.9975111484527588,
      "logps/chosen": -1.640123963356018,
      "logps/rejected": -2.6622049808502197,
      "loss": 1.6593,
      "nll_loss": 1.6315081119537354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16401241719722748,
      "rewards/margins": 0.10220807045698166,
      "rewards/rejected": -0.26622048020362854,
      "step": 1102
    },
    {
      "epoch": 1.7375886524822695,
      "grad_norm": 0.24258384108543396,
      "learning_rate": 2.0945726979575528e-07,
      "log_odds_chosen": 1.6687278747558594,
      "log_odds_ratio": -0.19728343188762665,
      "logits/chosen": -0.5355981588363647,
      "logits/rejected": -1.1707581281661987,
      "logps/chosen": -1.5089631080627441,
      "logps/rejected": -2.9873952865600586,
      "loss": 1.5302,
      "nll_loss": 1.510464072227478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15089631080627441,
      "rewards/margins": 0.14784322679042816,
      "rewards/rejected": -0.29873955249786377,
      "step": 1103
    },
    {
      "epoch": 1.739164696611505,
      "grad_norm": 0.257173091173172,
      "learning_rate": 2.0701496136920315e-07,
      "log_odds_chosen": 1.6476998329162598,
      "log_odds_ratio": -0.1820983737707138,
      "logits/chosen": -0.5421767234802246,
      "logits/rejected": -1.076603651046753,
      "logps/chosen": -1.4623558521270752,
      "logps/rejected": -2.899317502975464,
      "loss": 1.4803,
      "nll_loss": 1.4620407819747925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14623558521270752,
      "rewards/margins": 0.14369618892669678,
      "rewards/rejected": -0.2899317741394043,
      "step": 1104
    },
    {
      "epoch": 1.7407407407407407,
      "grad_norm": 0.22429010272026062,
      "learning_rate": 2.0458619808822352e-07,
      "log_odds_chosen": 1.5790514945983887,
      "log_odds_ratio": -0.19565466046333313,
      "logits/chosen": -0.5493534803390503,
      "logits/rejected": -1.0987398624420166,
      "logps/chosen": -1.419497013092041,
      "logps/rejected": -2.784424304962158,
      "loss": 1.4575,
      "nll_loss": 1.4379711151123047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14194971323013306,
      "rewards/margins": 0.13649271428585052,
      "rewards/rejected": -0.2784424126148224,
      "step": 1105
    },
    {
      "epoch": 1.7423167848699763,
      "grad_norm": 0.23635801672935486,
      "learning_rate": 2.0217099830096297e-07,
      "log_odds_chosen": 1.7080570459365845,
      "log_odds_ratio": -0.17423473298549652,
      "logits/chosen": -0.5766207575798035,
      "logits/rejected": -1.1279845237731934,
      "logps/chosen": -1.3899341821670532,
      "logps/rejected": -2.8659658432006836,
      "loss": 1.4216,
      "nll_loss": 1.4042110443115234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13899341225624084,
      "rewards/margins": 0.1476031392812729,
      "rewards/rejected": -0.2865965664386749,
      "step": 1106
    },
    {
      "epoch": 1.743892828999212,
      "grad_norm": 0.23576894402503967,
      "learning_rate": 1.9976938025310196e-07,
      "log_odds_chosen": 1.6113464832305908,
      "log_odds_ratio": -0.19843654334545135,
      "logits/chosen": -0.619926393032074,
      "logits/rejected": -1.1325708627700806,
      "logps/chosen": -1.495015263557434,
      "logps/rejected": -2.9112677574157715,
      "loss": 1.5139,
      "nll_loss": 1.4940965175628662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14950154721736908,
      "rewards/margins": 0.1416252702474594,
      "rewards/rejected": -0.2911267876625061,
      "step": 1107
    },
    {
      "epoch": 1.7454688731284476,
      "grad_norm": 0.2326350212097168,
      "learning_rate": 1.9738136208771783e-07,
      "log_odds_chosen": 1.5742504596710205,
      "log_odds_ratio": -0.1925155520439148,
      "logits/chosen": -0.5969733595848083,
      "logits/rejected": -1.2294520139694214,
      "logps/chosen": -1.5006991624832153,
      "logps/rejected": -2.8803024291992188,
      "loss": 1.5137,
      "nll_loss": 1.4944555759429932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.150069922208786,
      "rewards/margins": 0.13796032965183258,
      "rewards/rejected": -0.2880302667617798,
      "step": 1108
    },
    {
      "epoch": 1.7470449172576832,
      "grad_norm": 0.22903689742088318,
      "learning_rate": 1.950069618451473e-07,
      "log_odds_chosen": 1.3628047704696655,
      "log_odds_ratio": -0.24196141958236694,
      "logits/chosen": -0.5716612935066223,
      "logits/rejected": -1.1057016849517822,
      "logps/chosen": -1.405078411102295,
      "logps/rejected": -2.568618059158325,
      "loss": 1.4477,
      "nll_loss": 1.4235260486602783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14050784707069397,
      "rewards/margins": 0.11635395884513855,
      "rewards/rejected": -0.2568618059158325,
      "step": 1109
    },
    {
      "epoch": 1.7486209613869188,
      "grad_norm": 0.23709827661514282,
      "learning_rate": 1.9264619746285016e-07,
      "log_odds_chosen": 1.2230143547058105,
      "log_odds_ratio": -0.27061185240745544,
      "logits/chosen": -0.6539244651794434,
      "logits/rejected": -1.0932153463363647,
      "logps/chosen": -1.4552053213119507,
      "logps/rejected": -2.498868942260742,
      "loss": 1.4862,
      "nll_loss": 1.4591104984283447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14552052319049835,
      "rewards/margins": 0.10436636209487915,
      "rewards/rejected": -0.2498868852853775,
      "step": 1110
    },
    {
      "epoch": 1.7501970055161544,
      "grad_norm": 0.25280964374542236,
      "learning_rate": 1.9029908677527406e-07,
      "log_odds_chosen": 1.6682102680206299,
      "log_odds_ratio": -0.18214134871959686,
      "logits/chosen": -0.5879977345466614,
      "logits/rejected": -1.090448260307312,
      "logps/chosen": -1.4783356189727783,
      "logps/rejected": -2.9406332969665527,
      "loss": 1.4981,
      "nll_loss": 1.4798402786254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14783355593681335,
      "rewards/margins": 0.14622977375984192,
      "rewards/rejected": -0.2940633296966553,
      "step": 1111
    },
    {
      "epoch": 1.75177304964539,
      "grad_norm": 0.23208437860012054,
      "learning_rate": 1.8796564751371836e-07,
      "log_odds_chosen": 1.264046311378479,
      "log_odds_ratio": -0.2632746696472168,
      "logits/chosen": -0.5130308866500854,
      "logits/rejected": -1.035696268081665,
      "logps/chosen": -1.502026915550232,
      "logps/rejected": -2.597893238067627,
      "loss": 1.5482,
      "nll_loss": 1.5218838453292847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1502026915550232,
      "rewards/margins": 0.10958661884069443,
      "rewards/rejected": -0.2597893476486206,
      "step": 1112
    },
    {
      "epoch": 1.7533490937746257,
      "grad_norm": 0.2437879741191864,
      "learning_rate": 1.8564589730620272e-07,
      "log_odds_chosen": 1.5035381317138672,
      "log_odds_ratio": -0.21155960857868195,
      "logits/chosen": -0.5891042947769165,
      "logits/rejected": -1.0861083269119263,
      "logps/chosen": -1.5119155645370483,
      "logps/rejected": -2.829953670501709,
      "loss": 1.5479,
      "nll_loss": 1.5267839431762695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1511915773153305,
      "rewards/margins": 0.13180381059646606,
      "rewards/rejected": -0.2829953730106354,
      "step": 1113
    },
    {
      "epoch": 1.7549251379038613,
      "grad_norm": 0.22875437140464783,
      "learning_rate": 1.8333985367733206e-07,
      "log_odds_chosen": 1.604655146598816,
      "log_odds_ratio": -0.1950913965702057,
      "logits/chosen": -0.5138411521911621,
      "logits/rejected": -1.293717861175537,
      "logps/chosen": -1.5479425191879272,
      "logps/rejected": -2.970597267150879,
      "loss": 1.5674,
      "nll_loss": 1.5478612184524536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15479424595832825,
      "rewards/margins": 0.14226551353931427,
      "rewards/rejected": -0.2970597743988037,
      "step": 1114
    },
    {
      "epoch": 1.756501182033097,
      "grad_norm": 0.2387307584285736,
      "learning_rate": 1.810475340481641e-07,
      "log_odds_chosen": 1.3328739404678345,
      "log_odds_ratio": -0.26466062664985657,
      "logits/chosen": -0.5754011869430542,
      "logits/rejected": -0.8903505206108093,
      "logps/chosen": -1.457855463027954,
      "logps/rejected": -2.610539436340332,
      "loss": 1.4803,
      "nll_loss": 1.4537889957427979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14578554034233093,
      "rewards/margins": 0.11526839435100555,
      "rewards/rejected": -0.2610539197921753,
      "step": 1115
    },
    {
      "epoch": 1.7580772261623325,
      "grad_norm": 0.23562105000019073,
      "learning_rate": 1.7876895573607896e-07,
      "log_odds_chosen": 1.3142449855804443,
      "log_odds_ratio": -0.24168497323989868,
      "logits/chosen": -0.5512056946754456,
      "logits/rejected": -1.2204725742340088,
      "logps/chosen": -1.5092556476593018,
      "logps/rejected": -2.6459455490112305,
      "loss": 1.5318,
      "nll_loss": 1.5076590776443481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15092556178569794,
      "rewards/margins": 0.11366899311542511,
      "rewards/rejected": -0.26459455490112305,
      "step": 1116
    },
    {
      "epoch": 1.7596532702915682,
      "grad_norm": 0.24301816523075104,
      "learning_rate": 1.7650413595464776e-07,
      "log_odds_chosen": 1.3976953029632568,
      "log_odds_ratio": -0.23031678795814514,
      "logits/chosen": -0.6844440698623657,
      "logits/rejected": -1.1706030368804932,
      "logps/chosen": -1.4924999475479126,
      "logps/rejected": -2.705697536468506,
      "loss": 1.5139,
      "nll_loss": 1.490833044052124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14925000071525574,
      "rewards/margins": 0.1213197410106659,
      "rewards/rejected": -0.27056974172592163,
      "step": 1117
    },
    {
      "epoch": 1.7612293144208038,
      "grad_norm": 0.23426489531993866,
      "learning_rate": 1.7425309181350167e-07,
      "log_odds_chosen": 1.3673816919326782,
      "log_odds_ratio": -0.2353767305612564,
      "logits/chosen": -0.5370392203330994,
      "logits/rejected": -1.1950123310089111,
      "logps/chosen": -1.4533120393753052,
      "logps/rejected": -2.6291894912719727,
      "loss": 1.482,
      "nll_loss": 1.4584650993347168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14533120393753052,
      "rewards/margins": 0.11758776009082794,
      "rewards/rejected": -0.26291897892951965,
      "step": 1118
    },
    {
      "epoch": 1.7628053585500394,
      "grad_norm": 0.21861624717712402,
      "learning_rate": 1.7201584031820415e-07,
      "log_odds_chosen": 1.291368007659912,
      "log_odds_ratio": -0.25033318996429443,
      "logits/chosen": -0.4770206809043884,
      "logits/rejected": -1.1762526035308838,
      "logps/chosen": -1.356243371963501,
      "logps/rejected": -2.432551622390747,
      "loss": 1.407,
      "nll_loss": 1.3819990158081055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13562433421611786,
      "rewards/margins": 0.10763081908226013,
      "rewards/rejected": -0.24325516819953918,
      "step": 1119
    },
    {
      "epoch": 1.764381402679275,
      "grad_norm": 0.2299405336380005,
      "learning_rate": 1.6979239837012193e-07,
      "log_odds_chosen": 1.4253524541854858,
      "log_odds_ratio": -0.22961200773715973,
      "logits/chosen": -0.5115764737129211,
      "logits/rejected": -1.1207828521728516,
      "logps/chosen": -1.4428369998931885,
      "logps/rejected": -2.6688249111175537,
      "loss": 1.4714,
      "nll_loss": 1.4484328031539917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1442837119102478,
      "rewards/margins": 0.12259876728057861,
      "rewards/rejected": -0.2668824791908264,
      "step": 1120
    },
    {
      "epoch": 1.7659574468085106,
      "grad_norm": 0.2224302440881729,
      "learning_rate": 1.6758278276629655e-07,
      "log_odds_chosen": 1.4980851411819458,
      "log_odds_ratio": -0.20809093117713928,
      "logits/chosen": -0.4315674901008606,
      "logits/rejected": -1.1805915832519531,
      "logps/chosen": -1.5323741436004639,
      "logps/rejected": -2.848703622817993,
      "loss": 1.5601,
      "nll_loss": 1.5392621755599976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15323740243911743,
      "rewards/margins": 0.1316329687833786,
      "rewards/rejected": -0.2848703861236572,
      "step": 1121
    },
    {
      "epoch": 1.7675334909377463,
      "grad_norm": 0.23667281866073608,
      "learning_rate": 1.6538701019931867e-07,
      "log_odds_chosen": 1.5109301805496216,
      "log_odds_ratio": -0.2066952884197235,
      "logits/chosen": -0.6523215770721436,
      "logits/rejected": -1.149829626083374,
      "logps/chosen": -1.397404670715332,
      "logps/rejected": -2.6897897720336914,
      "loss": 1.4315,
      "nll_loss": 1.4108562469482422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1397404670715332,
      "rewards/margins": 0.1292385309934616,
      "rewards/rejected": -0.2689789831638336,
      "step": 1122
    },
    {
      "epoch": 1.7691095350669819,
      "grad_norm": 0.2427237629890442,
      "learning_rate": 1.6320509725720123e-07,
      "log_odds_chosen": 1.578816533088684,
      "log_odds_ratio": -0.19647452235221863,
      "logits/chosen": -0.5861450433731079,
      "logits/rejected": -1.2327039241790771,
      "logps/chosen": -1.4902857542037964,
      "logps/rejected": -2.873467206954956,
      "loss": 1.5239,
      "nll_loss": 1.5042532682418823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14902856945991516,
      "rewards/margins": 0.13831816613674164,
      "rewards/rejected": -0.287346750497818,
      "step": 1123
    },
    {
      "epoch": 1.7706855791962175,
      "grad_norm": 0.25164833664894104,
      "learning_rate": 1.6103706042325426e-07,
      "log_odds_chosen": 1.8435237407684326,
      "log_odds_ratio": -0.16308839619159698,
      "logits/chosen": -0.6450917720794678,
      "logits/rejected": -1.1081504821777344,
      "logps/chosen": -1.5246847867965698,
      "logps/rejected": -3.1681814193725586,
      "loss": 1.5444,
      "nll_loss": 1.5280615091323853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1524684876203537,
      "rewards/margins": 0.16434964537620544,
      "rewards/rejected": -0.31681814789772034,
      "step": 1124
    },
    {
      "epoch": 1.7722616233254531,
      "grad_norm": 0.26121556758880615,
      "learning_rate": 1.5888291607596127e-07,
      "log_odds_chosen": 1.791799783706665,
      "log_odds_ratio": -0.16217952966690063,
      "logits/chosen": -0.6861334443092346,
      "logits/rejected": -1.1726316213607788,
      "logps/chosen": -1.4166618585586548,
      "logps/rejected": -2.984755516052246,
      "loss": 1.4436,
      "nll_loss": 1.427383542060852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14166618883609772,
      "rewards/margins": 0.15680937469005585,
      "rewards/rejected": -0.29847556352615356,
      "step": 1125
    },
    {
      "epoch": 1.7738376674546887,
      "grad_norm": 0.25322824716567993,
      "learning_rate": 1.5674268048885275e-07,
      "log_odds_chosen": 1.5530320405960083,
      "log_odds_ratio": -0.20543813705444336,
      "logits/chosen": -0.5226525068283081,
      "logits/rejected": -1.11138916015625,
      "logps/chosen": -1.5070642232894897,
      "logps/rejected": -2.8732781410217285,
      "loss": 1.5449,
      "nll_loss": 1.5243852138519287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1507064253091812,
      "rewards/margins": 0.1366213858127594,
      "rewards/rejected": -0.2873277962207794,
      "step": 1126
    },
    {
      "epoch": 1.7754137115839244,
      "grad_norm": 0.24859434366226196,
      "learning_rate": 1.5461636983038685e-07,
      "log_odds_chosen": 1.436570405960083,
      "log_odds_ratio": -0.2238411009311676,
      "logits/chosen": -0.5067509412765503,
      "logits/rejected": -1.168760895729065,
      "logps/chosen": -1.5960397720336914,
      "logps/rejected": -2.8604254722595215,
      "loss": 1.5925,
      "nll_loss": 1.5701383352279663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15960398316383362,
      "rewards/margins": 0.12643857300281525,
      "rewards/rejected": -0.28604257106781006,
      "step": 1127
    },
    {
      "epoch": 1.77698975571316,
      "grad_norm": 0.28084903955459595,
      "learning_rate": 1.525040001638249e-07,
      "log_odds_chosen": 1.3962321281433105,
      "log_odds_ratio": -0.23591728508472443,
      "logits/chosen": -0.5788848996162415,
      "logits/rejected": -1.150484561920166,
      "logps/chosen": -1.4804142713546753,
      "logps/rejected": -2.690300464630127,
      "loss": 1.5038,
      "nll_loss": 1.480229377746582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14804142713546753,
      "rewards/margins": 0.12098865211009979,
      "rewards/rejected": -0.2690300941467285,
      "step": 1128
    },
    {
      "epoch": 1.7785657998423956,
      "grad_norm": 0.23581451177597046,
      "learning_rate": 1.5040558744711086e-07,
      "log_odds_chosen": 1.7294858694076538,
      "log_odds_ratio": -0.17035794258117676,
      "logits/chosen": -0.6185383796691895,
      "logits/rejected": -1.2803194522857666,
      "logps/chosen": -1.4648644924163818,
      "logps/rejected": -2.9831151962280273,
      "loss": 1.4935,
      "nll_loss": 1.4764363765716553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14648646116256714,
      "rewards/margins": 0.15182505548000336,
      "rewards/rejected": -0.2983115315437317,
      "step": 1129
    },
    {
      "epoch": 1.7801418439716312,
      "grad_norm": 0.232469841837883,
      "learning_rate": 1.4832114753275082e-07,
      "log_odds_chosen": 1.3254168033599854,
      "log_odds_ratio": -0.2541036605834961,
      "logits/chosen": -0.678654134273529,
      "logits/rejected": -1.1208058595657349,
      "logps/chosen": -1.4295200109481812,
      "logps/rejected": -2.5693106651306152,
      "loss": 1.4627,
      "nll_loss": 1.437256097793579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14295201003551483,
      "rewards/margins": 0.1139790415763855,
      "rewards/rejected": -0.25693103671073914,
      "step": 1130
    },
    {
      "epoch": 1.7817178881008668,
      "grad_norm": 0.26776939630508423,
      "learning_rate": 1.4625069616769215e-07,
      "log_odds_chosen": 1.5911612510681152,
      "log_odds_ratio": -0.18851245939731598,
      "logits/chosen": -0.6238042116165161,
      "logits/rejected": -1.1364049911499023,
      "logps/chosen": -1.4901176691055298,
      "logps/rejected": -2.878675699234009,
      "loss": 1.5118,
      "nll_loss": 1.4929159879684448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1490117609500885,
      "rewards/margins": 0.13885581493377686,
      "rewards/rejected": -0.28786760568618774,
      "step": 1131
    },
    {
      "epoch": 1.7832939322301025,
      "grad_norm": 0.2519194483757019,
      "learning_rate": 1.4419424899320643e-07,
      "log_odds_chosen": 1.6121854782104492,
      "log_odds_ratio": -0.19076931476593018,
      "logits/chosen": -0.5766335725784302,
      "logits/rejected": -1.2332581281661987,
      "logps/chosen": -1.562496542930603,
      "logps/rejected": -2.984877824783325,
      "loss": 1.5675,
      "nll_loss": 1.5483911037445068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15624965727329254,
      "rewards/margins": 0.14223815500736237,
      "rewards/rejected": -0.2984878122806549,
      "step": 1132
    },
    {
      "epoch": 1.784869976359338,
      "grad_norm": 0.24601571261882782,
      "learning_rate": 1.4215182154476945e-07,
      "log_odds_chosen": 1.5401935577392578,
      "log_odds_ratio": -0.20752079784870148,
      "logits/chosen": -0.5573534965515137,
      "logits/rejected": -1.2400035858154297,
      "logps/chosen": -1.4872345924377441,
      "logps/rejected": -2.8352136611938477,
      "loss": 1.5203,
      "nll_loss": 1.499539852142334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14872346818447113,
      "rewards/margins": 0.13479791581630707,
      "rewards/rejected": -0.2835213840007782,
      "step": 1133
    },
    {
      "epoch": 1.7864460204885737,
      "grad_norm": 0.23187468945980072,
      "learning_rate": 1.401234292519453e-07,
      "log_odds_chosen": 1.8193244934082031,
      "log_odds_ratio": -0.1579495221376419,
      "logits/chosen": -0.6030759811401367,
      "logits/rejected": -1.3250621557235718,
      "logps/chosen": -1.4479165077209473,
      "logps/rejected": -3.0471744537353516,
      "loss": 1.4771,
      "nll_loss": 1.461296558380127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14479166269302368,
      "rewards/margins": 0.15992580354213715,
      "rewards/rejected": -0.30471745133399963,
      "step": 1134
    },
    {
      "epoch": 1.7880220646178093,
      "grad_norm": 0.2558007538318634,
      "learning_rate": 1.3810908743826933e-07,
      "log_odds_chosen": 1.466287612915039,
      "log_odds_ratio": -0.21432740986347198,
      "logits/chosen": -0.6522049903869629,
      "logits/rejected": -1.1248016357421875,
      "logps/chosen": -1.4670310020446777,
      "logps/rejected": -2.7333500385284424,
      "loss": 1.499,
      "nll_loss": 1.4775538444519043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1467030942440033,
      "rewards/margins": 0.12663190066814423,
      "rewards/rejected": -0.2733350098133087,
      "step": 1135
    },
    {
      "epoch": 1.789598108747045,
      "grad_norm": 0.23109543323516846,
      "learning_rate": 1.3610881132113107e-07,
      "log_odds_chosen": 1.4306163787841797,
      "log_odds_ratio": -0.21694423258304596,
      "logits/chosen": -0.6143792867660522,
      "logits/rejected": -1.0567067861557007,
      "logps/chosen": -1.383036732673645,
      "logps/rejected": -2.6010324954986572,
      "loss": 1.4207,
      "nll_loss": 1.3990195989608765,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13830368220806122,
      "rewards/margins": 0.12179955840110779,
      "rewards/rejected": -0.2601032257080078,
      "step": 1136
    },
    {
      "epoch": 1.7911741528762806,
      "grad_norm": 0.24478048086166382,
      "learning_rate": 1.3412261601166131e-07,
      "log_odds_chosen": 1.3651950359344482,
      "log_odds_ratio": -0.23624052107334137,
      "logits/chosen": -0.40917930006980896,
      "logits/rejected": -1.1304564476013184,
      "logps/chosen": -1.3682565689086914,
      "logps/rejected": -2.5278658866882324,
      "loss": 1.4086,
      "nll_loss": 1.384965181350708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13682566583156586,
      "rewards/margins": 0.11596094071865082,
      "rewards/rejected": -0.25278663635253906,
      "step": 1137
    },
    {
      "epoch": 1.7927501970055162,
      "grad_norm": 0.24478507041931152,
      "learning_rate": 1.3215051651461706e-07,
      "log_odds_chosen": 1.6228561401367188,
      "log_odds_ratio": -0.1947271227836609,
      "logits/chosen": -0.5363482236862183,
      "logits/rejected": -1.220080018043518,
      "logps/chosen": -1.4001199007034302,
      "logps/rejected": -2.8073811531066895,
      "loss": 1.4314,
      "nll_loss": 1.4118925333023071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1400119960308075,
      "rewards/margins": 0.14072613418102264,
      "rewards/rejected": -0.28073811531066895,
      "step": 1138
    },
    {
      "epoch": 1.7943262411347518,
      "grad_norm": 0.2688179612159729,
      "learning_rate": 1.3019252772826872e-07,
      "log_odds_chosen": 1.6055681705474854,
      "log_odds_ratio": -0.18522927165031433,
      "logits/chosen": -0.5890136957168579,
      "logits/rejected": -1.2326722145080566,
      "logps/chosen": -1.4761970043182373,
      "logps/rejected": -2.8773746490478516,
      "loss": 1.5098,
      "nll_loss": 1.491325855255127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14761970937252045,
      "rewards/margins": 0.14011774957180023,
      "rewards/rejected": -0.2877374589443207,
      "step": 1139
    },
    {
      "epoch": 1.7959022852639874,
      "grad_norm": 0.2450924813747406,
      "learning_rate": 1.2824866444428527e-07,
      "log_odds_chosen": 1.5958898067474365,
      "log_odds_ratio": -0.22738602757453918,
      "logits/chosen": -0.5288405418395996,
      "logits/rejected": -1.1582632064819336,
      "logps/chosen": -1.5986201763153076,
      "logps/rejected": -3.0292088985443115,
      "loss": 1.6042,
      "nll_loss": 1.581416368484497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15986201167106628,
      "rewards/margins": 0.1430588811635971,
      "rewards/rejected": -0.3029209077358246,
      "step": 1140
    },
    {
      "epoch": 1.797478329393223,
      "grad_norm": 0.25950565934181213,
      "learning_rate": 1.2631894134762577e-07,
      "log_odds_chosen": 1.598397970199585,
      "log_odds_ratio": -0.19271260499954224,
      "logits/chosen": -0.571448802947998,
      "logits/rejected": -1.2156041860580444,
      "logps/chosen": -1.5011837482452393,
      "logps/rejected": -2.9058563709259033,
      "loss": 1.5121,
      "nll_loss": 1.4928737878799438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1501183658838272,
      "rewards/margins": 0.14046724140644073,
      "rewards/rejected": -0.29058563709259033,
      "step": 1141
    },
    {
      "epoch": 1.7990543735224587,
      "grad_norm": 0.22907140851020813,
      "learning_rate": 1.2440337301642668e-07,
      "log_odds_chosen": 1.6416165828704834,
      "log_odds_ratio": -0.18837302923202515,
      "logits/chosen": -0.6331337690353394,
      "logits/rejected": -1.1944806575775146,
      "logps/chosen": -1.4125367403030396,
      "logps/rejected": -2.840447187423706,
      "loss": 1.4605,
      "nll_loss": 1.441642165184021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14125367999076843,
      "rewards/margins": 0.1427910327911377,
      "rewards/rejected": -0.2840447425842285,
      "step": 1142
    },
    {
      "epoch": 1.8006304176516943,
      "grad_norm": 0.21542015671730042,
      "learning_rate": 1.2250197392189176e-07,
      "log_odds_chosen": 1.6471226215362549,
      "log_odds_ratio": -0.19488979876041412,
      "logits/chosen": -0.49330955743789673,
      "logits/rejected": -1.250149130821228,
      "logps/chosen": -1.3997342586517334,
      "logps/rejected": -2.8249361515045166,
      "loss": 1.4463,
      "nll_loss": 1.4268089532852173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13997341692447662,
      "rewards/margins": 0.1425202190876007,
      "rewards/rejected": -0.2824936509132385,
      "step": 1143
    },
    {
      "epoch": 1.80220646178093,
      "grad_norm": 0.27983951568603516,
      "learning_rate": 1.2061475842818335e-07,
      "log_odds_chosen": 1.6283080577850342,
      "log_odds_ratio": -0.19564102590084076,
      "logits/chosen": -0.5586127638816833,
      "logits/rejected": -0.9241060018539429,
      "logps/chosen": -1.4316761493682861,
      "logps/rejected": -2.848027229309082,
      "loss": 1.4553,
      "nll_loss": 1.4357397556304932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431676298379898,
      "rewards/margins": 0.14163509011268616,
      "rewards/rejected": -0.28480273485183716,
      "step": 1144
    },
    {
      "epoch": 1.8037825059101655,
      "grad_norm": 0.2472786009311676,
      "learning_rate": 1.1874174079231236e-07,
      "log_odds_chosen": 1.7731502056121826,
      "log_odds_ratio": -0.1726287454366684,
      "logits/chosen": -0.6281435489654541,
      "logits/rejected": -1.1600922346115112,
      "logps/chosen": -1.4704782962799072,
      "logps/rejected": -3.0359933376312256,
      "loss": 1.5094,
      "nll_loss": 1.492098093032837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14704781770706177,
      "rewards/margins": 0.15655149519443512,
      "rewards/rejected": -0.3035992980003357,
      "step": 1145
    },
    {
      "epoch": 1.8053585500394012,
      "grad_norm": 0.271118700504303,
      "learning_rate": 1.1688293516403258e-07,
      "log_odds_chosen": 1.699124813079834,
      "log_odds_ratio": -0.18273936212062836,
      "logits/chosen": -0.4603389799594879,
      "logits/rejected": -1.0809844732284546,
      "logps/chosen": -1.4392414093017578,
      "logps/rejected": -2.923978805541992,
      "loss": 1.4855,
      "nll_loss": 1.4672340154647827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14392416179180145,
      "rewards/margins": 0.14847372472286224,
      "rewards/rejected": -0.2923978865146637,
      "step": 1146
    },
    {
      "epoch": 1.8069345941686368,
      "grad_norm": 0.2350521832704544,
      "learning_rate": 1.1503835558573306e-07,
      "log_odds_chosen": 1.4446160793304443,
      "log_odds_ratio": -0.23039621114730835,
      "logits/chosen": -0.6084123849868774,
      "logits/rejected": -1.0470393896102905,
      "logps/chosen": -1.4448862075805664,
      "logps/rejected": -2.695429801940918,
      "loss": 1.4784,
      "nll_loss": 1.4553965330123901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1444886326789856,
      "rewards/margins": 0.12505437433719635,
      "rewards/rejected": -0.26954299211502075,
      "step": 1147
    },
    {
      "epoch": 1.8085106382978724,
      "grad_norm": 0.26984843611717224,
      "learning_rate": 1.1320801599233144e-07,
      "log_odds_chosen": 1.5045316219329834,
      "log_odds_ratio": -0.20273737609386444,
      "logits/chosen": -0.73576420545578,
      "logits/rejected": -1.2583810091018677,
      "logps/chosen": -1.4613394737243652,
      "logps/rejected": -2.7669265270233154,
      "loss": 1.5115,
      "nll_loss": 1.4911919832229614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14613395929336548,
      "rewards/margins": 0.13055871427059174,
      "rewards/rejected": -0.276692658662796,
      "step": 1148
    },
    {
      "epoch": 1.8100866824271078,
      "grad_norm": 0.2317042499780655,
      "learning_rate": 1.1139193021116877e-07,
      "log_odds_chosen": 1.6569287776947021,
      "log_odds_ratio": -0.18568651378154755,
      "logits/chosen": -0.5724750757217407,
      "logits/rejected": -1.0490769147872925,
      "logps/chosen": -1.4334256649017334,
      "logps/rejected": -2.8785037994384766,
      "loss": 1.48,
      "nll_loss": 1.461452603340149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14334256947040558,
      "rewards/margins": 0.14450781047344208,
      "rewards/rejected": -0.28785037994384766,
      "step": 1149
    },
    {
      "epoch": 1.8116627265563436,
      "grad_norm": 0.306060254573822,
      "learning_rate": 1.0959011196190582e-07,
      "log_odds_chosen": 1.196629285812378,
      "log_odds_ratio": -0.2705518305301666,
      "logits/chosen": -0.5663266181945801,
      "logits/rejected": -0.9415766000747681,
      "logps/chosen": -1.5206918716430664,
      "logps/rejected": -2.552351236343384,
      "loss": 1.5509,
      "nll_loss": 1.5237970352172852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15206919610500336,
      "rewards/margins": 0.10316591709852219,
      "rewards/rejected": -0.25523510575294495,
      "step": 1150
    },
    {
      "epoch": 1.813238770685579,
      "grad_norm": 0.24043582379817963,
      "learning_rate": 1.0780257485641908e-07,
      "log_odds_chosen": 1.4516184329986572,
      "log_odds_ratio": -0.21410569548606873,
      "logits/chosen": -0.5711349844932556,
      "logits/rejected": -1.1404423713684082,
      "logps/chosen": -1.5043845176696777,
      "logps/rejected": -2.7680187225341797,
      "loss": 1.529,
      "nll_loss": 1.5075910091400146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15043844282627106,
      "rewards/margins": 0.12636344134807587,
      "rewards/rejected": -0.2768018841743469,
      "step": 1151
    },
    {
      "epoch": 1.8148148148148149,
      "grad_norm": 0.24003872275352478,
      "learning_rate": 1.0602933239869738e-07,
      "log_odds_chosen": 1.6226463317871094,
      "log_odds_ratio": -0.19484736025333405,
      "logits/chosen": -0.5845081806182861,
      "logits/rejected": -1.2637017965316772,
      "logps/chosen": -1.4179508686065674,
      "logps/rejected": -2.828622579574585,
      "loss": 1.4439,
      "nll_loss": 1.4244590997695923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1417950838804245,
      "rewards/margins": 0.14106720685958862,
      "rewards/rejected": -0.28286227583885193,
      "step": 1152
    },
    {
      "epoch": 1.8163908589440503,
      "grad_norm": 0.23925338685512543,
      "learning_rate": 1.0427039798474035e-07,
      "log_odds_chosen": 1.6464208364486694,
      "log_odds_ratio": -0.18405862152576447,
      "logits/chosen": -0.5633742809295654,
      "logits/rejected": -1.26521635055542,
      "logps/chosen": -1.445478081703186,
      "logps/rejected": -2.881211757659912,
      "loss": 1.471,
      "nll_loss": 1.4526032209396362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14454780519008636,
      "rewards/margins": 0.1435733437538147,
      "rewards/rejected": -0.28812113404273987,
      "step": 1153
    },
    {
      "epoch": 1.8179669030732861,
      "grad_norm": 0.26578468084335327,
      "learning_rate": 1.0252578490245811e-07,
      "log_odds_chosen": 1.699029564857483,
      "log_odds_ratio": -0.18514785170555115,
      "logits/chosen": -0.6881905198097229,
      "logits/rejected": -1.1431119441986084,
      "logps/chosen": -1.5242648124694824,
      "logps/rejected": -3.031450033187866,
      "loss": 1.5448,
      "nll_loss": 1.5263149738311768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15242649614810944,
      "rewards/margins": 0.1507185399532318,
      "rewards/rejected": -0.30314502120018005,
      "step": 1154
    },
    {
      "epoch": 1.8195429472025215,
      "grad_norm": 0.275084525346756,
      "learning_rate": 1.007955063315682e-07,
      "log_odds_chosen": 1.3069223165512085,
      "log_odds_ratio": -0.24839037656784058,
      "logits/chosen": -0.6104614734649658,
      "logits/rejected": -1.193381667137146,
      "logps/chosen": -1.5222434997558594,
      "logps/rejected": -2.6599602699279785,
      "loss": 1.5462,
      "nll_loss": 1.5213173627853394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1522243618965149,
      "rewards/margins": 0.11377166956663132,
      "rewards/rejected": -0.2659960091114044,
      "step": 1155
    },
    {
      "epoch": 1.8211189913317574,
      "grad_norm": 0.2428610771894455,
      "learning_rate": 9.90795753434992e-08,
      "log_odds_chosen": 1.4637486934661865,
      "log_odds_ratio": -0.22659951448440552,
      "logits/chosen": -0.4747638702392578,
      "logits/rejected": -1.136139154434204,
      "logps/chosen": -1.5528843402862549,
      "logps/rejected": -2.8409459590911865,
      "loss": 1.5879,
      "nll_loss": 1.565219759941101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1552884429693222,
      "rewards/margins": 0.12880617380142212,
      "rewards/rejected": -0.28409460186958313,
      "step": 1156
    },
    {
      "epoch": 1.8226950354609928,
      "grad_norm": 0.24853679537773132,
      "learning_rate": 9.737800490129023e-08,
      "log_odds_chosen": 1.831891417503357,
      "log_odds_ratio": -0.16214488446712494,
      "logits/chosen": -0.6489849090576172,
      "logits/rejected": -1.2121307849884033,
      "logps/chosen": -1.4993021488189697,
      "logps/rejected": -3.123650312423706,
      "loss": 1.511,
      "nll_loss": 1.4947471618652344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1499302089214325,
      "rewards/margins": 0.16243483126163483,
      "rewards/rejected": -0.3123650550842285,
      "step": 1157
    },
    {
      "epoch": 1.8242710795902286,
      "grad_norm": 0.22928644716739655,
      "learning_rate": 9.569080785949335e-08,
      "log_odds_chosen": 1.47127366065979,
      "log_odds_ratio": -0.2207435518503189,
      "logits/chosen": -0.6057567596435547,
      "logits/rejected": -1.175225019454956,
      "logps/chosen": -1.3793108463287354,
      "logps/rejected": -2.6400327682495117,
      "loss": 1.4332,
      "nll_loss": 1.4110833406448364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13793107867240906,
      "rewards/margins": 0.12607219815254211,
      "rewards/rejected": -0.26400330662727356,
      "step": 1158
    },
    {
      "epoch": 1.825847123719464,
      "grad_norm": 0.24104629456996918,
      "learning_rate": 9.401799696407642e-08,
      "log_odds_chosen": 1.3871562480926514,
      "log_odds_ratio": -0.2319071888923645,
      "logits/chosen": -0.5985887050628662,
      "logits/rejected": -1.1420331001281738,
      "logps/chosen": -1.5273269414901733,
      "logps/rejected": -2.731901168823242,
      "loss": 1.5437,
      "nll_loss": 1.5204689502716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1527327001094818,
      "rewards/margins": 0.12045744061470032,
      "rewards/rejected": -0.27319014072418213,
      "step": 1159
    },
    {
      "epoch": 1.8274231678486998,
      "grad_norm": 0.2430720180273056,
      "learning_rate": 9.23595848523262e-08,
      "log_odds_chosen": 1.4418597221374512,
      "log_odds_ratio": -0.21728885173797607,
      "logits/chosen": -0.6833570003509521,
      "logits/rejected": -1.2130438089370728,
      "logps/chosen": -1.4670393466949463,
      "logps/rejected": -2.7137012481689453,
      "loss": 1.4796,
      "nll_loss": 1.457832932472229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14670394361019135,
      "rewards/margins": 0.12466619163751602,
      "rewards/rejected": -0.2713701128959656,
      "step": 1160
    },
    {
      "epoch": 1.8289992119779352,
      "grad_norm": 0.24411030113697052,
      "learning_rate": 9.071558405275426e-08,
      "log_odds_chosen": 1.5784224271774292,
      "log_odds_ratio": -0.2024582028388977,
      "logits/chosen": -0.6053391695022583,
      "logits/rejected": -1.2541894912719727,
      "logps/chosen": -1.485365867614746,
      "logps/rejected": -2.8712594509124756,
      "loss": 1.5091,
      "nll_loss": 1.4888719320297241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1485365927219391,
      "rewards/margins": 0.13858935236930847,
      "rewards/rejected": -0.28712594509124756,
      "step": 1161
    },
    {
      "epoch": 1.830575256107171,
      "grad_norm": 0.23016080260276794,
      "learning_rate": 8.90860069850019e-08,
      "log_odds_chosen": 1.6500474214553833,
      "log_odds_ratio": -0.1925586760044098,
      "logits/chosen": -0.5854189991950989,
      "logits/rejected": -1.2441606521606445,
      "logps/chosen": -1.4156028032302856,
      "logps/rejected": -2.84859037399292,
      "loss": 1.4316,
      "nll_loss": 1.4123188257217407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14156028628349304,
      "rewards/margins": 0.14329877495765686,
      "rewards/rejected": -0.2848590612411499,
      "step": 1162
    },
    {
      "epoch": 1.8321513002364065,
      "grad_norm": 0.22886443138122559,
      "learning_rate": 8.747086595974518e-08,
      "log_odds_chosen": 1.5232652425765991,
      "log_odds_ratio": -0.20012113451957703,
      "logits/chosen": -0.5851576924324036,
      "logits/rejected": -1.3232672214508057,
      "logps/chosen": -1.5682944059371948,
      "logps/rejected": -2.911965847015381,
      "loss": 1.574,
      "nll_loss": 1.5540049076080322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15682943165302277,
      "rewards/margins": 0.13436715304851532,
      "rewards/rejected": -0.2911965847015381,
      "step": 1163
    },
    {
      "epoch": 1.8337273443656423,
      "grad_norm": 0.25055304169654846,
      "learning_rate": 8.58701731786029e-08,
      "log_odds_chosen": 1.5959322452545166,
      "log_odds_ratio": -0.191603422164917,
      "logits/chosen": -0.5774615406990051,
      "logits/rejected": -1.1535186767578125,
      "logps/chosen": -1.5059378147125244,
      "logps/rejected": -2.9071922302246094,
      "loss": 1.5317,
      "nll_loss": 1.512511134147644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15059378743171692,
      "rewards/margins": 0.14012545347213745,
      "rewards/rejected": -0.29071924090385437,
      "step": 1164
    },
    {
      "epoch": 1.8353033884948777,
      "grad_norm": 0.2536662518978119,
      "learning_rate": 8.428394073404477e-08,
      "log_odds_chosen": 1.468266248703003,
      "log_odds_ratio": -0.21772223711013794,
      "logits/chosen": -0.5999587774276733,
      "logits/rejected": -1.230496883392334,
      "logps/chosen": -1.5414035320281982,
      "logps/rejected": -2.8307065963745117,
      "loss": 1.5742,
      "nll_loss": 1.552423357963562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15414035320281982,
      "rewards/margins": 0.12893033027648926,
      "rewards/rejected": -0.2830706834793091,
      "step": 1165
    },
    {
      "epoch": 1.8368794326241136,
      "grad_norm": 0.25595852732658386,
      "learning_rate": 8.271218060929919e-08,
      "log_odds_chosen": 1.4480369091033936,
      "log_odds_ratio": -0.2151479870080948,
      "logits/chosen": -0.5544027090072632,
      "logits/rejected": -1.1824607849121094,
      "logps/chosen": -1.4294812679290771,
      "logps/rejected": -2.670212745666504,
      "loss": 1.4636,
      "nll_loss": 1.4421330690383911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14294812083244324,
      "rewards/margins": 0.12407318502664566,
      "rewards/rejected": -0.2670213282108307,
      "step": 1166
    },
    {
      "epoch": 1.838455476753349,
      "grad_norm": 0.24518164992332458,
      "learning_rate": 8.115490467826291e-08,
      "log_odds_chosen": 1.726238489151001,
      "log_odds_ratio": -0.17394882440567017,
      "logits/chosen": -0.6498931646347046,
      "logits/rejected": -1.2249716520309448,
      "logps/chosen": -1.4331926107406616,
      "logps/rejected": -2.9379892349243164,
      "loss": 1.4448,
      "nll_loss": 1.4273598194122314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433192491531372,
      "rewards/margins": 0.15047965943813324,
      "rewards/rejected": -0.29379895329475403,
      "step": 1167
    },
    {
      "epoch": 1.8400315208825848,
      "grad_norm": 0.24167285859584808,
      "learning_rate": 7.961212470541245e-08,
      "log_odds_chosen": 1.5971952676773071,
      "log_odds_ratio": -0.19045525789260864,
      "logits/chosen": -0.4642918109893799,
      "logits/rejected": -1.1595354080200195,
      "logps/chosen": -1.444483757019043,
      "logps/rejected": -2.8355603218078613,
      "loss": 1.4687,
      "nll_loss": 1.4496995210647583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14444836974143982,
      "rewards/margins": 0.13910764455795288,
      "rewards/rejected": -0.2835560441017151,
      "step": 1168
    },
    {
      "epoch": 1.8416075650118202,
      "grad_norm": 0.24868352711200714,
      "learning_rate": 7.808385234571302e-08,
      "log_odds_chosen": 1.3600199222564697,
      "log_odds_ratio": -0.23851671814918518,
      "logits/chosen": -0.6629577875137329,
      "logits/rejected": -1.1663341522216797,
      "logps/chosen": -1.5568474531173706,
      "logps/rejected": -2.7476024627685547,
      "loss": 1.567,
      "nll_loss": 1.5431342124938965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15568475425243378,
      "rewards/margins": 0.11907552182674408,
      "rewards/rejected": -0.27476027607917786,
      "step": 1169
    },
    {
      "epoch": 1.843183609141056,
      "grad_norm": 0.23921307921409607,
      "learning_rate": 7.657009914453239e-08,
      "log_odds_chosen": 1.4671757221221924,
      "log_odds_ratio": -0.21867676079273224,
      "logits/chosen": -0.5650777220726013,
      "logits/rejected": -1.1829814910888672,
      "logps/chosen": -1.3992735147476196,
      "logps/rejected": -2.656461715698242,
      "loss": 1.4407,
      "nll_loss": 1.4187861680984497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13992735743522644,
      "rewards/margins": 0.12571880221366882,
      "rewards/rejected": -0.26564615964889526,
      "step": 1170
    },
    {
      "epoch": 1.8447596532702915,
      "grad_norm": 0.24403928220272064,
      "learning_rate": 7.507087653755318e-08,
      "log_odds_chosen": 1.9593538045883179,
      "log_odds_ratio": -0.14218644797801971,
      "logits/chosen": -0.6280905604362488,
      "logits/rejected": -1.2711557149887085,
      "logps/chosen": -1.448213815689087,
      "logps/rejected": -3.1796483993530273,
      "loss": 1.4774,
      "nll_loss": 1.463151216506958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1448213756084442,
      "rewards/margins": 0.1731434315443039,
      "rewards/rejected": -0.3179648220539093,
      "step": 1171
    },
    {
      "epoch": 1.8463356973995273,
      "grad_norm": 0.2433456927537918,
      "learning_rate": 7.358619585068582e-08,
      "log_odds_chosen": 1.3863154649734497,
      "log_odds_ratio": -0.24080650508403778,
      "logits/chosen": -0.5769352912902832,
      "logits/rejected": -1.0970388650894165,
      "logps/chosen": -1.323519229888916,
      "logps/rejected": -2.485218048095703,
      "loss": 1.3959,
      "nll_loss": 1.3718676567077637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13235193490982056,
      "rewards/margins": 0.11616986989974976,
      "rewards/rejected": -0.24852177500724792,
      "step": 1172
    },
    {
      "epoch": 1.8479117415287627,
      "grad_norm": 0.28372782468795776,
      "learning_rate": 7.21160682999844e-08,
      "log_odds_chosen": 1.1240544319152832,
      "log_odds_ratio": -0.2875262200832367,
      "logits/chosen": -0.5592198967933655,
      "logits/rejected": -0.977047324180603,
      "logps/chosen": -1.54170823097229,
      "logps/rejected": -2.5115842819213867,
      "loss": 1.558,
      "nll_loss": 1.5292651653289795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15417082607746124,
      "rewards/margins": 0.09698756039142609,
      "rewards/rejected": -0.2511584162712097,
      "step": 1173
    },
    {
      "epoch": 1.8494877856579985,
      "grad_norm": 0.2509617209434509,
      "learning_rate": 7.06605049915594e-08,
      "log_odds_chosen": 1.4860433340072632,
      "log_odds_ratio": -0.2117815464735031,
      "logits/chosen": -0.6291943788528442,
      "logits/rejected": -1.1202411651611328,
      "logps/chosen": -1.4575505256652832,
      "logps/rejected": -2.7478044033050537,
      "loss": 1.4948,
      "nll_loss": 1.4735780954360962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14575505256652832,
      "rewards/margins": 0.1290254145860672,
      "rewards/rejected": -0.2747804820537567,
      "step": 1174
    },
    {
      "epoch": 1.851063829787234,
      "grad_norm": 0.23649224638938904,
      "learning_rate": 6.92195169214973e-08,
      "log_odds_chosen": 1.544675350189209,
      "log_odds_ratio": -0.19638288021087646,
      "logits/chosen": -0.6275919675827026,
      "logits/rejected": -1.2005350589752197,
      "logps/chosen": -1.4502806663513184,
      "logps/rejected": -2.7880823612213135,
      "loss": 1.4787,
      "nll_loss": 1.4590308666229248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14502806961536407,
      "rewards/margins": 0.13378016650676727,
      "rewards/rejected": -0.27880823612213135,
      "step": 1175
    },
    {
      "epoch": 1.8526398739164698,
      "grad_norm": 0.24147120118141174,
      "learning_rate": 6.779311497577401e-08,
      "log_odds_chosen": 1.496665358543396,
      "log_odds_ratio": -0.2222577929496765,
      "logits/chosen": -0.6691316962242126,
      "logits/rejected": -1.0907988548278809,
      "logps/chosen": -1.4839872121810913,
      "logps/rejected": -2.78623628616333,
      "loss": 1.512,
      "nll_loss": 1.489769458770752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1483987271785736,
      "rewards/margins": 0.13022488355636597,
      "rewards/rejected": -0.2786236107349396,
      "step": 1176
    },
    {
      "epoch": 1.8542159180457052,
      "grad_norm": 0.29087069630622864,
      "learning_rate": 6.638130993017532e-08,
      "log_odds_chosen": 1.6382797956466675,
      "log_odds_ratio": -0.17910565435886383,
      "logits/chosen": -0.6125583052635193,
      "logits/rejected": -1.1643942594528198,
      "logps/chosen": -1.461418628692627,
      "logps/rejected": -2.892716407775879,
      "loss": 1.4864,
      "nll_loss": 1.4684547185897827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461418718099594,
      "rewards/margins": 0.14312976598739624,
      "rewards/rejected": -0.28927162289619446,
      "step": 1177
    },
    {
      "epoch": 1.855791962174941,
      "grad_norm": 0.2468017339706421,
      "learning_rate": 6.49841124502144e-08,
      "log_odds_chosen": 1.4363291263580322,
      "log_odds_ratio": -0.2296268194913864,
      "logits/chosen": -0.6099180579185486,
      "logits/rejected": -1.1326531171798706,
      "logps/chosen": -1.4900354146957397,
      "logps/rejected": -2.7417221069335938,
      "loss": 1.5181,
      "nll_loss": 1.4950907230377197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1490035504102707,
      "rewards/margins": 0.12516868114471436,
      "rewards/rejected": -0.27417224645614624,
      "step": 1178
    },
    {
      "epoch": 1.8573680063041764,
      "grad_norm": 0.2511499524116516,
      "learning_rate": 6.360153309104998e-08,
      "log_odds_chosen": 1.747485876083374,
      "log_odds_ratio": -0.19056683778762817,
      "logits/chosen": -0.7249822616577148,
      "logits/rejected": -1.214372992515564,
      "logps/chosen": -1.5202802419662476,
      "logps/rejected": -3.0783348083496094,
      "loss": 1.5456,
      "nll_loss": 1.5265775918960571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15202802419662476,
      "rewards/margins": 0.15580545365810394,
      "rewards/rejected": -0.3078334629535675,
      "step": 1179
    },
    {
      "epoch": 1.8589440504334123,
      "grad_norm": 0.23356066644191742,
      "learning_rate": 6.223358229740916e-08,
      "log_odds_chosen": 1.571268081665039,
      "log_odds_ratio": -0.19901350140571594,
      "logits/chosen": -0.5894266963005066,
      "logits/rejected": -1.1405231952667236,
      "logps/chosen": -1.5331056118011475,
      "logps/rejected": -2.917210102081299,
      "loss": 1.5401,
      "nll_loss": 1.5201832056045532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15331056714057922,
      "rewards/margins": 0.13841044902801514,
      "rewards/rejected": -0.29172104597091675,
      "step": 1180
    },
    {
      "epoch": 1.8605200945626477,
      "grad_norm": 0.35478755831718445,
      "learning_rate": 6.088027040350674e-08,
      "log_odds_chosen": 1.583790898323059,
      "log_odds_ratio": -0.19917601346969604,
      "logits/chosen": -0.6101710200309753,
      "logits/rejected": -1.164192795753479,
      "logps/chosen": -1.51638925075531,
      "logps/rejected": -2.9098730087280273,
      "loss": 1.5398,
      "nll_loss": 1.5199005603790283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.151638925075531,
      "rewards/margins": 0.1393483579158783,
      "rewards/rejected": -0.2909872829914093,
      "step": 1181
    },
    {
      "epoch": 1.8620961386918835,
      "grad_norm": 0.2506412863731384,
      "learning_rate": 5.9541607632968003e-08,
      "log_odds_chosen": 1.4895820617675781,
      "log_odds_ratio": -0.21182693541049957,
      "logits/chosen": -0.4843617081642151,
      "logits/rejected": -1.1084225177764893,
      "logps/chosen": -1.5208141803741455,
      "logps/rejected": -2.8216378688812256,
      "loss": 1.5331,
      "nll_loss": 1.511916160583496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1520814299583435,
      "rewards/margins": 0.13008235394954681,
      "rewards/rejected": -0.2821637988090515,
      "step": 1182
    },
    {
      "epoch": 1.863672182821119,
      "grad_norm": 0.2515678107738495,
      "learning_rate": 5.821760409875054e-08,
      "log_odds_chosen": 1.5013067722320557,
      "log_odds_ratio": -0.20741674304008484,
      "logits/chosen": -0.6004853248596191,
      "logits/rejected": -1.1596564054489136,
      "logps/chosen": -1.4958996772766113,
      "logps/rejected": -2.8069205284118652,
      "loss": 1.533,
      "nll_loss": 1.5122641324996948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14958997070789337,
      "rewards/margins": 0.13110211491584778,
      "rewards/rejected": -0.28069210052490234,
      "step": 1183
    },
    {
      "epoch": 1.8652482269503547,
      "grad_norm": 0.26463139057159424,
      "learning_rate": 5.690826980306851e-08,
      "log_odds_chosen": 1.4932578802108765,
      "log_odds_ratio": -0.21760308742523193,
      "logits/chosen": -0.6615166068077087,
      "logits/rejected": -1.0769801139831543,
      "logps/chosen": -1.4491699934005737,
      "logps/rejected": -2.7450881004333496,
      "loss": 1.4864,
      "nll_loss": 1.4646110534667969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14491701126098633,
      "rewards/margins": 0.12959182262420654,
      "rewards/rejected": -0.27450883388519287,
      "step": 1184
    },
    {
      "epoch": 1.8668242710795901,
      "grad_norm": 0.23942124843597412,
      "learning_rate": 5.56136146373174e-08,
      "log_odds_chosen": 1.7274751663208008,
      "log_odds_ratio": -0.18094345927238464,
      "logits/chosen": -0.6925673484802246,
      "logits/rejected": -1.2399039268493652,
      "logps/chosen": -1.5343858003616333,
      "logps/rejected": -3.068229913711548,
      "loss": 1.5529,
      "nll_loss": 1.5348503589630127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15343856811523438,
      "rewards/margins": 0.15338441729545593,
      "rewards/rejected": -0.3068230152130127,
      "step": 1185
    },
    {
      "epoch": 1.868400315208826,
      "grad_norm": 0.24394792318344116,
      "learning_rate": 5.4333648381998276e-08,
      "log_odds_chosen": 1.4242119789123535,
      "log_odds_ratio": -0.22015684843063354,
      "logits/chosen": -0.5924320816993713,
      "logits/rejected": -1.185723066329956,
      "logps/chosen": -1.5568714141845703,
      "logps/rejected": -2.8050525188446045,
      "loss": 1.5785,
      "nll_loss": 1.5564357042312622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1556871384382248,
      "rewards/margins": 0.12481814622879028,
      "rewards/rejected": -0.28050529956817627,
      "step": 1186
    },
    {
      "epoch": 1.8699763593380614,
      "grad_norm": 0.2381693422794342,
      "learning_rate": 5.306838070664521e-08,
      "log_odds_chosen": 1.1225930452346802,
      "log_odds_ratio": -0.2886594533920288,
      "logits/chosen": -0.6953135132789612,
      "logits/rejected": -1.1297504901885986,
      "logps/chosen": -1.470181941986084,
      "logps/rejected": -2.4246957302093506,
      "loss": 1.5198,
      "nll_loss": 1.4909061193466187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470182090997696,
      "rewards/margins": 0.09545139223337173,
      "rewards/rejected": -0.24246957898139954,
      "step": 1187
    },
    {
      "epoch": 1.8715524034672972,
      "grad_norm": 0.23353199660778046,
      "learning_rate": 5.181782116975109e-08,
      "log_odds_chosen": 1.6790249347686768,
      "log_odds_ratio": -0.18177203834056854,
      "logits/chosen": -0.6328272819519043,
      "logits/rejected": -1.2649166584014893,
      "logps/chosen": -1.450413465499878,
      "logps/rejected": -2.9208645820617676,
      "loss": 1.473,
      "nll_loss": 1.4547823667526245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1450413465499878,
      "rewards/margins": 0.14704512059688568,
      "rewards/rejected": -0.2920864522457123,
      "step": 1188
    },
    {
      "epoch": 1.8731284475965326,
      "grad_norm": 0.22058410942554474,
      "learning_rate": 5.058197921869567e-08,
      "log_odds_chosen": 1.46243155002594,
      "log_odds_ratio": -0.21322888135910034,
      "logits/chosen": -0.4883938133716583,
      "logits/rejected": -1.1047899723052979,
      "logps/chosen": -1.3906294107437134,
      "logps/rejected": -2.642023801803589,
      "loss": 1.4323,
      "nll_loss": 1.410979986190796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13906295597553253,
      "rewards/margins": 0.12513943016529083,
      "rewards/rejected": -0.26420238614082336,
      "step": 1189
    },
    {
      "epoch": 1.8747044917257685,
      "grad_norm": 0.24931298196315765,
      "learning_rate": 4.9360864189674775e-08,
      "log_odds_chosen": 1.6103966236114502,
      "log_odds_ratio": -0.19589650630950928,
      "logits/chosen": -0.6147593855857849,
      "logits/rejected": -1.1486550569534302,
      "logps/chosen": -1.466259241104126,
      "logps/rejected": -2.872734308242798,
      "loss": 1.4841,
      "nll_loss": 1.464479923248291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14662593603134155,
      "rewards/margins": 0.1406475007534027,
      "rewards/rejected": -0.2872734069824219,
      "step": 1190
    },
    {
      "epoch": 1.8762805358550039,
      "grad_norm": 0.23831091821193695,
      "learning_rate": 4.8154485307629226e-08,
      "log_odds_chosen": 1.8177056312561035,
      "log_odds_ratio": -0.16142575442790985,
      "logits/chosen": -0.6512100100517273,
      "logits/rejected": -1.2102559804916382,
      "logps/chosen": -1.388414978981018,
      "logps/rejected": -2.969661235809326,
      "loss": 1.4357,
      "nll_loss": 1.4195630550384521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13884149491786957,
      "rewards/margins": 0.158124640583992,
      "rewards/rejected": -0.2969661355018616,
      "step": 1191
    },
    {
      "epoch": 1.8778565799842397,
      "grad_norm": 0.23200058937072754,
      "learning_rate": 4.6962851686175306e-08,
      "log_odds_chosen": 1.244652271270752,
      "log_odds_ratio": -0.28540876507759094,
      "logits/chosen": -0.4811091423034668,
      "logits/rejected": -0.9004203677177429,
      "logps/chosen": -1.3916617631912231,
      "logps/rejected": -2.4556615352630615,
      "loss": 1.4397,
      "nll_loss": 1.4111251831054688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1391661912202835,
      "rewards/margins": 0.10639996081590652,
      "rewards/rejected": -0.24556614458560944,
      "step": 1192
    },
    {
      "epoch": 1.8794326241134751,
      "grad_norm": 0.2541332542896271,
      "learning_rate": 4.578597232753556e-08,
      "log_odds_chosen": 1.690503478050232,
      "log_odds_ratio": -0.17102432250976562,
      "logits/chosen": -0.5747317671775818,
      "logits/rejected": -1.1264772415161133,
      "logps/chosen": -1.5241562128067017,
      "logps/rejected": -3.0175442695617676,
      "loss": 1.5318,
      "nll_loss": 1.5147125720977783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15241561830043793,
      "rewards/margins": 0.14933884143829346,
      "rewards/rejected": -0.3017544746398926,
      "step": 1193
    },
    {
      "epoch": 1.881008668242711,
      "grad_norm": 0.24023234844207764,
      "learning_rate": 4.462385612247166e-08,
      "log_odds_chosen": 1.648472547531128,
      "log_odds_ratio": -0.18910396099090576,
      "logits/chosen": -0.590675413608551,
      "logits/rejected": -1.2631388902664185,
      "logps/chosen": -1.5802849531173706,
      "logps/rejected": -3.048142433166504,
      "loss": 1.585,
      "nll_loss": 1.56611967086792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1580284982919693,
      "rewards/margins": 0.14678576588630676,
      "rewards/rejected": -0.30481430888175964,
      "step": 1194
    },
    {
      "epoch": 1.8825847123719464,
      "grad_norm": 0.24550676345825195,
      "learning_rate": 4.34765118502165e-08,
      "log_odds_chosen": 1.5572752952575684,
      "log_odds_ratio": -0.19784438610076904,
      "logits/chosen": -0.5468028783798218,
      "logits/rejected": -1.2419499158859253,
      "logps/chosen": -1.4174818992614746,
      "logps/rejected": -2.763695001602173,
      "loss": 1.4585,
      "nll_loss": 1.4387515783309937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14174818992614746,
      "rewards/margins": 0.1346212923526764,
      "rewards/rejected": -0.27636948227882385,
      "step": 1195
    },
    {
      "epoch": 1.8841607565011822,
      "grad_norm": 0.22571580111980438,
      "learning_rate": 4.234394817840803e-08,
      "log_odds_chosen": 1.5554050207138062,
      "log_odds_ratio": -0.21167273819446564,
      "logits/chosen": -0.5912613868713379,
      "logits/rejected": -1.0462042093276978,
      "logps/chosen": -1.4380340576171875,
      "logps/rejected": -2.789679527282715,
      "loss": 1.4832,
      "nll_loss": 1.4619909524917603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1438034027814865,
      "rewards/margins": 0.1351645439863205,
      "rewards/rejected": -0.278967946767807,
      "step": 1196
    },
    {
      "epoch": 1.8857368006304176,
      "grad_norm": 0.2732309401035309,
      "learning_rate": 4.122617366302395e-08,
      "log_odds_chosen": 1.6063988208770752,
      "log_odds_ratio": -0.19011656939983368,
      "logits/chosen": -0.6127675175666809,
      "logits/rejected": -1.1821507215499878,
      "logps/chosen": -1.516514778137207,
      "logps/rejected": -2.930086374282837,
      "loss": 1.5412,
      "nll_loss": 1.522180199623108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15165148675441742,
      "rewards/margins": 0.1413571536540985,
      "rewards/rejected": -0.29300862550735474,
      "step": 1197
    },
    {
      "epoch": 1.8873128447596532,
      "grad_norm": 0.22373974323272705,
      "learning_rate": 4.0123196748316436e-08,
      "log_odds_chosen": 1.344637393951416,
      "log_odds_ratio": -0.2410965859889984,
      "logits/chosen": -0.6006601452827454,
      "logits/rejected": -1.2293412685394287,
      "logps/chosen": -1.3876357078552246,
      "logps/rejected": -2.5299019813537598,
      "loss": 1.4256,
      "nll_loss": 1.40147864818573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13876357674598694,
      "rewards/margins": 0.11422662436962128,
      "rewards/rejected": -0.252990186214447,
      "step": 1198
    },
    {
      "epoch": 1.8888888888888888,
      "grad_norm": 0.24617208540439606,
      "learning_rate": 3.903502576674933e-08,
      "log_odds_chosen": 1.478872537612915,
      "log_odds_ratio": -0.2235289067029953,
      "logits/chosen": -0.6237600445747375,
      "logits/rejected": -1.2239437103271484,
      "logps/chosen": -1.5263254642486572,
      "logps/rejected": -2.8254432678222656,
      "loss": 1.5517,
      "nll_loss": 1.5293018817901611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15263253450393677,
      "rewards/margins": 0.12991181015968323,
      "rewards/rejected": -0.28254434466362,
      "step": 1199
    },
    {
      "epoch": 1.8904649330181245,
      "grad_norm": 0.24981695413589478,
      "learning_rate": 3.796166893893504e-08,
      "log_odds_chosen": 1.3706992864608765,
      "log_odds_ratio": -0.2318522185087204,
      "logits/chosen": -0.4953332841396332,
      "logits/rejected": -1.217788577079773,
      "logps/chosen": -1.5913227796554565,
      "logps/rejected": -2.796452522277832,
      "loss": 1.6045,
      "nll_loss": 1.5812830924987793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15913227200508118,
      "rewards/margins": 0.12051299214363098,
      "rewards/rejected": -0.27964529395103455,
      "step": 1200
    },
    {
      "epoch": 1.89204097714736,
      "grad_norm": 0.2388419359922409,
      "learning_rate": 3.690313437357151e-08,
      "log_odds_chosen": 1.5016231536865234,
      "log_odds_ratio": -0.2110929787158966,
      "logits/chosen": -0.5454793572425842,
      "logits/rejected": -1.1230204105377197,
      "logps/chosen": -1.40786612033844,
      "logps/rejected": -2.7019646167755127,
      "loss": 1.4461,
      "nll_loss": 1.4249509572982788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14078661799430847,
      "rewards/margins": 0.12940986454486847,
      "rewards/rejected": -0.27019649744033813,
      "step": 1201
    },
    {
      "epoch": 1.8936170212765957,
      "grad_norm": 0.24973392486572266,
      "learning_rate": 3.585943006738179e-08,
      "log_odds_chosen": 1.4153707027435303,
      "log_odds_ratio": -0.22754548490047455,
      "logits/chosen": -0.5610604286193848,
      "logits/rejected": -1.0250792503356934,
      "logps/chosen": -1.5144851207733154,
      "logps/rejected": -2.7494850158691406,
      "loss": 1.552,
      "nll_loss": 1.5292463302612305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15144851803779602,
      "rewards/margins": 0.12350001186132431,
      "rewards/rejected": -0.2749485373497009,
      "step": 1202
    },
    {
      "epoch": 1.8951930654058313,
      "grad_norm": 0.25958681106567383,
      "learning_rate": 3.483056390505346e-08,
      "log_odds_chosen": 1.5287858247756958,
      "log_odds_ratio": -0.20268514752388,
      "logits/chosen": -0.6628624796867371,
      "logits/rejected": -1.2608020305633545,
      "logps/chosen": -1.468293309211731,
      "logps/rejected": -2.7976088523864746,
      "loss": 1.4854,
      "nll_loss": 1.4650827646255493,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14682933688163757,
      "rewards/margins": 0.13293156027793884,
      "rewards/rejected": -0.2797608971595764,
      "step": 1203
    },
    {
      "epoch": 1.896769109535067,
      "grad_norm": 0.24871715903282166,
      "learning_rate": 3.381654365917863e-08,
      "log_odds_chosen": 1.713343620300293,
      "log_odds_ratio": -0.17234216630458832,
      "logits/chosen": -0.608275830745697,
      "logits/rejected": -1.3599003553390503,
      "logps/chosen": -1.579482913017273,
      "logps/rejected": -3.1058247089385986,
      "loss": 1.5908,
      "nll_loss": 1.5735443830490112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15794828534126282,
      "rewards/margins": 0.15263423323631287,
      "rewards/rejected": -0.3105825185775757,
      "step": 1204
    },
    {
      "epoch": 1.8983451536643026,
      "grad_norm": 0.22707518935203552,
      "learning_rate": 3.281737699019627e-08,
      "log_odds_chosen": 1.4300590753555298,
      "log_odds_ratio": -0.21901832520961761,
      "logits/chosen": -0.537946343421936,
      "logits/rejected": -1.10988187789917,
      "logps/chosen": -1.3983299732208252,
      "logps/rejected": -2.622438907623291,
      "loss": 1.4521,
      "nll_loss": 1.4301915168762207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.139833003282547,
      "rewards/margins": 0.12241089344024658,
      "rewards/rejected": -0.26224392652511597,
      "step": 1205
    },
    {
      "epoch": 1.8999211977935382,
      "grad_norm": 0.2379302978515625,
      "learning_rate": 3.1833071446333295e-08,
      "log_odds_chosen": 1.423020839691162,
      "log_odds_ratio": -0.22871743142604828,
      "logits/chosen": -0.5865651965141296,
      "logits/rejected": -1.0209757089614868,
      "logps/chosen": -1.4931223392486572,
      "logps/rejected": -2.7328810691833496,
      "loss": 1.5172,
      "nll_loss": 1.4942941665649414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14931224286556244,
      "rewards/margins": 0.12397588789463043,
      "rewards/rejected": -0.2732881009578705,
      "step": 1206
    },
    {
      "epoch": 1.9014972419227738,
      "grad_norm": 0.23153118789196014,
      "learning_rate": 3.0863634463548005e-08,
      "log_odds_chosen": 1.6894265413284302,
      "log_odds_ratio": -0.18151816725730896,
      "logits/chosen": -0.5200134515762329,
      "logits/rejected": -1.1972780227661133,
      "logps/chosen": -1.4216879606246948,
      "logps/rejected": -2.8913066387176514,
      "loss": 1.4591,
      "nll_loss": 1.4409376382827759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.142168790102005,
      "rewards/margins": 0.14696188271045685,
      "rewards/rejected": -0.28913065791130066,
      "step": 1207
    },
    {
      "epoch": 1.9030732860520094,
      "grad_norm": 0.23932644724845886,
      "learning_rate": 2.9909073365473435e-08,
      "log_odds_chosen": 1.8210878372192383,
      "log_odds_ratio": -0.16425687074661255,
      "logits/chosen": -0.5752071142196655,
      "logits/rejected": -1.1916686296463013,
      "logps/chosen": -1.4713082313537598,
      "logps/rejected": -3.0809309482574463,
      "loss": 1.4946,
      "nll_loss": 1.4781354665756226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1471308171749115,
      "rewards/margins": 0.16096223890781403,
      "rewards/rejected": -0.3080930709838867,
      "step": 1208
    },
    {
      "epoch": 1.904649330181245,
      "grad_norm": 0.256862610578537,
      "learning_rate": 2.896939536336296e-08,
      "log_odds_chosen": 1.3558437824249268,
      "log_odds_ratio": -0.23434004187583923,
      "logits/chosen": -0.5553317070007324,
      "logits/rejected": -1.019963026046753,
      "logps/chosen": -1.4894185066223145,
      "logps/rejected": -2.6589369773864746,
      "loss": 1.5182,
      "nll_loss": 1.4947596788406372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14894185960292816,
      "rewards/margins": 0.1169518381357193,
      "rewards/rejected": -0.2658936679363251,
      "step": 1209
    },
    {
      "epoch": 1.9062253743104807,
      "grad_norm": 0.2510707378387451,
      "learning_rate": 2.8044607556035216e-08,
      "log_odds_chosen": 1.7909249067306519,
      "log_odds_ratio": -0.17898029088974,
      "logits/chosen": -0.7037605047225952,
      "logits/rejected": -1.2248471975326538,
      "logps/chosen": -1.515710711479187,
      "logps/rejected": -3.11247181892395,
      "loss": 1.5265,
      "nll_loss": 1.5085948705673218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15157108008861542,
      "rewards/margins": 0.15967608988285065,
      "rewards/rejected": -0.31124716997146606,
      "step": 1210
    },
    {
      "epoch": 1.9078014184397163,
      "grad_norm": 0.23612287640571594,
      "learning_rate": 2.7134716929820167e-08,
      "log_odds_chosen": 1.4966120719909668,
      "log_odds_ratio": -0.2065732777118683,
      "logits/chosen": -0.4843590259552002,
      "logits/rejected": -1.1729447841644287,
      "logps/chosen": -1.5309144258499146,
      "logps/rejected": -2.8435990810394287,
      "loss": 1.5509,
      "nll_loss": 1.5302910804748535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1530914455652237,
      "rewards/margins": 0.1312684863805771,
      "rewards/rejected": -0.2843599319458008,
      "step": 1211
    },
    {
      "epoch": 1.909377462568952,
      "grad_norm": 0.24082671105861664,
      "learning_rate": 2.6239730358506905e-08,
      "log_odds_chosen": 1.3767813444137573,
      "log_odds_ratio": -0.23245924711227417,
      "logits/chosen": -0.548603892326355,
      "logits/rejected": -1.270906686782837,
      "logps/chosen": -1.478569746017456,
      "logps/rejected": -2.6696934700012207,
      "loss": 1.5105,
      "nll_loss": 1.487239122390747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14785698056221008,
      "rewards/margins": 0.11911237239837646,
      "rewards/rejected": -0.26696935296058655,
      "step": 1212
    },
    {
      "epoch": 1.9109535066981875,
      "grad_norm": 0.24856548011302948,
      "learning_rate": 2.535965460329148e-08,
      "log_odds_chosen": 1.9648873805999756,
      "log_odds_ratio": -0.1395641267299652,
      "logits/chosen": -0.6576493382453918,
      "logits/rejected": -1.2277202606201172,
      "logps/chosen": -1.4042415618896484,
      "logps/rejected": -3.1344590187072754,
      "loss": 1.4302,
      "nll_loss": 1.4162611961364746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14042417705059052,
      "rewards/margins": 0.17302173376083374,
      "rewards/rejected": -0.31344589591026306,
      "step": 1213
    },
    {
      "epoch": 1.9125295508274232,
      "grad_norm": 0.22661907970905304,
      "learning_rate": 2.4494496312726043e-08,
      "log_odds_chosen": 1.772661566734314,
      "log_odds_ratio": -0.16517002880573273,
      "logits/chosen": -0.4925994277000427,
      "logits/rejected": -1.3398517370224,
      "logps/chosen": -1.403010606765747,
      "logps/rejected": -2.9484195709228516,
      "loss": 1.4336,
      "nll_loss": 1.4171255826950073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14030106365680695,
      "rewards/margins": 0.15454091131687164,
      "rewards/rejected": -0.2948419749736786,
      "step": 1214
    },
    {
      "epoch": 1.9141055949566588,
      "grad_norm": 0.2345627099275589,
      "learning_rate": 2.3644262022668005e-08,
      "log_odds_chosen": 1.595162034034729,
      "log_odds_ratio": -0.18965987861156464,
      "logits/chosen": -0.5965507626533508,
      "logits/rejected": -1.2371855974197388,
      "logps/chosen": -1.416590690612793,
      "logps/rejected": -2.7970035076141357,
      "loss": 1.4459,
      "nll_loss": 1.4269111156463623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14165908098220825,
      "rewards/margins": 0.13804125785827637,
      "rewards/rejected": -0.2797003388404846,
      "step": 1215
    },
    {
      "epoch": 1.9156816390858944,
      "grad_norm": 0.2800058126449585,
      "learning_rate": 2.280895815623185e-08,
      "log_odds_chosen": 1.449442744255066,
      "log_odds_ratio": -0.22805725038051605,
      "logits/chosen": -0.5213080644607544,
      "logits/rejected": -1.0111852884292603,
      "logps/chosen": -1.473227620124817,
      "logps/rejected": -2.733121871948242,
      "loss": 1.519,
      "nll_loss": 1.4962294101715088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14732275903224945,
      "rewards/margins": 0.1259894222021103,
      "rewards/rejected": -0.27331221103668213,
      "step": 1216
    },
    {
      "epoch": 1.91725768321513,
      "grad_norm": 0.27597102522850037,
      "learning_rate": 2.1988591023738513e-08,
      "log_odds_chosen": 1.4824138879776,
      "log_odds_ratio": -0.2200300395488739,
      "logits/chosen": -0.5362604260444641,
      "logits/rejected": -0.9742292165756226,
      "logps/chosen": -1.5755212306976318,
      "logps/rejected": -2.8891713619232178,
      "loss": 1.5981,
      "nll_loss": 1.576067328453064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15755213797092438,
      "rewards/margins": 0.13136501610279083,
      "rewards/rejected": -0.2889171838760376,
      "step": 1217
    },
    {
      "epoch": 1.9188337273443656,
      "grad_norm": 0.2543787956237793,
      "learning_rate": 2.1183166822670518e-08,
      "log_odds_chosen": 1.6037790775299072,
      "log_odds_ratio": -0.19120575487613678,
      "logits/chosen": -0.5324169397354126,
      "logits/rejected": -1.2713404893875122,
      "logps/chosen": -1.49089515209198,
      "logps/rejected": -2.894613742828369,
      "loss": 1.5189,
      "nll_loss": 1.4997775554656982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149089515209198,
      "rewards/margins": 0.14037185907363892,
      "rewards/rejected": -0.2894613742828369,
      "step": 1218
    },
    {
      "epoch": 1.9204097714736013,
      "grad_norm": 0.24087487161159515,
      "learning_rate": 2.0392691637622696e-08,
      "log_odds_chosen": 1.4946128129959106,
      "log_odds_ratio": -0.204665869474411,
      "logits/chosen": -0.5038161277770996,
      "logits/rejected": -1.2420196533203125,
      "logps/chosen": -1.494870662689209,
      "logps/rejected": -2.793170690536499,
      "loss": 1.5344,
      "nll_loss": 1.5139398574829102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14948706328868866,
      "rewards/margins": 0.1298299878835678,
      "rewards/rejected": -0.27931705117225647,
      "step": 1219
    },
    {
      "epoch": 1.9219858156028369,
      "grad_norm": 0.24739228188991547,
      "learning_rate": 1.961717144025732e-08,
      "log_odds_chosen": 1.4634636640548706,
      "log_odds_ratio": -0.21317851543426514,
      "logits/chosen": -0.5066157579421997,
      "logits/rejected": -1.1498215198516846,
      "logps/chosen": -1.4459948539733887,
      "logps/rejected": -2.705977439880371,
      "loss": 1.496,
      "nll_loss": 1.4747183322906494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14459948241710663,
      "rewards/margins": 0.12599825859069824,
      "rewards/rejected": -0.2705977261066437,
      "step": 1220
    },
    {
      "epoch": 1.9235618597320725,
      "grad_norm": 0.23965193331241608,
      "learning_rate": 1.8856612089259482e-08,
      "log_odds_chosen": 1.8252582550048828,
      "log_odds_ratio": -0.18039268255233765,
      "logits/chosen": -0.6071827411651611,
      "logits/rejected": -1.147829294204712,
      "logps/chosen": -1.5138267278671265,
      "logps/rejected": -3.1408510208129883,
      "loss": 1.541,
      "nll_loss": 1.522979974746704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1513826549053192,
      "rewards/margins": 0.16270242631435394,
      "rewards/rejected": -0.31408509612083435,
      "step": 1221
    },
    {
      "epoch": 1.9251379038613081,
      "grad_norm": 0.24508748948574066,
      "learning_rate": 1.8111019330291798e-08,
      "log_odds_chosen": 1.4505467414855957,
      "log_odds_ratio": -0.21619337797164917,
      "logits/chosen": -0.5955579876899719,
      "logits/rejected": -1.0565415620803833,
      "logps/chosen": -1.5017815828323364,
      "logps/rejected": -2.766631841659546,
      "loss": 1.5331,
      "nll_loss": 1.5115046501159668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15017816424369812,
      "rewards/margins": 0.12648500502109528,
      "rewards/rejected": -0.2766631841659546,
      "step": 1222
    },
    {
      "epoch": 1.9267139479905437,
      "grad_norm": 0.22265101969242096,
      "learning_rate": 1.738039879595088e-08,
      "log_odds_chosen": 1.7294243574142456,
      "log_odds_ratio": -0.1963079571723938,
      "logits/chosen": -0.5768942832946777,
      "logits/rejected": -1.138431191444397,
      "logps/chosen": -1.4441934823989868,
      "logps/rejected": -2.967768669128418,
      "loss": 1.4672,
      "nll_loss": 1.4476124048233032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1444193571805954,
      "rewards/margins": 0.15235751867294312,
      "rewards/rejected": -0.2967768609523773,
      "step": 1223
    },
    {
      "epoch": 1.9282899921197794,
      "grad_norm": 0.22454138100147247,
      "learning_rate": 1.6664756005726478e-08,
      "log_odds_chosen": 1.5163257122039795,
      "log_odds_ratio": -0.23830324411392212,
      "logits/chosen": -0.6747280359268188,
      "logits/rejected": -1.2130556106567383,
      "logps/chosen": -1.5061383247375488,
      "logps/rejected": -2.833340644836426,
      "loss": 1.5234,
      "nll_loss": 1.4996147155761719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15061382949352264,
      "rewards/margins": 0.1327202320098877,
      "rewards/rejected": -0.28333407640457153,
      "step": 1224
    },
    {
      "epoch": 1.929866036249015,
      "grad_norm": 0.2451227754354477,
      "learning_rate": 1.596409636595797e-08,
      "log_odds_chosen": 1.6912767887115479,
      "log_odds_ratio": -0.18132060766220093,
      "logits/chosen": -0.5936199426651001,
      "logits/rejected": -1.1482666730880737,
      "logps/chosen": -1.4519983530044556,
      "logps/rejected": -2.926872491836548,
      "loss": 1.4574,
      "nll_loss": 1.439260482788086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14519983530044556,
      "rewards/margins": 0.14748738706111908,
      "rewards/rejected": -0.29268723726272583,
      "step": 1225
    },
    {
      "epoch": 1.9314420803782506,
      "grad_norm": 0.25076785683631897,
      "learning_rate": 1.5278425169794163e-08,
      "log_odds_chosen": 1.6578562259674072,
      "log_odds_ratio": -0.18391281366348267,
      "logits/chosen": -0.5951311588287354,
      "logits/rejected": -1.2527538537979126,
      "logps/chosen": -1.4907348155975342,
      "logps/rejected": -2.9489011764526367,
      "loss": 1.5163,
      "nll_loss": 1.4979292154312134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14907348155975342,
      "rewards/margins": 0.1458166241645813,
      "rewards/rejected": -0.2948901057243347,
      "step": 1226
    },
    {
      "epoch": 1.9330181245074862,
      "grad_norm": 0.22415392100811005,
      "learning_rate": 1.4607747597153109e-08,
      "log_odds_chosen": 1.5138769149780273,
      "log_odds_ratio": -0.22110243141651154,
      "logits/chosen": -0.5883634090423584,
      "logits/rejected": -1.180192470550537,
      "logps/chosen": -1.4263536930084229,
      "logps/rejected": -2.7398767471313477,
      "loss": 1.4578,
      "nll_loss": 1.435681939125061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14263537526130676,
      "rewards/margins": 0.13135230541229248,
      "rewards/rejected": -0.27398768067359924,
      "step": 1227
    },
    {
      "epoch": 1.9345941686367218,
      "grad_norm": 0.2633359432220459,
      "learning_rate": 1.3952068714684129e-08,
      "log_odds_chosen": 1.479492425918579,
      "log_odds_ratio": -0.23165026307106018,
      "logits/chosen": -0.6251652836799622,
      "logits/rejected": -1.022170066833496,
      "logps/chosen": -1.5145764350891113,
      "logps/rejected": -2.813664197921753,
      "loss": 1.532,
      "nll_loss": 1.508789300918579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15145765244960785,
      "rewards/margins": 0.1299087554216385,
      "rewards/rejected": -0.28136640787124634,
      "step": 1228
    },
    {
      "epoch": 1.9361702127659575,
      "grad_norm": 0.2404380440711975,
      "learning_rate": 1.3311393475727628e-08,
      "log_odds_chosen": 1.6764748096466064,
      "log_odds_ratio": -0.1856500804424286,
      "logits/chosen": -0.6217374205589294,
      "logits/rejected": -1.3135581016540527,
      "logps/chosen": -1.4982596635818481,
      "logps/rejected": -2.9694433212280273,
      "loss": 1.5219,
      "nll_loss": 1.5033360719680786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14982597529888153,
      "rewards/margins": 0.14711835980415344,
      "rewards/rejected": -0.2969443202018738,
      "step": 1229
    },
    {
      "epoch": 1.937746256895193,
      "grad_norm": 0.22987554967403412,
      "learning_rate": 1.2685726720279344e-08,
      "log_odds_chosen": 1.5563294887542725,
      "log_odds_ratio": -0.19770973920822144,
      "logits/chosen": -0.6152740716934204,
      "logits/rejected": -1.2235338687896729,
      "logps/chosen": -1.594419002532959,
      "logps/rejected": -2.9775350093841553,
      "loss": 1.614,
      "nll_loss": 1.5942394733428955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15944190323352814,
      "rewards/margins": 0.13831160962581635,
      "rewards/rejected": -0.2977535128593445,
      "step": 1230
    },
    {
      "epoch": 1.9393223010244287,
      "grad_norm": 0.2675061523914337,
      "learning_rate": 1.2075073174952378e-08,
      "log_odds_chosen": 1.6666655540466309,
      "log_odds_ratio": -0.1831759810447693,
      "logits/chosen": -0.5624330043792725,
      "logits/rejected": -1.0631803274154663,
      "logps/chosen": -1.4501714706420898,
      "logps/rejected": -2.9060215950012207,
      "loss": 1.4728,
      "nll_loss": 1.454498529434204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14501716196537018,
      "rewards/margins": 0.14558503031730652,
      "rewards/rejected": -0.2906022071838379,
      "step": 1231
    },
    {
      "epoch": 1.9408983451536643,
      "grad_norm": 0.25427544116973877,
      "learning_rate": 1.1479437452942997e-08,
      "log_odds_chosen": 1.4342033863067627,
      "log_odds_ratio": -0.21727526187896729,
      "logits/chosen": -0.5201137065887451,
      "logits/rejected": -1.174269437789917,
      "logps/chosen": -1.4699643850326538,
      "logps/rejected": -2.7121121883392334,
      "loss": 1.4863,
      "nll_loss": 1.4645917415618896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14699643850326538,
      "rewards/margins": 0.1242147833108902,
      "rewards/rejected": -0.2712112069129944,
      "step": 1232
    },
    {
      "epoch": 1.9424743892829,
      "grad_norm": 0.25500911474227905,
      "learning_rate": 1.0898824053994227e-08,
      "log_odds_chosen": 1.581897497177124,
      "log_odds_ratio": -0.19705425202846527,
      "logits/chosen": -0.5554385185241699,
      "logits/rejected": -1.2053784132003784,
      "logps/chosen": -1.556564211845398,
      "logps/rejected": -2.957549571990967,
      "loss": 1.5589,
      "nll_loss": 1.539212703704834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15565641224384308,
      "rewards/margins": 0.14009852707386017,
      "rewards/rejected": -0.29575493931770325,
      "step": 1233
    },
    {
      "epoch": 1.9440504334121356,
      "grad_norm": 0.2842603623867035,
      "learning_rate": 1.0333237364362978e-08,
      "log_odds_chosen": 1.613905906677246,
      "log_odds_ratio": -0.2091207504272461,
      "logits/chosen": -0.4995366334915161,
      "logits/rejected": -1.0226919651031494,
      "logps/chosen": -1.4732149839401245,
      "logps/rejected": -2.889878273010254,
      "loss": 1.499,
      "nll_loss": 1.4780728816986084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14732149243354797,
      "rewards/margins": 0.14166630804538727,
      "rewards/rejected": -0.2889878451824188,
      "step": 1234
    },
    {
      "epoch": 1.9456264775413712,
      "grad_norm": 0.24073565006256104,
      "learning_rate": 9.782681656786973e-09,
      "log_odds_chosen": 1.5523452758789062,
      "log_odds_ratio": -0.19343072175979614,
      "logits/chosen": -0.5765637755393982,
      "logits/rejected": -1.1617844104766846,
      "logps/chosen": -1.445525050163269,
      "logps/rejected": -2.7901933193206787,
      "loss": 1.4862,
      "nll_loss": 1.4668691158294678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14455249905586243,
      "rewards/margins": 0.13446684181690216,
      "rewards/rejected": -0.2790193557739258,
      "step": 1235
    },
    {
      "epoch": 1.9472025216706068,
      "grad_norm": 0.2727525532245636,
      "learning_rate": 9.247161090451206e-09,
      "log_odds_chosen": 1.429125428199768,
      "log_odds_ratio": -0.21855179965496063,
      "logits/chosen": -0.5664371252059937,
      "logits/rejected": -1.0441747903823853,
      "logps/chosen": -1.5691709518432617,
      "logps/rejected": -2.8258721828460693,
      "loss": 1.5683,
      "nll_loss": 1.546474814414978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15691711008548737,
      "rewards/margins": 0.12567013502120972,
      "rewards/rejected": -0.2825872600078583,
      "step": 1236
    },
    {
      "epoch": 1.9487785657998424,
      "grad_norm": 0.25064021348953247,
      "learning_rate": 8.726679710957752e-09,
      "log_odds_chosen": 1.4847660064697266,
      "log_odds_ratio": -0.21513798832893372,
      "logits/chosen": -0.5887483358383179,
      "logits/rejected": -1.1240547895431519,
      "logps/chosen": -1.4997950792312622,
      "logps/rejected": -2.7971580028533936,
      "loss": 1.5198,
      "nll_loss": 1.4982860088348389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14997953176498413,
      "rewards/margins": 0.1297363042831421,
      "rewards/rejected": -0.2797158360481262,
      "step": 1237
    },
    {
      "epoch": 1.950354609929078,
      "grad_norm": 0.24634447693824768,
      "learning_rate": 8.221241450295124e-09,
      "log_odds_chosen": 1.601959466934204,
      "log_odds_ratio": -0.20689457654953003,
      "logits/chosen": -0.5835831761360168,
      "logits/rejected": -1.1361600160598755,
      "logps/chosen": -1.5163729190826416,
      "logps/rejected": -2.9279205799102783,
      "loss": 1.5268,
      "nll_loss": 1.5061551332473755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15163730084896088,
      "rewards/margins": 0.14115475118160248,
      "rewards/rejected": -0.29279205203056335,
      "step": 1238
    },
    {
      "epoch": 1.9519306540583137,
      "grad_norm": 0.23574163019657135,
      "learning_rate": 7.730850126807853e-09,
      "log_odds_chosen": 1.7678840160369873,
      "log_odds_ratio": -0.16229620575904846,
      "logits/chosen": -0.5642778873443604,
      "logits/rejected": -1.181726098060608,
      "logps/chosen": -1.3877137899398804,
      "logps/rejected": -2.9212727546691895,
      "loss": 1.4338,
      "nll_loss": 1.417576551437378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13877137005329132,
      "rewards/margins": 0.1533558964729309,
      "rewards/rejected": -0.2921272814273834,
      "step": 1239
    },
    {
      "epoch": 1.9535066981875493,
      "grad_norm": 0.23919585347175598,
      "learning_rate": 7.255509445168062e-09,
      "log_odds_chosen": 1.4537686109542847,
      "log_odds_ratio": -0.24287199974060059,
      "logits/chosen": -0.46138328313827515,
      "logits/rejected": -1.137482762336731,
      "logps/chosen": -1.430019736289978,
      "logps/rejected": -2.6916005611419678,
      "loss": 1.4532,
      "nll_loss": 1.428902506828308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430019736289978,
      "rewards/margins": 0.12615808844566345,
      "rewards/rejected": -0.26916006207466125,
      "step": 1240
    },
    {
      "epoch": 1.955082742316785,
      "grad_norm": 0.22261321544647217,
      "learning_rate": 6.795222996347494e-09,
      "log_odds_chosen": 1.4821674823760986,
      "log_odds_ratio": -0.2140667736530304,
      "logits/chosen": -0.5457820892333984,
      "logits/rejected": -1.0856201648712158,
      "logps/chosen": -1.3843501806259155,
      "logps/rejected": -2.648566722869873,
      "loss": 1.4218,
      "nll_loss": 1.4003808498382568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1384350210428238,
      "rewards/margins": 0.12642163038253784,
      "rewards/rejected": -0.26485663652420044,
      "step": 1241
    },
    {
      "epoch": 1.9566587864460205,
      "grad_norm": 0.2750306725502014,
      "learning_rate": 6.349994257590862e-09,
      "log_odds_chosen": 1.560042381286621,
      "log_odds_ratio": -0.20762906968593597,
      "logits/chosen": -0.6289823055267334,
      "logits/rejected": -1.2155874967575073,
      "logps/chosen": -1.5861676931381226,
      "logps/rejected": -2.9747793674468994,
      "loss": 1.6006,
      "nll_loss": 1.5798269510269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15861676633358002,
      "rewards/margins": 0.13886116445064545,
      "rewards/rejected": -0.29747796058654785,
      "step": 1242
    },
    {
      "epoch": 1.9582348305752562,
      "grad_norm": 0.23600639402866364,
      "learning_rate": 5.9198265923881e-09,
      "log_odds_chosen": 1.7496486902236938,
      "log_odds_ratio": -0.1800561249256134,
      "logits/chosen": -0.6179554462432861,
      "logits/rejected": -1.2912925481796265,
      "logps/chosen": -1.490044116973877,
      "logps/rejected": -3.041320323944092,
      "loss": 1.5085,
      "nll_loss": 1.4905422925949097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14900439977645874,
      "rewards/margins": 0.1551276445388794,
      "rewards/rejected": -0.30413204431533813,
      "step": 1243
    },
    {
      "epoch": 1.9598108747044918,
      "grad_norm": 0.24325112998485565,
      "learning_rate": 5.504723250450593e-09,
      "log_odds_chosen": 1.5230752229690552,
      "log_odds_ratio": -0.20526538789272308,
      "logits/chosen": -0.6034608483314514,
      "logits/rejected": -1.262913465499878,
      "logps/chosen": -1.4928333759307861,
      "logps/rejected": -2.8210668563842773,
      "loss": 1.5183,
      "nll_loss": 1.4978017807006836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14928331971168518,
      "rewards/margins": 0.13282336294651031,
      "rewards/rejected": -0.2821066975593567,
      "step": 1244
    },
    {
      "epoch": 1.9613869188337274,
      "grad_norm": 0.26483285427093506,
      "learning_rate": 5.1046873676861005e-09,
      "log_odds_chosen": 1.48130202293396,
      "log_odds_ratio": -0.20924536883831024,
      "logits/chosen": -0.5241674780845642,
      "logits/rejected": -1.33096182346344,
      "logps/chosen": -1.5158768892288208,
      "logps/rejected": -2.810894727706909,
      "loss": 1.5437,
      "nll_loss": 1.5227303504943848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15158769488334656,
      "rewards/margins": 0.12950178980827332,
      "rewards/rejected": -0.2810894846916199,
      "step": 1245
    },
    {
      "epoch": 1.9629629629629628,
      "grad_norm": 0.26010143756866455,
      "learning_rate": 4.719721966174317e-09,
      "log_odds_chosen": 1.5984944105148315,
      "log_odds_ratio": -0.19065454602241516,
      "logits/chosen": -0.6097192764282227,
      "logits/rejected": -1.140574336051941,
      "logps/chosen": -1.459886074066162,
      "logps/rejected": -2.854323625564575,
      "loss": 1.484,
      "nll_loss": 1.4649107456207275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1459886133670807,
      "rewards/margins": 0.1394437551498413,
      "rewards/rejected": -0.2854323983192444,
      "step": 1246
    },
    {
      "epoch": 1.9645390070921986,
      "grad_norm": 0.25776347517967224,
      "learning_rate": 4.349829954145123e-09,
      "log_odds_chosen": 1.3453749418258667,
      "log_odds_ratio": -0.24993421137332916,
      "logits/chosen": -0.6820028424263,
      "logits/rejected": -0.9831753969192505,
      "logps/chosen": -1.435805082321167,
      "logps/rejected": -2.5907487869262695,
      "loss": 1.4865,
      "nll_loss": 1.4615468978881836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14358051121234894,
      "rewards/margins": 0.11549436300992966,
      "rewards/rejected": -0.259074866771698,
      "step": 1247
    },
    {
      "epoch": 1.966115051221434,
      "grad_norm": 0.24005144834518433,
      "learning_rate": 3.995014125956153e-09,
      "log_odds_chosen": 1.480599284172058,
      "log_odds_ratio": -0.22187894582748413,
      "logits/chosen": -0.6296998858451843,
      "logits/rejected": -1.2187302112579346,
      "logps/chosen": -1.4557812213897705,
      "logps/rejected": -2.7413909435272217,
      "loss": 1.4846,
      "nll_loss": 1.4624024629592896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14557813107967377,
      "rewards/margins": 0.12856096029281616,
      "rewards/rejected": -0.2741391062736511,
      "step": 1248
    },
    {
      "epoch": 1.9676910953506699,
      "grad_norm": 0.22547145187854767,
      "learning_rate": 3.6552771620712573e-09,
      "log_odds_chosen": 1.7240605354309082,
      "log_odds_ratio": -0.17077501118183136,
      "logits/chosen": -0.6393252611160278,
      "logits/rejected": -1.3706183433532715,
      "logps/chosen": -1.4487979412078857,
      "logps/rejected": -2.9510762691497803,
      "loss": 1.4579,
      "nll_loss": 1.4407744407653809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1448797881603241,
      "rewards/margins": 0.15022781491279602,
      "rewards/rejected": -0.2951076328754425,
      "step": 1249
    },
    {
      "epoch": 1.9692671394799053,
      "grad_norm": 0.32271239161491394,
      "learning_rate": 3.3306216290409637e-09,
      "log_odds_chosen": 1.4469444751739502,
      "log_odds_ratio": -0.21564699709415436,
      "logits/chosen": -0.5960330963134766,
      "logits/rejected": -1.1813716888427734,
      "logps/chosen": -1.3483270406723022,
      "logps/rejected": -2.5726072788238525,
      "loss": 1.4019,
      "nll_loss": 1.3802921772003174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1348326951265335,
      "rewards/margins": 0.1224280297756195,
      "rewards/rejected": -0.2572607398033142,
      "step": 1250
    },
    {
      "epoch": 1.9708431836091411,
      "grad_norm": 0.2598452568054199,
      "learning_rate": 3.021049979482715e-09,
      "log_odds_chosen": 1.3215519189834595,
      "log_odds_ratio": -0.2433094084262848,
      "logits/chosen": -0.4853062033653259,
      "logits/rejected": -1.1187504529953003,
      "logps/chosen": -1.4518523216247559,
      "logps/rejected": -2.5887539386749268,
      "loss": 1.4865,
      "nll_loss": 1.462129831314087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14518523216247559,
      "rewards/margins": 0.11369017511606216,
      "rewards/rejected": -0.25887542963027954,
      "step": 1251
    },
    {
      "epoch": 1.9724192277383765,
      "grad_norm": 0.2414374202489853,
      "learning_rate": 2.72656455206266e-09,
      "log_odds_chosen": 1.413500189781189,
      "log_odds_ratio": -0.2291947603225708,
      "logits/chosen": -0.5328694581985474,
      "logits/rejected": -1.083142876625061,
      "logps/chosen": -1.5253971815109253,
      "logps/rejected": -2.761502504348755,
      "loss": 1.5603,
      "nll_loss": 1.537413239479065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15253973007202148,
      "rewards/margins": 0.12361055612564087,
      "rewards/rejected": -0.27615025639533997,
      "step": 1252
    },
    {
      "epoch": 1.9739952718676124,
      "grad_norm": 0.22480349242687225,
      "learning_rate": 2.447167571477449e-09,
      "log_odds_chosen": 1.5263303518295288,
      "log_odds_ratio": -0.19853481650352478,
      "logits/chosen": -0.4815334975719452,
      "logits/rejected": -1.0565820932388306,
      "logps/chosen": -1.4597349166870117,
      "logps/rejected": -2.78013277053833,
      "loss": 1.4917,
      "nll_loss": 1.471849799156189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14597350358963013,
      "rewards/margins": 0.13203978538513184,
      "rewards/rejected": -0.27801328897476196,
      "step": 1253
    },
    {
      "epoch": 1.9755713159968478,
      "grad_norm": 0.2529234290122986,
      "learning_rate": 2.182861148437798e-09,
      "log_odds_chosen": 1.5894269943237305,
      "log_odds_ratio": -0.21810925006866455,
      "logits/chosen": -0.6243840456008911,
      "logits/rejected": -1.1135503053665161,
      "logps/chosen": -1.5673398971557617,
      "logps/rejected": -2.9797096252441406,
      "loss": 1.5757,
      "nll_loss": 1.553900122642517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15673398971557617,
      "rewards/margins": 0.14123694598674774,
      "rewards/rejected": -0.29797089099884033,
      "step": 1254
    },
    {
      "epoch": 1.9771473601260836,
      "grad_norm": 0.23752756416797638,
      "learning_rate": 1.933647279652506e-09,
      "log_odds_chosen": 1.7463414669036865,
      "log_odds_ratio": -0.20896771550178528,
      "logits/chosen": -0.6086210608482361,
      "logits/rejected": -1.1639318466186523,
      "logps/chosen": -1.4526927471160889,
      "logps/rejected": -2.998058557510376,
      "loss": 1.5019,
      "nll_loss": 1.481015682220459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452692747116089,
      "rewards/margins": 0.15453660488128662,
      "rewards/rejected": -0.2998058795928955,
      "step": 1255
    },
    {
      "epoch": 1.978723404255319,
      "grad_norm": 0.23521293699741364,
      "learning_rate": 1.6995278478133534e-09,
      "log_odds_chosen": 1.3194330930709839,
      "log_odds_ratio": -0.24331331253051758,
      "logits/chosen": -0.5043147802352905,
      "logits/rejected": -1.0727133750915527,
      "logps/chosen": -1.4686273336410522,
      "logps/rejected": -2.6020939350128174,
      "loss": 1.5264,
      "nll_loss": 1.5021082162857056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14686273038387299,
      "rewards/margins": 0.11334666609764099,
      "rewards/rejected": -0.2602093815803528,
      "step": 1256
    },
    {
      "epoch": 1.9802994483845549,
      "grad_norm": 0.23937982320785522,
      "learning_rate": 1.4805046215806693e-09,
      "log_odds_chosen": 1.5560564994812012,
      "log_odds_ratio": -0.19879305362701416,
      "logits/chosen": -0.5874810218811035,
      "logits/rejected": -1.1548365354537964,
      "logps/chosen": -1.43878173828125,
      "logps/rejected": -2.787869453430176,
      "loss": 1.4687,
      "nll_loss": 1.4488128423690796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14387817680835724,
      "rewards/margins": 0.1349087804555893,
      "rewards/rejected": -0.27878695726394653,
      "step": 1257
    },
    {
      "epoch": 1.9818754925137902,
      "grad_norm": 0.22531044483184814,
      "learning_rate": 1.2765792555704536e-09,
      "log_odds_chosen": 1.4314652681350708,
      "log_odds_ratio": -0.21993236243724823,
      "logits/chosen": -0.6161269545555115,
      "logits/rejected": -1.252550482749939,
      "logps/chosen": -1.492631196975708,
      "logps/rejected": -2.733046531677246,
      "loss": 1.5128,
      "nll_loss": 1.4908132553100586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14926311373710632,
      "rewards/margins": 0.12404152750968933,
      "rewards/rejected": -0.27330464124679565,
      "step": 1258
    },
    {
      "epoch": 1.983451536643026,
      "grad_norm": 0.23570935428142548,
      "learning_rate": 1.0877532903414977e-09,
      "log_odds_chosen": 1.740430474281311,
      "log_odds_ratio": -0.17677739262580872,
      "logits/chosen": -0.5560349225997925,
      "logits/rejected": -1.0577348470687866,
      "logps/chosen": -1.4009289741516113,
      "logps/rejected": -2.920013666152954,
      "loss": 1.4397,
      "nll_loss": 1.4220633506774902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1400929093360901,
      "rewards/margins": 0.1519085019826889,
      "rewards/rejected": -0.2920013964176178,
      "step": 1259
    },
    {
      "epoch": 1.9850275807722615,
      "grad_norm": 0.2756683826446533,
      "learning_rate": 9.140281523836168e-10,
      "log_odds_chosen": 1.5552338361740112,
      "log_odds_ratio": -0.19513314962387085,
      "logits/chosen": -0.622211754322052,
      "logits/rejected": -1.0341156721115112,
      "logps/chosen": -1.5147669315338135,
      "logps/rejected": -2.875410318374634,
      "loss": 1.5405,
      "nll_loss": 1.5210151672363281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1514766961336136,
      "rewards/margins": 0.136064350605011,
      "rewards/rejected": -0.2875410318374634,
      "step": 1260
    },
    {
      "epoch": 1.9866036249014973,
      "grad_norm": 0.2340654730796814,
      "learning_rate": 7.554051541074357e-10,
      "log_odds_chosen": 1.504246711730957,
      "log_odds_ratio": -0.21054834127426147,
      "logits/chosen": -0.5831047892570496,
      "logits/rejected": -1.2525290250778198,
      "logps/chosen": -1.428353190422058,
      "logps/rejected": -2.723766803741455,
      "loss": 1.4651,
      "nll_loss": 1.4440265893936157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.142835333943367,
      "rewards/margins": 0.12954135239124298,
      "rewards/rejected": -0.27237668633461,
      "step": 1261
    },
    {
      "epoch": 1.9881796690307327,
      "grad_norm": 0.24898484349250793,
      "learning_rate": 6.118854938337304e-10,
      "log_odds_chosen": 1.4546529054641724,
      "log_odds_ratio": -0.22209064662456512,
      "logits/chosen": -0.6307198405265808,
      "logits/rejected": -1.156106948852539,
      "logps/chosen": -1.4630794525146484,
      "logps/rejected": -2.726752758026123,
      "loss": 1.498,
      "nll_loss": 1.4757537841796875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14630794525146484,
      "rewards/margins": 0.12636736035346985,
      "rewards/rejected": -0.2726753056049347,
      "step": 1262
    },
    {
      "epoch": 1.9897557131599686,
      "grad_norm": 0.23436900973320007,
      "learning_rate": 4.834702557852121e-10,
      "log_odds_chosen": 1.502882719039917,
      "log_odds_ratio": -0.20643973350524902,
      "logits/chosen": -0.6238887906074524,
      "logits/rejected": -1.295356273651123,
      "logps/chosen": -1.545674204826355,
      "logps/rejected": -2.868143081665039,
      "loss": 1.5626,
      "nll_loss": 1.5419411659240723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1545674204826355,
      "rewards/margins": 0.13224688172340393,
      "rewards/rejected": -0.28681430220603943,
      "step": 1263
    },
    {
      "epoch": 1.991331757289204,
      "grad_norm": 0.24168695509433746,
      "learning_rate": 3.7016041007742473e-10,
      "log_odds_chosen": 1.764907956123352,
      "log_odds_ratio": -0.16733038425445557,
      "logits/chosen": -0.5464268326759338,
      "logits/rejected": -1.2948414087295532,
      "logps/chosen": -1.4873374700546265,
      "logps/rejected": -3.0441391468048096,
      "loss": 1.5134,
      "nll_loss": 1.4966189861297607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14873374998569489,
      "rewards/margins": 0.15568016469478607,
      "rewards/rejected": -0.30441391468048096,
      "step": 1264
    },
    {
      "epoch": 1.9929078014184398,
      "grad_norm": 0.2419695258140564,
      "learning_rate": 2.71956812712304e-10,
      "log_odds_chosen": 1.7183727025985718,
      "log_odds_ratio": -0.17193645238876343,
      "logits/chosen": -0.612159013748169,
      "logits/rejected": -1.3997570276260376,
      "logps/chosen": -1.502026081085205,
      "logps/rejected": -3.0187392234802246,
      "loss": 1.5378,
      "nll_loss": 1.5205950736999512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15020261704921722,
      "rewards/margins": 0.15167130529880524,
      "rewards/rejected": -0.30187392234802246,
      "step": 1265
    },
    {
      "epoch": 1.9944838455476752,
      "grad_norm": 0.25079745054244995,
      "learning_rate": 1.8886020557107308e-10,
      "log_odds_chosen": 1.691375732421875,
      "log_odds_ratio": -0.1741671860218048,
      "logits/chosen": -0.5419628620147705,
      "logits/rejected": -1.2016863822937012,
      "logps/chosen": -1.4981317520141602,
      "logps/rejected": -2.987499952316284,
      "loss": 1.5276,
      "nll_loss": 1.5101757049560547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14981317520141602,
      "rewards/margins": 0.14893679320812225,
      "rewards/rejected": -0.29874998331069946,
      "step": 1266
    },
    {
      "epoch": 1.996059889676911,
      "grad_norm": 0.26726582646369934,
      "learning_rate": 1.208712164091352e-10,
      "log_odds_chosen": 1.5760363340377808,
      "log_odds_ratio": -0.19377130270004272,
      "logits/chosen": -0.6167798042297363,
      "logits/rejected": -1.1072314977645874,
      "logps/chosen": -1.5077298879623413,
      "logps/rejected": -2.8876824378967285,
      "loss": 1.5279,
      "nll_loss": 1.5085444450378418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15077300369739532,
      "rewards/margins": 0.13799524307250977,
      "rewards/rejected": -0.2887682318687439,
      "step": 1267
    },
    {
      "epoch": 1.9976359338061465,
      "grad_norm": 0.24022099375724792,
      "learning_rate": 6.799035885030057e-11,
      "log_odds_chosen": 1.8151551485061646,
      "log_odds_ratio": -0.16386988759040833,
      "logits/chosen": -0.6020281314849854,
      "logits/rejected": -1.1378754377365112,
      "logps/chosen": -1.5072858333587646,
      "logps/rejected": -3.116183042526245,
      "loss": 1.5234,
      "nll_loss": 1.5070056915283203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15072858333587646,
      "rewards/margins": 0.16088971495628357,
      "rewards/rejected": -0.3116183280944824,
      "step": 1268
    },
    {
      "epoch": 1.9992119779353823,
      "grad_norm": 0.23787525296211243,
      "learning_rate": 3.0218032384565774e-11,
      "log_odds_chosen": 1.6236554384231567,
      "log_odds_ratio": -0.1878548115491867,
      "logits/chosen": -0.5019420385360718,
      "logits/rejected": -1.138771414756775,
      "logps/chosen": -1.3867213726043701,
      "logps/rejected": -2.7829477787017822,
      "loss": 1.4367,
      "nll_loss": 1.4179333448410034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1386721283197403,
      "rewards/margins": 0.13962264358997345,
      "rewards/rejected": -0.27829480171203613,
      "step": 1269
    },
    {
      "epoch": 2.0,
      "grad_norm": 0.31517863273620605,
      "learning_rate": 7.554522363895089e-12,
      "log_odds_chosen": 1.6290042400360107,
      "log_odds_ratio": -0.20081059634685516,
      "logits/chosen": -0.55748450756073,
      "logits/rejected": -1.2785966396331787,
      "logps/chosen": -1.4307972192764282,
      "logps/rejected": -2.8508033752441406,
      "loss": 1.4614,
      "nll_loss": 1.4413384199142456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430797278881073,
      "rewards/margins": 0.14200060069561005,
      "rewards/rejected": -0.28508034348487854,
      "step": 1270
    }
  ],
  "logging_steps": 1,
  "max_steps": 1270,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}