{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.7880220646178093,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015760441292356187,
      "grad_norm": 0.8194138407707214,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.35938164591789246,
      "log_odds_ratio": -0.5514221787452698,
      "logits/chosen": -0.6073167324066162,
      "logits/rejected": 0.20061472058296204,
      "logps/chosen": -2.176460027694702,
      "logps/rejected": -2.502065420150757,
      "loss": 2.4622,
      "nll_loss": 2.407036304473877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21764598786830902,
      "rewards/margins": 0.03256051242351532,
      "rewards/rejected": -0.25020653009414673,
      "step": 1
    },
    {
      "epoch": 0.0031520882584712374,
      "grad_norm": 0.7936525940895081,
      "learning_rate": 3.1496062992125985e-08,
      "log_odds_chosen": 0.35638517141342163,
      "log_odds_ratio": -0.5405049324035645,
      "logits/chosen": -0.6276549100875854,
      "logits/rejected": 0.3162096440792084,
      "logps/chosen": -2.069308280944824,
      "logps/rejected": -2.385652780532837,
      "loss": 2.343,
      "nll_loss": 2.288930892944336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20693084597587585,
      "rewards/margins": 0.031634461134672165,
      "rewards/rejected": -0.23856529593467712,
      "step": 2
    },
    {
      "epoch": 0.004728132387706856,
      "grad_norm": 0.9091457724571228,
      "learning_rate": 6.299212598425197e-08,
      "log_odds_chosen": 0.7094058394432068,
      "log_odds_ratio": -0.4360072910785675,
      "logits/chosen": -0.7982139587402344,
      "logits/rejected": 0.22547274827957153,
      "logps/chosen": -2.2025773525238037,
      "logps/rejected": -2.8602232933044434,
      "loss": 2.4665,
      "nll_loss": 2.4228932857513428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2202577441930771,
      "rewards/margins": 0.06576459109783173,
      "rewards/rejected": -0.2860223054885864,
      "step": 3
    },
    {
      "epoch": 0.006304176516942475,
      "grad_norm": 0.7839515209197998,
      "learning_rate": 9.448818897637795e-08,
      "log_odds_chosen": 0.6190818548202515,
      "log_odds_ratio": -0.45430099964141846,
      "logits/chosen": -0.5776420831680298,
      "logits/rejected": 0.3159808814525604,
      "logps/chosen": -2.197322130203247,
      "logps/rejected": -2.7688748836517334,
      "loss": 2.4613,
      "nll_loss": 2.4159181118011475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21973221004009247,
      "rewards/margins": 0.057155244052410126,
      "rewards/rejected": -0.2768874764442444,
      "step": 4
    },
    {
      "epoch": 0.007880220646178092,
      "grad_norm": 0.8797410726547241,
      "learning_rate": 1.2598425196850394e-07,
      "log_odds_chosen": 0.6505839228630066,
      "log_odds_ratio": -0.43997013568878174,
      "logits/chosen": -0.6981326937675476,
      "logits/rejected": 0.13587594032287598,
      "logps/chosen": -2.136472702026367,
      "logps/rejected": -2.7251079082489014,
      "loss": 2.395,
      "nll_loss": 2.3510234355926514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2136472761631012,
      "rewards/margins": 0.05886353179812431,
      "rewards/rejected": -0.272510826587677,
      "step": 5
    },
    {
      "epoch": 0.009456264775413711,
      "grad_norm": 0.8133273720741272,
      "learning_rate": 1.5748031496062992e-07,
      "log_odds_chosen": 0.39265337586402893,
      "log_odds_ratio": -0.5214492678642273,
      "logits/chosen": -0.5345965027809143,
      "logits/rejected": 0.3466244637966156,
      "logps/chosen": -2.0630874633789062,
      "logps/rejected": -2.4150893688201904,
      "loss": 2.3692,
      "nll_loss": 2.317031145095825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2063087522983551,
      "rewards/margins": 0.035200200974941254,
      "rewards/rejected": -0.24150894582271576,
      "step": 6
    },
    {
      "epoch": 0.01103230890464933,
      "grad_norm": 0.7818942666053772,
      "learning_rate": 1.889763779527559e-07,
      "log_odds_chosen": 0.7252346277236938,
      "log_odds_ratio": -0.4240948557853699,
      "logits/chosen": -0.5406574606895447,
      "logits/rejected": 0.29844143986701965,
      "logps/chosen": -2.129410982131958,
      "logps/rejected": -2.7971620559692383,
      "loss": 2.3917,
      "nll_loss": 2.34930419921875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21294109523296356,
      "rewards/margins": 0.06677509844303131,
      "rewards/rejected": -0.2797161936759949,
      "step": 7
    },
    {
      "epoch": 0.01260835303388495,
      "grad_norm": 1.0168143510818481,
      "learning_rate": 2.2047244094488187e-07,
      "log_odds_chosen": 0.9424365162849426,
      "log_odds_ratio": -0.34798142313957214,
      "logits/chosen": -0.6615124344825745,
      "logits/rejected": 0.22647148370742798,
      "logps/chosen": -2.2724151611328125,
      "logps/rejected": -3.144073963165283,
      "loss": 2.5346,
      "nll_loss": 2.4997568130493164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22724151611328125,
      "rewards/margins": 0.08716587722301483,
      "rewards/rejected": -0.3144074082374573,
      "step": 8
    },
    {
      "epoch": 0.014184397163120567,
      "grad_norm": 0.7796529531478882,
      "learning_rate": 2.519685039370079e-07,
      "log_odds_chosen": 0.4133250415325165,
      "log_odds_ratio": -0.5154058337211609,
      "logits/chosen": -0.5772379636764526,
      "logits/rejected": 0.26968488097190857,
      "logps/chosen": -2.124163866043091,
      "logps/rejected": -2.496925115585327,
      "loss": 2.3769,
      "nll_loss": 2.325348377227783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2124163955450058,
      "rewards/margins": 0.03727611154317856,
      "rewards/rejected": -0.24969248473644257,
      "step": 9
    },
    {
      "epoch": 0.015760441292356184,
      "grad_norm": 0.8118637800216675,
      "learning_rate": 2.8346456692913386e-07,
      "log_odds_chosen": 0.4085013270378113,
      "log_odds_ratio": -0.5165191292762756,
      "logits/chosen": -0.6135594844818115,
      "logits/rejected": 0.11062480509281158,
      "logps/chosen": -2.1375293731689453,
      "logps/rejected": -2.5072574615478516,
      "loss": 2.4048,
      "nll_loss": 2.3531653881073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21375294029712677,
      "rewards/margins": 0.03697281330823898,
      "rewards/rejected": -0.25072574615478516,
      "step": 10
    },
    {
      "epoch": 0.017336485421591805,
      "grad_norm": 0.8751649856567383,
      "learning_rate": 3.1496062992125984e-07,
      "log_odds_chosen": 0.4661106765270233,
      "log_odds_ratio": -0.49116337299346924,
      "logits/chosen": -0.5825808644294739,
      "logits/rejected": 0.18361972272396088,
      "logps/chosen": -2.239102840423584,
      "logps/rejected": -2.662409782409668,
      "loss": 2.5002,
      "nll_loss": 2.451051950454712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22391027212142944,
      "rewards/margins": 0.042330723255872726,
      "rewards/rejected": -0.2662409842014313,
      "step": 11
    },
    {
      "epoch": 0.018912529550827423,
      "grad_norm": 1.0198794603347778,
      "learning_rate": 3.464566929133858e-07,
      "log_odds_chosen": 0.6043168902397156,
      "log_odds_ratio": -0.4442984163761139,
      "logits/chosen": -0.756458580493927,
      "logits/rejected": 0.36159712076187134,
      "logps/chosen": -2.3110899925231934,
      "logps/rejected": -2.8696181774139404,
      "loss": 2.5785,
      "nll_loss": 2.5340256690979004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.23110897839069366,
      "rewards/margins": 0.055852800607681274,
      "rewards/rejected": -0.2869618237018585,
      "step": 12
    },
    {
      "epoch": 0.02048857368006304,
      "grad_norm": 0.9937220215797424,
      "learning_rate": 3.779527559055118e-07,
      "log_odds_chosen": 0.543169379234314,
      "log_odds_ratio": -0.48258134722709656,
      "logits/chosen": -0.8145802021026611,
      "logits/rejected": 0.4112021327018738,
      "logps/chosen": -2.206066608428955,
      "logps/rejected": -2.700916290283203,
      "loss": 2.4702,
      "nll_loss": 2.421976089477539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22060665488243103,
      "rewards/margins": 0.0494849756360054,
      "rewards/rejected": -0.27009162306785583,
      "step": 13
    },
    {
      "epoch": 0.02206461780929866,
      "grad_norm": 0.8126177787780762,
      "learning_rate": 4.0944881889763777e-07,
      "log_odds_chosen": 0.5306293368339539,
      "log_odds_ratio": -0.5026512145996094,
      "logits/chosen": -0.6288756728172302,
      "logits/rejected": 0.034559354186058044,
      "logps/chosen": -2.1632676124572754,
      "logps/rejected": -2.653379440307617,
      "loss": 2.425,
      "nll_loss": 2.3747172355651855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2163267731666565,
      "rewards/margins": 0.04901116341352463,
      "rewards/rejected": -0.2653379440307617,
      "step": 14
    },
    {
      "epoch": 0.02364066193853428,
      "grad_norm": 0.9252188801765442,
      "learning_rate": 4.4094488188976375e-07,
      "log_odds_chosen": 0.4157894253730774,
      "log_odds_ratio": -0.5104730129241943,
      "logits/chosen": -0.7391470074653625,
      "logits/rejected": 0.21298080682754517,
      "logps/chosen": -2.185275077819824,
      "logps/rejected": -2.563283920288086,
      "loss": 2.4617,
      "nll_loss": 2.4106781482696533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21852751076221466,
      "rewards/margins": 0.03780088573694229,
      "rewards/rejected": -0.25632840394973755,
      "step": 15
    },
    {
      "epoch": 0.0252167060677699,
      "grad_norm": 0.9326465129852295,
      "learning_rate": 4.7244094488188973e-07,
      "log_odds_chosen": 0.42695683240890503,
      "log_odds_ratio": -0.5144686102867126,
      "logits/chosen": -0.7023543119430542,
      "logits/rejected": 0.20582355558872223,
      "logps/chosen": -2.171863079071045,
      "logps/rejected": -2.559831380844116,
      "loss": 2.4416,
      "nll_loss": 2.3901824951171875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2171863466501236,
      "rewards/margins": 0.0387967973947525,
      "rewards/rejected": -0.2559831440448761,
      "step": 16
    },
    {
      "epoch": 0.026792750197005517,
      "grad_norm": 0.7562754154205322,
      "learning_rate": 5.039370078740158e-07,
      "log_odds_chosen": 0.6844577789306641,
      "log_odds_ratio": -0.42741507291793823,
      "logits/chosen": -0.5460200309753418,
      "logits/rejected": 0.18708395957946777,
      "logps/chosen": -2.098813056945801,
      "logps/rejected": -2.723797082901001,
      "loss": 2.3725,
      "nll_loss": 2.3297088146209717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20988130569458008,
      "rewards/margins": 0.062498390674591064,
      "rewards/rejected": -0.27237969636917114,
      "step": 17
    },
    {
      "epoch": 0.028368794326241134,
      "grad_norm": 0.8840509653091431,
      "learning_rate": 5.354330708661418e-07,
      "log_odds_chosen": 0.3034907281398773,
      "log_odds_ratio": -0.5711926221847534,
      "logits/chosen": -0.6161488890647888,
      "logits/rejected": 0.47962120175361633,
      "logps/chosen": -2.1487925052642822,
      "logps/rejected": -2.4217610359191895,
      "loss": 2.4355,
      "nll_loss": 2.3784008026123047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21487924456596375,
      "rewards/margins": 0.0272968839854002,
      "rewards/rejected": -0.2421761304140091,
      "step": 18
    },
    {
      "epoch": 0.029944838455476755,
      "grad_norm": 0.8342726230621338,
      "learning_rate": 5.669291338582677e-07,
      "log_odds_chosen": 0.6132441163063049,
      "log_odds_ratio": -0.4440116584300995,
      "logits/chosen": -0.7522751688957214,
      "logits/rejected": 0.13862034678459167,
      "logps/chosen": -2.0390007495880127,
      "logps/rejected": -2.5903594493865967,
      "loss": 2.3287,
      "nll_loss": 2.2843337059020996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20390008389949799,
      "rewards/margins": 0.055135857313871384,
      "rewards/rejected": -0.25903594493865967,
      "step": 19
    },
    {
      "epoch": 0.03152088258471237,
      "grad_norm": 0.8772868514060974,
      "learning_rate": 5.984251968503937e-07,
      "log_odds_chosen": 0.6135118007659912,
      "log_odds_ratio": -0.460088849067688,
      "logits/chosen": -0.5715646147727966,
      "logits/rejected": 0.18610909581184387,
      "logps/chosen": -2.1998162269592285,
      "logps/rejected": -2.76267147064209,
      "loss": 2.483,
      "nll_loss": 2.436969757080078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2199816256761551,
      "rewards/margins": 0.05628550797700882,
      "rewards/rejected": -0.2762671411037445,
      "step": 20
    },
    {
      "epoch": 0.03309692671394799,
      "grad_norm": 0.9545782208442688,
      "learning_rate": 6.299212598425197e-07,
      "log_odds_chosen": 0.3836444914340973,
      "log_odds_ratio": -0.5285393595695496,
      "logits/chosen": -0.6870481371879578,
      "logits/rejected": 0.3343619704246521,
      "logps/chosen": -2.276167154312134,
      "logps/rejected": -2.6286871433258057,
      "loss": 2.5437,
      "nll_loss": 2.490856885910034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22761669754981995,
      "rewards/margins": 0.03525201976299286,
      "rewards/rejected": -0.262868732213974,
      "step": 21
    },
    {
      "epoch": 0.03467297084318361,
      "grad_norm": 0.8653661012649536,
      "learning_rate": 6.614173228346457e-07,
      "log_odds_chosen": 0.3887983560562134,
      "log_odds_ratio": -0.5376300811767578,
      "logits/chosen": -0.7233757972717285,
      "logits/rejected": 0.4169387221336365,
      "logps/chosen": -2.1862027645111084,
      "logps/rejected": -2.5423219203948975,
      "loss": 2.4644,
      "nll_loss": 2.410614490509033,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21862027049064636,
      "rewards/margins": 0.03561191260814667,
      "rewards/rejected": -0.25423216819763184,
      "step": 22
    },
    {
      "epoch": 0.036249014972419225,
      "grad_norm": 0.8709145784378052,
      "learning_rate": 6.929133858267716e-07,
      "log_odds_chosen": 0.4194765090942383,
      "log_odds_ratio": -0.5103728771209717,
      "logits/chosen": -0.7145715355873108,
      "logits/rejected": 0.29370206594467163,
      "logps/chosen": -2.231127977371216,
      "logps/rejected": -2.613429307937622,
      "loss": 2.4803,
      "nll_loss": 2.429227352142334,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2231128066778183,
      "rewards/margins": 0.03823015093803406,
      "rewards/rejected": -0.2613429129123688,
      "step": 23
    },
    {
      "epoch": 0.037825059101654845,
      "grad_norm": 0.8538913130760193,
      "learning_rate": 7.244094488188977e-07,
      "log_odds_chosen": 0.5760198831558228,
      "log_odds_ratio": -0.4734610915184021,
      "logits/chosen": -0.41316404938697815,
      "logits/rejected": 0.3446556031703949,
      "logps/chosen": -2.180354595184326,
      "logps/rejected": -2.710923671722412,
      "loss": 2.4467,
      "nll_loss": 2.3993289470672607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21803542971611023,
      "rewards/margins": 0.053056925535202026,
      "rewards/rejected": -0.27109238505363464,
      "step": 24
    },
    {
      "epoch": 0.039401103230890466,
      "grad_norm": 0.9757054448127747,
      "learning_rate": 7.559055118110236e-07,
      "log_odds_chosen": 0.35894954204559326,
      "log_odds_ratio": -0.5444148778915405,
      "logits/chosen": -0.7573986053466797,
      "logits/rejected": 0.08482818305492401,
      "logps/chosen": -2.2291455268859863,
      "logps/rejected": -2.5626137256622314,
      "loss": 2.4993,
      "nll_loss": 2.4448294639587402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22291454672813416,
      "rewards/margins": 0.033346809446811676,
      "rewards/rejected": -0.25626134872436523,
      "step": 25
    },
    {
      "epoch": 0.04097714736012608,
      "grad_norm": 0.9316413998603821,
      "learning_rate": 7.874015748031496e-07,
      "log_odds_chosen": 0.4797845780849457,
      "log_odds_ratio": -0.4976828992366791,
      "logits/chosen": -0.8583399653434753,
      "logits/rejected": 0.1226923018693924,
      "logps/chosen": -2.17708420753479,
      "logps/rejected": -2.617652177810669,
      "loss": 2.4456,
      "nll_loss": 2.3958029747009277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21770845353603363,
      "rewards/margins": 0.044056810438632965,
      "rewards/rejected": -0.2617652416229248,
      "step": 26
    },
    {
      "epoch": 0.0425531914893617,
      "grad_norm": 0.8912092447280884,
      "learning_rate": 8.188976377952755e-07,
      "log_odds_chosen": 0.8782732486724854,
      "log_odds_ratio": -0.38095948100090027,
      "logits/chosen": -0.6175351142883301,
      "logits/rejected": 0.3754832446575165,
      "logps/chosen": -2.209217071533203,
      "logps/rejected": -3.024287223815918,
      "loss": 2.4801,
      "nll_loss": 2.4420361518859863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22092171013355255,
      "rewards/margins": 0.08150701969861984,
      "rewards/rejected": -0.3024287223815918,
      "step": 27
    },
    {
      "epoch": 0.04412923561859732,
      "grad_norm": 0.9370722770690918,
      "learning_rate": 8.503937007874016e-07,
      "log_odds_chosen": 0.38346901535987854,
      "log_odds_ratio": -0.5323441624641418,
      "logits/chosen": -0.6895672082901001,
      "logits/rejected": 0.3445851802825928,
      "logps/chosen": -2.1683366298675537,
      "logps/rejected": -2.516519546508789,
      "loss": 2.4565,
      "nll_loss": 2.4032399654388428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2168336808681488,
      "rewards/margins": 0.03481828421354294,
      "rewards/rejected": -0.25165194272994995,
      "step": 28
    },
    {
      "epoch": 0.045705279747832936,
      "grad_norm": 0.8310018181800842,
      "learning_rate": 8.818897637795275e-07,
      "log_odds_chosen": 0.3637988865375519,
      "log_odds_ratio": -0.5361331105232239,
      "logits/chosen": -0.5572834610939026,
      "logits/rejected": 0.25494274497032166,
      "logps/chosen": -2.240579843521118,
      "logps/rejected": -2.57593035697937,
      "loss": 2.5104,
      "nll_loss": 2.4568018913269043,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22405798733234406,
      "rewards/margins": 0.03353505954146385,
      "rewards/rejected": -0.257593035697937,
      "step": 29
    },
    {
      "epoch": 0.04728132387706856,
      "grad_norm": 1.0258733034133911,
      "learning_rate": 9.133858267716535e-07,
      "log_odds_chosen": 0.6511096954345703,
      "log_odds_ratio": -0.4371214210987091,
      "logits/chosen": -0.6823744177818298,
      "logits/rejected": 0.13312608003616333,
      "logps/chosen": -2.2421658039093018,
      "logps/rejected": -2.842336654663086,
      "loss": 2.5426,
      "nll_loss": 2.49886155128479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22421656548976898,
      "rewards/margins": 0.06001708656549454,
      "rewards/rejected": -0.28423362970352173,
      "step": 30
    },
    {
      "epoch": 0.04885736800630418,
      "grad_norm": 0.9316391944885254,
      "learning_rate": 9.448818897637795e-07,
      "log_odds_chosen": 0.6505148410797119,
      "log_odds_ratio": -0.4398355185985565,
      "logits/chosen": -0.631649911403656,
      "logits/rejected": 0.41568225622177124,
      "logps/chosen": -2.274989604949951,
      "logps/rejected": -2.876878261566162,
      "loss": 2.5297,
      "nll_loss": 2.485715389251709,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22749897837638855,
      "rewards/margins": 0.06018887460231781,
      "rewards/rejected": -0.28768783807754517,
      "step": 31
    },
    {
      "epoch": 0.0504334121355398,
      "grad_norm": 0.7761370539665222,
      "learning_rate": 9.763779527559055e-07,
      "log_odds_chosen": 0.8358607292175293,
      "log_odds_ratio": -0.3823116719722748,
      "logits/chosen": -0.5432754158973694,
      "logits/rejected": 0.14401675760746002,
      "logps/chosen": -2.1370320320129395,
      "logps/rejected": -2.906792402267456,
      "loss": 2.3966,
      "nll_loss": 2.3583412170410156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2137032151222229,
      "rewards/margins": 0.07697603851556778,
      "rewards/rejected": -0.2906792461872101,
      "step": 32
    },
    {
      "epoch": 0.05200945626477541,
      "grad_norm": 0.7844627499580383,
      "learning_rate": 1.0078740157480315e-06,
      "log_odds_chosen": 0.6228745579719543,
      "log_odds_ratio": -0.4355998933315277,
      "logits/chosen": -0.5304853320121765,
      "logits/rejected": 0.25808629393577576,
      "logps/chosen": -2.138329029083252,
      "logps/rejected": -2.7067372798919678,
      "loss": 2.3954,
      "nll_loss": 2.351879119873047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21383287012577057,
      "rewards/margins": 0.056840844452381134,
      "rewards/rejected": -0.2706737220287323,
      "step": 33
    },
    {
      "epoch": 0.05358550039401103,
      "grad_norm": 0.8664770126342773,
      "learning_rate": 1.0393700787401573e-06,
      "log_odds_chosen": 0.3209899663925171,
      "log_odds_ratio": -0.5560799837112427,
      "logits/chosen": -0.529889702796936,
      "logits/rejected": 0.25037455558776855,
      "logps/chosen": -2.1071910858154297,
      "logps/rejected": -2.391819715499878,
      "loss": 2.3922,
      "nll_loss": 2.3365979194641113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21071910858154297,
      "rewards/margins": 0.028462877497076988,
      "rewards/rejected": -0.2391819953918457,
      "step": 34
    },
    {
      "epoch": 0.055161544523246654,
      "grad_norm": 1.0934066772460938,
      "learning_rate": 1.0708661417322836e-06,
      "log_odds_chosen": 0.39015841484069824,
      "log_odds_ratio": -0.5191880464553833,
      "logits/chosen": -0.7120888233184814,
      "logits/rejected": 0.5324327945709229,
      "logps/chosen": -2.3131232261657715,
      "logps/rejected": -2.6696863174438477,
      "loss": 2.587,
      "nll_loss": 2.535099506378174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2313123196363449,
      "rewards/margins": 0.03565630316734314,
      "rewards/rejected": -0.26696863770484924,
      "step": 35
    },
    {
      "epoch": 0.05673758865248227,
      "grad_norm": 0.9424160122871399,
      "learning_rate": 1.1023622047244094e-06,
      "log_odds_chosen": 0.6048139333724976,
      "log_odds_ratio": -0.4379541575908661,
      "logits/chosen": -0.8271058201789856,
      "logits/rejected": 0.32734885811805725,
      "logps/chosen": -2.1414098739624023,
      "logps/rejected": -2.6914708614349365,
      "loss": 2.4076,
      "nll_loss": 2.3637936115264893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21414096653461456,
      "rewards/margins": 0.055006109178066254,
      "rewards/rejected": -0.2691470980644226,
      "step": 36
    },
    {
      "epoch": 0.05831363278171789,
      "grad_norm": 0.8267931342124939,
      "learning_rate": 1.1338582677165354e-06,
      "log_odds_chosen": 0.511355996131897,
      "log_odds_ratio": -0.48152798414230347,
      "logits/chosen": -0.6461195945739746,
      "logits/rejected": 0.38446712493896484,
      "logps/chosen": -2.110015392303467,
      "logps/rejected": -2.5689022541046143,
      "loss": 2.3946,
      "nll_loss": 2.346407651901245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21100151538848877,
      "rewards/margins": 0.04588867723941803,
      "rewards/rejected": -0.256890207529068,
      "step": 37
    },
    {
      "epoch": 0.05988967691095351,
      "grad_norm": 0.9441820383071899,
      "learning_rate": 1.1653543307086612e-06,
      "log_odds_chosen": 0.4044821858406067,
      "log_odds_ratio": -0.5209547281265259,
      "logits/chosen": -0.7661936283111572,
      "logits/rejected": 0.18499918282032013,
      "logps/chosen": -2.1512036323547363,
      "logps/rejected": -2.5139970779418945,
      "loss": 2.4393,
      "nll_loss": 2.3872311115264893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2151203751564026,
      "rewards/margins": 0.036279335618019104,
      "rewards/rejected": -0.2513996958732605,
      "step": 38
    },
    {
      "epoch": 0.061465721040189124,
      "grad_norm": 1.0256890058517456,
      "learning_rate": 1.1968503937007875e-06,
      "log_odds_chosen": 0.16465213894844055,
      "log_odds_ratio": -0.6248909831047058,
      "logits/chosen": -0.6871511340141296,
      "logits/rejected": 0.38549405336380005,
      "logps/chosen": -2.2681920528411865,
      "logps/rejected": -2.4200401306152344,
      "loss": 2.5632,
      "nll_loss": 2.5007407665252686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2268192023038864,
      "rewards/margins": 0.015184784308075905,
      "rewards/rejected": -0.24200400710105896,
      "step": 39
    },
    {
      "epoch": 0.06304176516942474,
      "grad_norm": 0.8143957853317261,
      "learning_rate": 1.2283464566929133e-06,
      "log_odds_chosen": 0.38844311237335205,
      "log_odds_ratio": -0.5253883004188538,
      "logits/chosen": -0.675887405872345,
      "logits/rejected": 0.2226046621799469,
      "logps/chosen": -2.0888419151306152,
      "logps/rejected": -2.4366869926452637,
      "loss": 2.3744,
      "nll_loss": 2.321840763092041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20888419449329376,
      "rewards/margins": 0.034784507006406784,
      "rewards/rejected": -0.24366869032382965,
      "step": 40
    },
    {
      "epoch": 0.06461780929866036,
      "grad_norm": 0.8119162917137146,
      "learning_rate": 1.2598425196850393e-06,
      "log_odds_chosen": 0.20214533805847168,
      "log_odds_ratio": -0.6077091693878174,
      "logits/chosen": -0.6068657636642456,
      "logits/rejected": 0.30688706040382385,
      "logps/chosen": -2.137021780014038,
      "logps/rejected": -2.3217053413391113,
      "loss": 2.4076,
      "nll_loss": 2.3468475341796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21370220184326172,
      "rewards/margins": 0.018468374386429787,
      "rewards/rejected": -0.23217056691646576,
      "step": 41
    },
    {
      "epoch": 0.06619385342789598,
      "grad_norm": 0.8659912943840027,
      "learning_rate": 1.2913385826771652e-06,
      "log_odds_chosen": 0.29553914070129395,
      "log_odds_ratio": -0.5623306035995483,
      "logits/chosen": -0.7116361260414124,
      "logits/rejected": 0.290731281042099,
      "logps/chosen": -2.154006004333496,
      "logps/rejected": -2.42051362991333,
      "loss": 2.4304,
      "nll_loss": 2.3741416931152344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21540062129497528,
      "rewards/margins": 0.02665073424577713,
      "rewards/rejected": -0.24205134809017181,
      "step": 42
    },
    {
      "epoch": 0.0677698975571316,
      "grad_norm": 0.8237044215202332,
      "learning_rate": 1.3228346456692914e-06,
      "log_odds_chosen": 0.16473819315433502,
      "log_odds_ratio": -0.6206480264663696,
      "logits/chosen": -0.5807620286941528,
      "logits/rejected": 0.22030681371688843,
      "logps/chosen": -2.1601130962371826,
      "logps/rejected": -2.30804705619812,
      "loss": 2.4377,
      "nll_loss": 2.3756585121154785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21601131558418274,
      "rewards/margins": 0.014793392270803452,
      "rewards/rejected": -0.2308046966791153,
      "step": 43
    },
    {
      "epoch": 0.06934594168636722,
      "grad_norm": 0.9198518991470337,
      "learning_rate": 1.3543307086614172e-06,
      "log_odds_chosen": 0.5057801008224487,
      "log_odds_ratio": -0.486497700214386,
      "logits/chosen": -0.7227851152420044,
      "logits/rejected": 0.4384616017341614,
      "logps/chosen": -2.154522657394409,
      "logps/rejected": -2.6166176795959473,
      "loss": 2.4283,
      "nll_loss": 2.3796355724334717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21545226871967316,
      "rewards/margins": 0.04620949178934097,
      "rewards/rejected": -0.2616617679595947,
      "step": 44
    },
    {
      "epoch": 0.07092198581560284,
      "grad_norm": 0.7992345094680786,
      "learning_rate": 1.3858267716535433e-06,
      "log_odds_chosen": 0.6110447645187378,
      "log_odds_ratio": -0.4478919804096222,
      "logits/chosen": -0.5170164108276367,
      "logits/rejected": 0.38270628452301025,
      "logps/chosen": -2.089824676513672,
      "logps/rejected": -2.644026756286621,
      "loss": 2.3858,
      "nll_loss": 2.3409948348999023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20898248255252838,
      "rewards/margins": 0.05542018264532089,
      "rewards/rejected": -0.26440268754959106,
      "step": 45
    },
    {
      "epoch": 0.07249802994483845,
      "grad_norm": 1.0367867946624756,
      "learning_rate": 1.417322834645669e-06,
      "log_odds_chosen": 0.6249645352363586,
      "log_odds_ratio": -0.4582594037055969,
      "logits/chosen": -0.7600032687187195,
      "logits/rejected": 0.12511876225471497,
      "logps/chosen": -2.15745210647583,
      "logps/rejected": -2.7306129932403564,
      "loss": 2.4426,
      "nll_loss": 2.396728992462158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.215745210647583,
      "rewards/margins": 0.05731611326336861,
      "rewards/rejected": -0.2730613350868225,
      "step": 46
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 0.8897382020950317,
      "learning_rate": 1.4488188976377953e-06,
      "log_odds_chosen": 0.3489275276660919,
      "log_odds_ratio": -0.5626947283744812,
      "logits/chosen": -0.8047929406166077,
      "logits/rejected": 0.2395666539669037,
      "logps/chosen": -2.175475835800171,
      "logps/rejected": -2.4979097843170166,
      "loss": 2.4499,
      "nll_loss": 2.3936004638671875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21754758059978485,
      "rewards/margins": 0.032243404537439346,
      "rewards/rejected": -0.2497909665107727,
      "step": 47
    },
    {
      "epoch": 0.07565011820330969,
      "grad_norm": 1.0149489641189575,
      "learning_rate": 1.4803149606299211e-06,
      "log_odds_chosen": 0.43032294511795044,
      "log_odds_ratio": -0.5186169147491455,
      "logits/chosen": -0.6099531054496765,
      "logits/rejected": 0.2379540652036667,
      "logps/chosen": -2.2526159286499023,
      "logps/rejected": -2.6506638526916504,
      "loss": 2.5361,
      "nll_loss": 2.4842634201049805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2252615988254547,
      "rewards/margins": 0.03980477899312973,
      "rewards/rejected": -0.26506638526916504,
      "step": 48
    },
    {
      "epoch": 0.07722616233254531,
      "grad_norm": 0.8126904368400574,
      "learning_rate": 1.5118110236220472e-06,
      "log_odds_chosen": 0.5315326452255249,
      "log_odds_ratio": -0.47535181045532227,
      "logits/chosen": -0.6085718870162964,
      "logits/rejected": 0.533661425113678,
      "logps/chosen": -2.1099607944488525,
      "logps/rejected": -2.5920519828796387,
      "loss": 2.355,
      "nll_loss": 2.3075127601623535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2109961062669754,
      "rewards/margins": 0.04820908233523369,
      "rewards/rejected": -0.2592051923274994,
      "step": 49
    },
    {
      "epoch": 0.07880220646178093,
      "grad_norm": 0.8765050768852234,
      "learning_rate": 1.543307086614173e-06,
      "log_odds_chosen": 0.32944872975349426,
      "log_odds_ratio": -0.5625548362731934,
      "logits/chosen": -0.6740020513534546,
      "logits/rejected": 0.2358410358428955,
      "logps/chosen": -2.1824560165405273,
      "logps/rejected": -2.4877140522003174,
      "loss": 2.4638,
      "nll_loss": 2.407498836517334,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21824561059474945,
      "rewards/margins": 0.030525820329785347,
      "rewards/rejected": -0.24877141416072845,
      "step": 50
    },
    {
      "epoch": 0.08037825059101655,
      "grad_norm": 0.8369614481925964,
      "learning_rate": 1.5748031496062992e-06,
      "log_odds_chosen": 0.6268947720527649,
      "log_odds_ratio": -0.4370884597301483,
      "logits/chosen": -0.5486613512039185,
      "logits/rejected": 0.3741488754749298,
      "logps/chosen": -2.047851800918579,
      "logps/rejected": -2.6127448081970215,
      "loss": 2.3196,
      "nll_loss": 2.2759056091308594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20478516817092896,
      "rewards/margins": 0.05648929998278618,
      "rewards/rejected": -0.2612744867801666,
      "step": 51
    },
    {
      "epoch": 0.08195429472025216,
      "grad_norm": 0.848452627658844,
      "learning_rate": 1.6062992125984253e-06,
      "log_odds_chosen": 0.5586876273155212,
      "log_odds_ratio": -0.46114546060562134,
      "logits/chosen": -0.4986473619937897,
      "logits/rejected": 0.39973095059394836,
      "logps/chosen": -2.234095335006714,
      "logps/rejected": -2.7459263801574707,
      "loss": 2.4913,
      "nll_loss": 2.445152997970581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22340954840183258,
      "rewards/margins": 0.05118311941623688,
      "rewards/rejected": -0.27459266781806946,
      "step": 52
    },
    {
      "epoch": 0.08353033884948778,
      "grad_norm": 0.8976253271102905,
      "learning_rate": 1.637795275590551e-06,
      "log_odds_chosen": 0.31935352087020874,
      "log_odds_ratio": -0.5499067902565002,
      "logits/chosen": -0.5869264602661133,
      "logits/rejected": 0.4152706563472748,
      "logps/chosen": -2.1620757579803467,
      "logps/rejected": -2.449148654937744,
      "loss": 2.4373,
      "nll_loss": 2.38226580619812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2162075787782669,
      "rewards/margins": 0.028707284480333328,
      "rewards/rejected": -0.24491485953330994,
      "step": 53
    },
    {
      "epoch": 0.0851063829787234,
      "grad_norm": 0.8512571454048157,
      "learning_rate": 1.6692913385826771e-06,
      "log_odds_chosen": 0.288684606552124,
      "log_odds_ratio": -0.5614148378372192,
      "logits/chosen": -0.6818708181381226,
      "logits/rejected": 0.2810644507408142,
      "logps/chosen": -2.0992839336395264,
      "logps/rejected": -2.356931209564209,
      "loss": 2.3834,
      "nll_loss": 2.327263832092285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20992840826511383,
      "rewards/margins": 0.02576470375061035,
      "rewards/rejected": -0.23569311201572418,
      "step": 54
    },
    {
      "epoch": 0.08668242710795902,
      "grad_norm": 0.939967930316925,
      "learning_rate": 1.7007874015748031e-06,
      "log_odds_chosen": 0.40921398997306824,
      "log_odds_ratio": -0.5136203765869141,
      "logits/chosen": -0.6111175417900085,
      "logits/rejected": 0.1454327255487442,
      "logps/chosen": -2.2065820693969727,
      "logps/rejected": -2.579739570617676,
      "loss": 2.4918,
      "nll_loss": 2.4404354095458984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22065819799900055,
      "rewards/margins": 0.037315767258405685,
      "rewards/rejected": -0.2579739987850189,
      "step": 55
    },
    {
      "epoch": 0.08825847123719464,
      "grad_norm": 1.0213046073913574,
      "learning_rate": 1.7322834645669292e-06,
      "log_odds_chosen": 0.27949830889701843,
      "log_odds_ratio": -0.5767449140548706,
      "logits/chosen": -0.5307231545448303,
      "logits/rejected": 0.4212001860141754,
      "logps/chosen": -2.3242146968841553,
      "logps/rejected": -2.5813708305358887,
      "loss": 2.5978,
      "nll_loss": 2.540128707885742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23242147266864777,
      "rewards/margins": 0.025715606287121773,
      "rewards/rejected": -0.2581370770931244,
      "step": 56
    },
    {
      "epoch": 0.08983451536643026,
      "grad_norm": 0.7659716010093689,
      "learning_rate": 1.763779527559055e-06,
      "log_odds_chosen": 0.36814042925834656,
      "log_odds_ratio": -0.5291614532470703,
      "logits/chosen": -0.5885671973228455,
      "logits/rejected": 0.36917299032211304,
      "logps/chosen": -2.1409497261047363,
      "logps/rejected": -2.473041296005249,
      "loss": 2.4074,
      "nll_loss": 2.35445499420166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21409496665000916,
      "rewards/margins": 0.03320915997028351,
      "rewards/rejected": -0.24730412662029266,
      "step": 57
    },
    {
      "epoch": 0.09141055949566587,
      "grad_norm": 0.8884453773498535,
      "learning_rate": 1.795275590551181e-06,
      "log_odds_chosen": 0.3462219834327698,
      "log_odds_ratio": -0.5492972135543823,
      "logits/chosen": -0.58001309633255,
      "logits/rejected": 0.07347771525382996,
      "logps/chosen": -2.1368823051452637,
      "logps/rejected": -2.4556267261505127,
      "loss": 2.4149,
      "nll_loss": 2.359976053237915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2136882245540619,
      "rewards/margins": 0.031874436885118484,
      "rewards/rejected": -0.24556267261505127,
      "step": 58
    },
    {
      "epoch": 0.09298660362490149,
      "grad_norm": 0.9240288734436035,
      "learning_rate": 1.826771653543307e-06,
      "log_odds_chosen": 0.7683625221252441,
      "log_odds_ratio": -0.3947170376777649,
      "logits/chosen": -0.5223172903060913,
      "logits/rejected": 0.41549327969551086,
      "logps/chosen": -2.2466952800750732,
      "logps/rejected": -2.959190607070923,
      "loss": 2.4924,
      "nll_loss": 2.452920436859131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22466953098773956,
      "rewards/margins": 0.0712495669722557,
      "rewards/rejected": -0.2959190905094147,
      "step": 59
    },
    {
      "epoch": 0.09456264775413711,
      "grad_norm": 0.9423351883888245,
      "learning_rate": 1.858267716535433e-06,
      "log_odds_chosen": 0.5180408954620361,
      "log_odds_ratio": -0.48297566175460815,
      "logits/chosen": -0.5866909623146057,
      "logits/rejected": 0.16357183456420898,
      "logps/chosen": -2.1894171237945557,
      "logps/rejected": -2.6666948795318604,
      "loss": 2.4597,
      "nll_loss": 2.4114181995391846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21894173324108124,
      "rewards/margins": 0.04772777110338211,
      "rewards/rejected": -0.26666948199272156,
      "step": 60
    },
    {
      "epoch": 0.09613869188337273,
      "grad_norm": 0.9445580840110779,
      "learning_rate": 1.889763779527559e-06,
      "log_odds_chosen": 0.3669084906578064,
      "log_odds_ratio": -0.5591684579849243,
      "logits/chosen": -0.5270159840583801,
      "logits/rejected": 0.23815563321113586,
      "logps/chosen": -2.309187412261963,
      "logps/rejected": -2.6533546447753906,
      "loss": 2.5673,
      "nll_loss": 2.511385917663574,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2309187352657318,
      "rewards/margins": 0.03441672772169113,
      "rewards/rejected": -0.26533547043800354,
      "step": 61
    },
    {
      "epoch": 0.09771473601260836,
      "grad_norm": 1.0885119438171387,
      "learning_rate": 1.9212598425196847e-06,
      "log_odds_chosen": 0.24820619821548462,
      "log_odds_ratio": -0.5866947174072266,
      "logits/chosen": -0.6077854037284851,
      "logits/rejected": 0.2984403371810913,
      "logps/chosen": -2.3013339042663574,
      "logps/rejected": -2.5299530029296875,
      "loss": 2.5716,
      "nll_loss": 2.512897253036499,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23013338446617126,
      "rewards/margins": 0.022861897945404053,
      "rewards/rejected": -0.2529952824115753,
      "step": 62
    },
    {
      "epoch": 0.09929078014184398,
      "grad_norm": 0.8842782378196716,
      "learning_rate": 1.952755905511811e-06,
      "log_odds_chosen": 0.6375288963317871,
      "log_odds_ratio": -0.43413716554641724,
      "logits/chosen": -0.7599831819534302,
      "logits/rejected": 0.21808521449565887,
      "logps/chosen": -2.001779556274414,
      "logps/rejected": -2.5689616203308105,
      "loss": 2.2902,
      "nll_loss": 2.2467427253723145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20017798244953156,
      "rewards/margins": 0.05671820789575577,
      "rewards/rejected": -0.2568961977958679,
      "step": 63
    },
    {
      "epoch": 0.1008668242710796,
      "grad_norm": 0.8473801612854004,
      "learning_rate": 1.9842519685039368e-06,
      "log_odds_chosen": 0.7632179260253906,
      "log_odds_ratio": -0.4094430208206177,
      "logits/chosen": -0.6404256820678711,
      "logits/rejected": 0.3081037402153015,
      "logps/chosen": -2.0274438858032227,
      "logps/rejected": -2.7193148136138916,
      "loss": 2.3124,
      "nll_loss": 2.2714569568634033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20274437963962555,
      "rewards/margins": 0.06918711960315704,
      "rewards/rejected": -0.271931529045105,
      "step": 64
    },
    {
      "epoch": 0.1024428684003152,
      "grad_norm": 0.8429259657859802,
      "learning_rate": 2.015748031496063e-06,
      "log_odds_chosen": 0.6748821139335632,
      "log_odds_ratio": -0.41991835832595825,
      "logits/chosen": -0.6493370532989502,
      "logits/rejected": 0.1356709748506546,
      "logps/chosen": -2.115241765975952,
      "logps/rejected": -2.729600429534912,
      "loss": 2.3877,
      "nll_loss": 2.3457255363464355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21152418851852417,
      "rewards/margins": 0.061435848474502563,
      "rewards/rejected": -0.27296003699302673,
      "step": 65
    },
    {
      "epoch": 0.10401891252955082,
      "grad_norm": 0.8441055417060852,
      "learning_rate": 2.047244094488189e-06,
      "log_odds_chosen": 0.48374417424201965,
      "log_odds_ratio": -0.49196910858154297,
      "logits/chosen": -0.6177844405174255,
      "logits/rejected": 0.2689531445503235,
      "logps/chosen": -2.2088959217071533,
      "logps/rejected": -2.6486873626708984,
      "loss": 2.4956,
      "nll_loss": 2.446399688720703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2208895981311798,
      "rewards/margins": 0.04397913068532944,
      "rewards/rejected": -0.26486870646476746,
      "step": 66
    },
    {
      "epoch": 0.10559495665878645,
      "grad_norm": 0.8736575245857239,
      "learning_rate": 2.0787401574803147e-06,
      "log_odds_chosen": 0.5440582633018494,
      "log_odds_ratio": -0.4641622006893158,
      "logits/chosen": -0.5756601691246033,
      "logits/rejected": 0.4042266011238098,
      "logps/chosen": -2.1569972038269043,
      "logps/rejected": -2.6500601768493652,
      "loss": 2.4113,
      "nll_loss": 2.3649327754974365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2156997174024582,
      "rewards/margins": 0.04930633306503296,
      "rewards/rejected": -0.26500603556632996,
      "step": 67
    },
    {
      "epoch": 0.10717100078802207,
      "grad_norm": 1.0219923257827759,
      "learning_rate": 2.1102362204724405e-06,
      "log_odds_chosen": 0.38612306118011475,
      "log_odds_ratio": -0.5304967761039734,
      "logits/chosen": -0.7021428942680359,
      "logits/rejected": 0.2923763394355774,
      "logps/chosen": -2.1771302223205566,
      "logps/rejected": -2.529010057449341,
      "loss": 2.4909,
      "nll_loss": 2.437896966934204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21771301329135895,
      "rewards/margins": 0.0351879820227623,
      "rewards/rejected": -0.25290098786354065,
      "step": 68
    },
    {
      "epoch": 0.10874704491725769,
      "grad_norm": 1.0103472471237183,
      "learning_rate": 2.141732283464567e-06,
      "log_odds_chosen": 0.2671879529953003,
      "log_odds_ratio": -0.5846587419509888,
      "logits/chosen": -0.39670664072036743,
      "logits/rejected": 0.3414006531238556,
      "logps/chosen": -2.2567994594573975,
      "logps/rejected": -2.502344846725464,
      "loss": 2.5488,
      "nll_loss": 2.4903371334075928,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22567996382713318,
      "rewards/margins": 0.02455453760921955,
      "rewards/rejected": -0.2502345144748688,
      "step": 69
    },
    {
      "epoch": 0.11032308904649331,
      "grad_norm": 0.8793603777885437,
      "learning_rate": 2.173228346456693e-06,
      "log_odds_chosen": 0.4352240562438965,
      "log_odds_ratio": -0.505509614944458,
      "logits/chosen": -0.5309305787086487,
      "logits/rejected": 0.3599890172481537,
      "logps/chosen": -2.1654765605926514,
      "logps/rejected": -2.5600106716156006,
      "loss": 2.4516,
      "nll_loss": 2.4010517597198486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2165476679801941,
      "rewards/margins": 0.039453424513339996,
      "rewards/rejected": -0.2560010850429535,
      "step": 70
    },
    {
      "epoch": 0.11189913317572892,
      "grad_norm": 1.018066644668579,
      "learning_rate": 2.204724409448819e-06,
      "log_odds_chosen": 0.4932715892791748,
      "log_odds_ratio": -0.49797213077545166,
      "logits/chosen": -0.7851399183273315,
      "logits/rejected": 0.06452173739671707,
      "logps/chosen": -2.137401819229126,
      "logps/rejected": -2.58742618560791,
      "loss": 2.4109,
      "nll_loss": 2.3611340522766113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21374019980430603,
      "rewards/margins": 0.04500245302915573,
      "rewards/rejected": -0.25874263048171997,
      "step": 71
    },
    {
      "epoch": 0.11347517730496454,
      "grad_norm": 0.8898210525512695,
      "learning_rate": 2.2362204724409446e-06,
      "log_odds_chosen": 0.2936049997806549,
      "log_odds_ratio": -0.5717579126358032,
      "logits/chosen": -0.5596552491188049,
      "logits/rejected": 0.31953391432762146,
      "logps/chosen": -2.1422340869903564,
      "logps/rejected": -2.4082069396972656,
      "loss": 2.4216,
      "nll_loss": 2.3644003868103027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21422341465950012,
      "rewards/margins": 0.026597272604703903,
      "rewards/rejected": -0.24082070589065552,
      "step": 72
    },
    {
      "epoch": 0.11505122143420016,
      "grad_norm": 0.8397022485733032,
      "learning_rate": 2.267716535433071e-06,
      "log_odds_chosen": 0.37629467248916626,
      "log_odds_ratio": -0.526333749294281,
      "logits/chosen": -0.5056473612785339,
      "logits/rejected": 0.19583015143871307,
      "logps/chosen": -2.1670448780059814,
      "logps/rejected": -2.50854754447937,
      "loss": 2.4448,
      "nll_loss": 2.392120599746704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21670450270175934,
      "rewards/margins": 0.034150246530771255,
      "rewards/rejected": -0.2508547604084015,
      "step": 73
    },
    {
      "epoch": 0.11662726556343578,
      "grad_norm": 0.9695731401443481,
      "learning_rate": 2.2992125984251967e-06,
      "log_odds_chosen": 0.37078186869621277,
      "log_odds_ratio": -0.5291284918785095,
      "logits/chosen": -0.6568418741226196,
      "logits/rejected": 0.33567366003990173,
      "logps/chosen": -2.0653738975524902,
      "logps/rejected": -2.3936045169830322,
      "loss": 2.3679,
      "nll_loss": 2.3150012493133545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2065374106168747,
      "rewards/margins": 0.03282305598258972,
      "rewards/rejected": -0.23936045169830322,
      "step": 74
    },
    {
      "epoch": 0.1182033096926714,
      "grad_norm": 0.9203644394874573,
      "learning_rate": 2.3307086614173225e-06,
      "log_odds_chosen": 0.6003991961479187,
      "log_odds_ratio": -0.4472671151161194,
      "logits/chosen": -0.5809924006462097,
      "logits/rejected": 0.31674546003341675,
      "logps/chosen": -2.1095938682556152,
      "logps/rejected": -2.6563313007354736,
      "loss": 2.3819,
      "nll_loss": 2.337153434753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21095938980579376,
      "rewards/margins": 0.05467373877763748,
      "rewards/rejected": -0.26563313603401184,
      "step": 75
    },
    {
      "epoch": 0.11977935382190702,
      "grad_norm": 0.8541455864906311,
      "learning_rate": 2.3622047244094483e-06,
      "log_odds_chosen": 0.5599644184112549,
      "log_odds_ratio": -0.47028374671936035,
      "logits/chosen": -0.513896107673645,
      "logits/rejected": 0.44032493233680725,
      "logps/chosen": -2.139996290206909,
      "logps/rejected": -2.651939630508423,
      "loss": 2.3981,
      "nll_loss": 2.3510262966156006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21399962902069092,
      "rewards/margins": 0.05119434371590614,
      "rewards/rejected": -0.26519396901130676,
      "step": 76
    },
    {
      "epoch": 0.12135539795114263,
      "grad_norm": 0.9734304547309875,
      "learning_rate": 2.393700787401575e-06,
      "log_odds_chosen": 0.5413910150527954,
      "log_odds_ratio": -0.47939157485961914,
      "logits/chosen": -0.5768397450447083,
      "logits/rejected": 0.2747974097728729,
      "logps/chosen": -2.185297727584839,
      "logps/rejected": -2.681164026260376,
      "loss": 2.4654,
      "nll_loss": 2.4174275398254395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21852977573871613,
      "rewards/margins": 0.04958663508296013,
      "rewards/rejected": -0.26811644434928894,
      "step": 77
    },
    {
      "epoch": 0.12293144208037825,
      "grad_norm": 0.7510354518890381,
      "learning_rate": 2.425196850393701e-06,
      "log_odds_chosen": 0.4635729193687439,
      "log_odds_ratio": -0.5051595568656921,
      "logits/chosen": -0.3848555088043213,
      "logits/rejected": 0.07427231967449188,
      "logps/chosen": -2.1879336833953857,
      "logps/rejected": -2.612847089767456,
      "loss": 2.4399,
      "nll_loss": 2.389348268508911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21879339218139648,
      "rewards/margins": 0.04249133914709091,
      "rewards/rejected": -0.2612847089767456,
      "step": 78
    },
    {
      "epoch": 0.12450748620961387,
      "grad_norm": 0.9081109762191772,
      "learning_rate": 2.4566929133858266e-06,
      "log_odds_chosen": 0.5393069982528687,
      "log_odds_ratio": -0.46829110383987427,
      "logits/chosen": -0.5025306940078735,
      "logits/rejected": 0.21235036849975586,
      "logps/chosen": -2.1488659381866455,
      "logps/rejected": -2.639071464538574,
      "loss": 2.4115,
      "nll_loss": 2.3646249771118164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21488657593727112,
      "rewards/margins": 0.04902056232094765,
      "rewards/rejected": -0.26390716433525085,
      "step": 79
    },
    {
      "epoch": 0.12608353033884948,
      "grad_norm": 0.8297442197799683,
      "learning_rate": 2.488188976377953e-06,
      "log_odds_chosen": 0.644790530204773,
      "log_odds_ratio": -0.43552231788635254,
      "logits/chosen": -0.5664839744567871,
      "logits/rejected": 0.24572807550430298,
      "logps/chosen": -2.059072256088257,
      "logps/rejected": -2.6416335105895996,
      "loss": 2.321,
      "nll_loss": 2.2774367332458496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20590724050998688,
      "rewards/margins": 0.05825610086321831,
      "rewards/rejected": -0.2641633450984955,
      "step": 80
    },
    {
      "epoch": 0.1276595744680851,
      "grad_norm": 0.7417708039283752,
      "learning_rate": 2.5196850393700787e-06,
      "log_odds_chosen": 0.4702707529067993,
      "log_odds_ratio": -0.5074190497398376,
      "logits/chosen": -0.4309433698654175,
      "logits/rejected": 0.25342458486557007,
      "logps/chosen": -2.1097593307495117,
      "logps/rejected": -2.5410802364349365,
      "loss": 2.3641,
      "nll_loss": 2.31331729888916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21097594499588013,
      "rewards/margins": 0.04313211143016815,
      "rewards/rejected": -0.2541080713272095,
      "step": 81
    },
    {
      "epoch": 0.12923561859732072,
      "grad_norm": 0.7992884516716003,
      "learning_rate": 2.5511811023622045e-06,
      "log_odds_chosen": 0.6740604639053345,
      "log_odds_ratio": -0.43822091817855835,
      "logits/chosen": -0.5042348504066467,
      "logits/rejected": 0.23487010598182678,
      "logps/chosen": -2.0556299686431885,
      "logps/rejected": -2.6750376224517822,
      "loss": 2.3225,
      "nll_loss": 2.2787139415740967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2055629938840866,
      "rewards/margins": 0.061940789222717285,
      "rewards/rejected": -0.2675037682056427,
      "step": 82
    },
    {
      "epoch": 0.13081166272655634,
      "grad_norm": 0.7936323285102844,
      "learning_rate": 2.5826771653543303e-06,
      "log_odds_chosen": 0.33946871757507324,
      "log_odds_ratio": -0.5478945970535278,
      "logits/chosen": -0.3788740634918213,
      "logits/rejected": 0.251629114151001,
      "logps/chosen": -2.0991952419281006,
      "logps/rejected": -2.4062960147857666,
      "loss": 2.3638,
      "nll_loss": 2.309030532836914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2099195122718811,
      "rewards/margins": 0.030710091814398766,
      "rewards/rejected": -0.2406296283006668,
      "step": 83
    },
    {
      "epoch": 0.13238770685579196,
      "grad_norm": 0.9158631563186646,
      "learning_rate": 2.6141732283464566e-06,
      "log_odds_chosen": 0.6110659837722778,
      "log_odds_ratio": -0.4645346403121948,
      "logits/chosen": -0.5568953156471252,
      "logits/rejected": 0.15132026374340057,
      "logps/chosen": -2.069227933883667,
      "logps/rejected": -2.6256966590881348,
      "loss": 2.3621,
      "nll_loss": 2.315664052963257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20692278444766998,
      "rewards/margins": 0.0556468665599823,
      "rewards/rejected": -0.2625696659088135,
      "step": 84
    },
    {
      "epoch": 0.13396375098502758,
      "grad_norm": 0.7886312007904053,
      "learning_rate": 2.645669291338583e-06,
      "log_odds_chosen": 0.525355875492096,
      "log_odds_ratio": -0.4657125771045685,
      "logits/chosen": -0.37237846851348877,
      "logits/rejected": 0.3347388207912445,
      "logps/chosen": -2.123147487640381,
      "logps/rejected": -2.598121166229248,
      "loss": 2.3755,
      "nll_loss": 2.3288934230804443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21231473982334137,
      "rewards/margins": 0.04749736934900284,
      "rewards/rejected": -0.2598121166229248,
      "step": 85
    },
    {
      "epoch": 0.1355397951142632,
      "grad_norm": 0.8968652486801147,
      "learning_rate": 2.6771653543307086e-06,
      "log_odds_chosen": 0.5338146686553955,
      "log_odds_ratio": -0.46815913915634155,
      "logits/chosen": -0.6325598955154419,
      "logits/rejected": 0.32120490074157715,
      "logps/chosen": -2.123342752456665,
      "logps/rejected": -2.605854034423828,
      "loss": 2.3892,
      "nll_loss": 2.342399835586548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2123342901468277,
      "rewards/margins": 0.04825112968683243,
      "rewards/rejected": -0.2605854272842407,
      "step": 86
    },
    {
      "epoch": 0.13711583924349882,
      "grad_norm": 0.7727012634277344,
      "learning_rate": 2.7086614173228344e-06,
      "log_odds_chosen": 0.3835631310939789,
      "log_odds_ratio": -0.5259324312210083,
      "logits/chosen": -0.3659380078315735,
      "logits/rejected": 0.1596633940935135,
      "logps/chosen": -2.104921340942383,
      "logps/rejected": -2.4491097927093506,
      "loss": 2.3625,
      "nll_loss": 2.309898614883423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21049213409423828,
      "rewards/margins": 0.034418847411870956,
      "rewards/rejected": -0.24491098523139954,
      "step": 87
    },
    {
      "epoch": 0.13869188337273444,
      "grad_norm": 0.9119899868965149,
      "learning_rate": 2.7401574803149607e-06,
      "log_odds_chosen": 0.7017119526863098,
      "log_odds_ratio": -0.42803943157196045,
      "logits/chosen": -0.6377043724060059,
      "logits/rejected": 0.3170080780982971,
      "logps/chosen": -2.032348871231079,
      "logps/rejected": -2.671203136444092,
      "loss": 2.3239,
      "nll_loss": 2.2810964584350586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20323489606380463,
      "rewards/margins": 0.0638853907585144,
      "rewards/rejected": -0.2671203017234802,
      "step": 88
    },
    {
      "epoch": 0.14026792750197006,
      "grad_norm": 0.9140077233314514,
      "learning_rate": 2.7716535433070865e-06,
      "log_odds_chosen": 0.48430994153022766,
      "log_odds_ratio": -0.4811188280582428,
      "logits/chosen": -0.46536654233932495,
      "logits/rejected": 0.35523223876953125,
      "logps/chosen": -2.1665518283843994,
      "logps/rejected": -2.6059086322784424,
      "loss": 2.4371,
      "nll_loss": 2.3890204429626465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2166551947593689,
      "rewards/margins": 0.04393570125102997,
      "rewards/rejected": -0.26059091091156006,
      "step": 89
    },
    {
      "epoch": 0.14184397163120568,
      "grad_norm": 0.8198168277740479,
      "learning_rate": 2.8031496062992123e-06,
      "log_odds_chosen": 0.40435484051704407,
      "log_odds_ratio": -0.5177271962165833,
      "logits/chosen": -0.3415748178958893,
      "logits/rejected": 0.31405988335609436,
      "logps/chosen": -2.2057156562805176,
      "logps/rejected": -2.5755887031555176,
      "loss": 2.4455,
      "nll_loss": 2.3937432765960693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2205715775489807,
      "rewards/margins": 0.036987315863370895,
      "rewards/rejected": -0.2575588822364807,
      "step": 90
    },
    {
      "epoch": 0.1434200157604413,
      "grad_norm": 0.7300707101821899,
      "learning_rate": 2.834645669291338e-06,
      "log_odds_chosen": 0.5259378552436829,
      "log_odds_ratio": -0.4800778329372406,
      "logits/chosen": -0.3468290865421295,
      "logits/rejected": 0.19894689321517944,
      "logps/chosen": -1.9790496826171875,
      "logps/rejected": -2.4511303901672363,
      "loss": 2.2363,
      "nll_loss": 2.188265085220337,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19790495932102203,
      "rewards/margins": 0.04720810800790787,
      "rewards/rejected": -0.2451130747795105,
      "step": 91
    },
    {
      "epoch": 0.1449960598896769,
      "grad_norm": 0.7919360995292664,
      "learning_rate": 2.8661417322834644e-06,
      "log_odds_chosen": 0.5125716924667358,
      "log_odds_ratio": -0.4782637357711792,
      "logits/chosen": -0.4530889689922333,
      "logits/rejected": 0.2999853193759918,
      "logps/chosen": -2.0266165733337402,
      "logps/rejected": -2.486846685409546,
      "loss": 2.299,
      "nll_loss": 2.251190185546875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2026616334915161,
      "rewards/margins": 0.046023011207580566,
      "rewards/rejected": -0.24868465960025787,
      "step": 92
    },
    {
      "epoch": 0.14657210401891252,
      "grad_norm": 0.8349723815917969,
      "learning_rate": 2.8976377952755906e-06,
      "log_odds_chosen": 0.4769258499145508,
      "log_odds_ratio": -0.5019131898880005,
      "logits/chosen": -0.3912510275840759,
      "logits/rejected": 0.2896612584590912,
      "logps/chosen": -2.136798858642578,
      "logps/rejected": -2.5724754333496094,
      "loss": 2.3948,
      "nll_loss": 2.344578266143799,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21367989480495453,
      "rewards/margins": 0.043567635118961334,
      "rewards/rejected": -0.25724753737449646,
      "step": 93
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 0.7782561182975769,
      "learning_rate": 2.9291338582677165e-06,
      "log_odds_chosen": 0.46800780296325684,
      "log_odds_ratio": -0.4987240731716156,
      "logits/chosen": -0.3847990334033966,
      "logits/rejected": 0.16628870368003845,
      "logps/chosen": -2.1040778160095215,
      "logps/rejected": -2.523435592651367,
      "loss": 2.3559,
      "nll_loss": 2.306053400039673,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2104077935218811,
      "rewards/margins": 0.04193580150604248,
      "rewards/rejected": -0.2523435950279236,
      "step": 94
    },
    {
      "epoch": 0.14972419227738376,
      "grad_norm": 0.8788174986839294,
      "learning_rate": 2.9606299212598423e-06,
      "log_odds_chosen": 0.6574824452400208,
      "log_odds_ratio": -0.4303269386291504,
      "logits/chosen": -0.41694146394729614,
      "logits/rejected": 0.33073097467422485,
      "logps/chosen": -2.1245923042297363,
      "logps/rejected": -2.725464344024658,
      "loss": 2.3872,
      "nll_loss": 2.3441953659057617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21245922148227692,
      "rewards/margins": 0.06008722260594368,
      "rewards/rejected": -0.2725464701652527,
      "step": 95
    },
    {
      "epoch": 0.15130023640661938,
      "grad_norm": 0.7738668322563171,
      "learning_rate": 2.9921259842519685e-06,
      "log_odds_chosen": 0.6411773562431335,
      "log_odds_ratio": -0.4322814643383026,
      "logits/chosen": -0.45478928089141846,
      "logits/rejected": 0.2121216505765915,
      "logps/chosen": -2.096010684967041,
      "logps/rejected": -2.6793506145477295,
      "loss": 2.3287,
      "nll_loss": 2.285444498062134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20960105955600739,
      "rewards/margins": 0.05833399295806885,
      "rewards/rejected": -0.26793503761291504,
      "step": 96
    },
    {
      "epoch": 0.152876280535855,
      "grad_norm": 0.733166515827179,
      "learning_rate": 3.0236220472440943e-06,
      "log_odds_chosen": 0.35148048400878906,
      "log_odds_ratio": -0.5345249176025391,
      "logits/chosen": -0.4715975224971771,
      "logits/rejected": 0.29987144470214844,
      "logps/chosen": -2.0591440200805664,
      "logps/rejected": -2.3701171875,
      "loss": 2.3035,
      "nll_loss": 2.25003719329834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20591440796852112,
      "rewards/margins": 0.031097283586859703,
      "rewards/rejected": -0.23701170086860657,
      "step": 97
    },
    {
      "epoch": 0.15445232466509062,
      "grad_norm": 0.7708275318145752,
      "learning_rate": 3.05511811023622e-06,
      "log_odds_chosen": 0.43126457929611206,
      "log_odds_ratio": -0.5121496319770813,
      "logits/chosen": -0.2948766052722931,
      "logits/rejected": 0.15091395378112793,
      "logps/chosen": -2.189704656600952,
      "logps/rejected": -2.583603858947754,
      "loss": 2.4482,
      "nll_loss": 2.3969645500183105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21897049248218536,
      "rewards/margins": 0.03938991576433182,
      "rewards/rejected": -0.2583604156970978,
      "step": 98
    },
    {
      "epoch": 0.15602836879432624,
      "grad_norm": 0.8191169500350952,
      "learning_rate": 3.086614173228346e-06,
      "log_odds_chosen": 0.22818513214588165,
      "log_odds_ratio": -0.5883071422576904,
      "logits/chosen": -0.5125383138656616,
      "logits/rejected": 0.10425034165382385,
      "logps/chosen": -2.1129236221313477,
      "logps/rejected": -2.3160712718963623,
      "loss": 2.3859,
      "nll_loss": 2.3271055221557617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2112923562526703,
      "rewards/margins": 0.020314780995249748,
      "rewards/rejected": -0.23160713911056519,
      "step": 99
    },
    {
      "epoch": 0.15760441292356187,
      "grad_norm": 0.7705802321434021,
      "learning_rate": 3.1181102362204722e-06,
      "log_odds_chosen": 0.4004606008529663,
      "log_odds_ratio": -0.5196990370750427,
      "logits/chosen": -0.46414798498153687,
      "logits/rejected": 0.17729449272155762,
      "logps/chosen": -2.0139927864074707,
      "logps/rejected": -2.3726866245269775,
      "loss": 2.2862,
      "nll_loss": 2.234262466430664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20139926671981812,
      "rewards/margins": 0.035869404673576355,
      "rewards/rejected": -0.23726868629455566,
      "step": 100
    },
    {
      "epoch": 0.15918045705279749,
      "grad_norm": 0.7530562877655029,
      "learning_rate": 3.1496062992125985e-06,
      "log_odds_chosen": 0.43779271841049194,
      "log_odds_ratio": -0.5091973543167114,
      "logits/chosen": -0.3823246657848358,
      "logits/rejected": 0.2523665428161621,
      "logps/chosen": -2.008505344390869,
      "logps/rejected": -2.4028048515319824,
      "loss": 2.2858,
      "nll_loss": 2.2348995208740234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2008505016565323,
      "rewards/margins": 0.03942997381091118,
      "rewards/rejected": -0.24028049409389496,
      "step": 101
    },
    {
      "epoch": 0.1607565011820331,
      "grad_norm": 0.6781676411628723,
      "learning_rate": 3.1811023622047243e-06,
      "log_odds_chosen": 0.4132150709629059,
      "log_odds_ratio": -0.5262648463249207,
      "logits/chosen": -0.3420214354991913,
      "logits/rejected": 0.1129278838634491,
      "logps/chosen": -1.9598687887191772,
      "logps/rejected": -2.329220771789551,
      "loss": 2.2327,
      "nll_loss": 2.18007493019104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19598689675331116,
      "rewards/margins": 0.03693518415093422,
      "rewards/rejected": -0.23292206227779388,
      "step": 102
    },
    {
      "epoch": 0.16233254531126873,
      "grad_norm": 0.6758076548576355,
      "learning_rate": 3.2125984251968505e-06,
      "log_odds_chosen": 0.4648427963256836,
      "log_odds_ratio": -0.5020236968994141,
      "logits/chosen": -0.2520603537559509,
      "logits/rejected": 0.437377393245697,
      "logps/chosen": -1.937645673751831,
      "logps/rejected": -2.347379684448242,
      "loss": 2.2144,
      "nll_loss": 2.1642251014709473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1937645673751831,
      "rewards/margins": 0.040973398834466934,
      "rewards/rejected": -0.23473796248435974,
      "step": 103
    },
    {
      "epoch": 0.16390858944050432,
      "grad_norm": 0.7805861234664917,
      "learning_rate": 3.2440944881889763e-06,
      "log_odds_chosen": 0.4882758557796478,
      "log_odds_ratio": -0.5017855763435364,
      "logits/chosen": -0.20399755239486694,
      "logits/rejected": 0.5055577754974365,
      "logps/chosen": -2.1117475032806396,
      "logps/rejected": -2.558027744293213,
      "loss": 2.3732,
      "nll_loss": 2.3230180740356445,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21117472648620605,
      "rewards/margins": 0.044628046452999115,
      "rewards/rejected": -0.25580278038978577,
      "step": 104
    },
    {
      "epoch": 0.16548463356973994,
      "grad_norm": 0.6345953345298767,
      "learning_rate": 3.275590551181102e-06,
      "log_odds_chosen": 0.4859236478805542,
      "log_odds_ratio": -0.4872024357318878,
      "logits/chosen": -0.3714447617530823,
      "logits/rejected": 0.3421481251716614,
      "logps/chosen": -1.947684407234192,
      "logps/rejected": -2.3787930011749268,
      "loss": 2.2301,
      "nll_loss": 2.1814191341400146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19476845860481262,
      "rewards/margins": 0.04311085492372513,
      "rewards/rejected": -0.23787930607795715,
      "step": 105
    },
    {
      "epoch": 0.16706067769897556,
      "grad_norm": 0.6725803017616272,
      "learning_rate": 3.307086614173228e-06,
      "log_odds_chosen": 0.4803786873817444,
      "log_odds_ratio": -0.494854211807251,
      "logits/chosen": -0.38588568568229675,
      "logits/rejected": 0.39132970571517944,
      "logps/chosen": -1.9737625122070312,
      "logps/rejected": -2.3936586380004883,
      "loss": 2.2348,
      "nll_loss": 2.1852762699127197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19737625122070312,
      "rewards/margins": 0.041989609599113464,
      "rewards/rejected": -0.23936587572097778,
      "step": 106
    },
    {
      "epoch": 0.16863672182821118,
      "grad_norm": 0.6772481203079224,
      "learning_rate": 3.3385826771653542e-06,
      "log_odds_chosen": 0.4360727071762085,
      "log_odds_ratio": -0.5130836963653564,
      "logits/chosen": -0.22578240931034088,
      "logits/rejected": 0.21718156337738037,
      "logps/chosen": -2.05010986328125,
      "logps/rejected": -2.44297456741333,
      "loss": 2.3066,
      "nll_loss": 2.2553317546844482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20501098036766052,
      "rewards/margins": 0.03928646445274353,
      "rewards/rejected": -0.24429744482040405,
      "step": 107
    },
    {
      "epoch": 0.1702127659574468,
      "grad_norm": 0.6258271932601929,
      "learning_rate": 3.37007874015748e-06,
      "log_odds_chosen": 0.37870439887046814,
      "log_odds_ratio": -0.5316060185432434,
      "logits/chosen": -0.20526322722434998,
      "logits/rejected": 0.22974106669425964,
      "logps/chosen": -2.0181500911712646,
      "logps/rejected": -2.354243278503418,
      "loss": 2.2772,
      "nll_loss": 2.22402024269104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20181502401828766,
      "rewards/margins": 0.03360932320356369,
      "rewards/rejected": -0.23542433977127075,
      "step": 108
    },
    {
      "epoch": 0.17178881008668243,
      "grad_norm": 0.6688246130943298,
      "learning_rate": 3.4015748031496063e-06,
      "log_odds_chosen": 0.468779593706131,
      "log_odds_ratio": -0.4883277416229248,
      "logits/chosen": -0.12419804185628891,
      "logits/rejected": 0.2353970855474472,
      "logps/chosen": -2.091757297515869,
      "logps/rejected": -2.511913299560547,
      "loss": 2.3467,
      "nll_loss": 2.2978553771972656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20917576551437378,
      "rewards/margins": 0.042015574872493744,
      "rewards/rejected": -0.25119131803512573,
      "step": 109
    },
    {
      "epoch": 0.17336485421591805,
      "grad_norm": 0.6432668566703796,
      "learning_rate": 3.433070866141732e-06,
      "log_odds_chosen": 0.5067487955093384,
      "log_odds_ratio": -0.4804004430770874,
      "logits/chosen": -0.24377617239952087,
      "logits/rejected": 0.29760029911994934,
      "logps/chosen": -1.9592363834381104,
      "logps/rejected": -2.402676582336426,
      "loss": 2.1931,
      "nll_loss": 2.1450953483581543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19592364132404327,
      "rewards/margins": 0.04434400796890259,
      "rewards/rejected": -0.24026766419410706,
      "step": 110
    },
    {
      "epoch": 0.17494089834515367,
      "grad_norm": 0.7077887654304504,
      "learning_rate": 3.4645669291338583e-06,
      "log_odds_chosen": 0.5833423733711243,
      "log_odds_ratio": -0.4554339051246643,
      "logits/chosen": -0.22579139471054077,
      "logits/rejected": 0.24488507211208344,
      "logps/chosen": -2.0807836055755615,
      "logps/rejected": -2.604956865310669,
      "loss": 2.3392,
      "nll_loss": 2.2936172485351562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20807836949825287,
      "rewards/margins": 0.0524173304438591,
      "rewards/rejected": -0.26049572229385376,
      "step": 111
    },
    {
      "epoch": 0.1765169424743893,
      "grad_norm": 0.6585553288459778,
      "learning_rate": 3.496062992125984e-06,
      "log_odds_chosen": 0.3966713547706604,
      "log_odds_ratio": -0.5204365253448486,
      "logits/chosen": -0.16819843649864197,
      "logits/rejected": 0.3559090793132782,
      "logps/chosen": -2.107595205307007,
      "logps/rejected": -2.462603807449341,
      "loss": 2.3602,
      "nll_loss": 2.308168411254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2107595056295395,
      "rewards/margins": 0.035500891506671906,
      "rewards/rejected": -0.24626041948795319,
      "step": 112
    },
    {
      "epoch": 0.1780929866036249,
      "grad_norm": 0.6397602558135986,
      "learning_rate": 3.52755905511811e-06,
      "log_odds_chosen": 0.35250332951545715,
      "log_odds_ratio": -0.5411746501922607,
      "logits/chosen": -0.3075207769870758,
      "logits/rejected": 0.21061550080776215,
      "logps/chosen": -1.9689809083938599,
      "logps/rejected": -2.2822649478912354,
      "loss": 2.2367,
      "nll_loss": 2.1825618743896484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19689807295799255,
      "rewards/margins": 0.03132840245962143,
      "rewards/rejected": -0.22822648286819458,
      "step": 113
    },
    {
      "epoch": 0.17966903073286053,
      "grad_norm": 0.593964159488678,
      "learning_rate": 3.559055118110236e-06,
      "log_odds_chosen": 0.32149726152420044,
      "log_odds_ratio": -0.5685874223709106,
      "logits/chosen": -0.049146618694067,
      "logits/rejected": 0.2939941883087158,
      "logps/chosen": -2.0819308757781982,
      "logps/rejected": -2.375070333480835,
      "loss": 2.3471,
      "nll_loss": 2.290268898010254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2081930786371231,
      "rewards/margins": 0.029313962906599045,
      "rewards/rejected": -0.23750706017017365,
      "step": 114
    },
    {
      "epoch": 0.18124507486209615,
      "grad_norm": 0.5885688066482544,
      "learning_rate": 3.590551181102362e-06,
      "log_odds_chosen": 0.4819232225418091,
      "log_odds_ratio": -0.49382418394088745,
      "logits/chosen": -0.16471779346466064,
      "logits/rejected": 0.19575665891170502,
      "logps/chosen": -1.9886581897735596,
      "logps/rejected": -2.418565273284912,
      "loss": 2.2384,
      "nll_loss": 2.1889805793762207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1988658308982849,
      "rewards/margins": 0.04299072176218033,
      "rewards/rejected": -0.24185653030872345,
      "step": 115
    },
    {
      "epoch": 0.18282111899133174,
      "grad_norm": 0.5882607102394104,
      "learning_rate": 3.622047244094488e-06,
      "log_odds_chosen": 0.38364386558532715,
      "log_odds_ratio": -0.5387082695960999,
      "logits/chosen": -0.14860567450523376,
      "logits/rejected": 0.33340632915496826,
      "logps/chosen": -2.0445148944854736,
      "logps/rejected": -2.388934373855591,
      "loss": 2.2878,
      "nll_loss": 2.233961582183838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20445148646831512,
      "rewards/margins": 0.03444194793701172,
      "rewards/rejected": -0.23889344930648804,
      "step": 116
    },
    {
      "epoch": 0.18439716312056736,
      "grad_norm": 0.6176384091377258,
      "learning_rate": 3.653543307086614e-06,
      "log_odds_chosen": 0.5844452977180481,
      "log_odds_ratio": -0.4591362476348877,
      "logits/chosen": -0.27243772149086,
      "logits/rejected": 0.04437294602394104,
      "logps/chosen": -1.9680087566375732,
      "logps/rejected": -2.489656925201416,
      "loss": 2.2336,
      "nll_loss": 2.1876473426818848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19680088758468628,
      "rewards/margins": 0.05216481536626816,
      "rewards/rejected": -0.24896571040153503,
      "step": 117
    },
    {
      "epoch": 0.18597320724980299,
      "grad_norm": 0.5718716979026794,
      "learning_rate": 3.68503937007874e-06,
      "log_odds_chosen": 0.3214079439640045,
      "log_odds_ratio": -0.5579454302787781,
      "logits/chosen": -0.010645353235304356,
      "logits/rejected": 0.30515971779823303,
      "logps/chosen": -1.9919353723526,
      "logps/rejected": -2.2776200771331787,
      "loss": 2.254,
      "nll_loss": 2.1981608867645264,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19919352233409882,
      "rewards/margins": 0.028568467125296593,
      "rewards/rejected": -0.22776199877262115,
      "step": 118
    },
    {
      "epoch": 0.1875492513790386,
      "grad_norm": 0.5524156093597412,
      "learning_rate": 3.716535433070866e-06,
      "log_odds_chosen": 0.3362745940685272,
      "log_odds_ratio": -0.5541406869888306,
      "logits/chosen": -0.08601650595664978,
      "logits/rejected": 0.28649064898490906,
      "logps/chosen": -1.9557974338531494,
      "logps/rejected": -2.2547686100006104,
      "loss": 2.2373,
      "nll_loss": 2.1819238662719727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19557975232601166,
      "rewards/margins": 0.029897117987275124,
      "rewards/rejected": -0.22547689080238342,
      "step": 119
    },
    {
      "epoch": 0.18912529550827423,
      "grad_norm": 0.5379864573478699,
      "learning_rate": 3.748031496062992e-06,
      "log_odds_chosen": 0.6901645064353943,
      "log_odds_ratio": -0.42435628175735474,
      "logits/chosen": -0.12895634770393372,
      "logits/rejected": 0.31545358896255493,
      "logps/chosen": -1.8600318431854248,
      "logps/rejected": -2.4725053310394287,
      "loss": 2.1111,
      "nll_loss": 2.0686473846435547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1860032081604004,
      "rewards/margins": 0.06124735623598099,
      "rewards/rejected": -0.2472505420446396,
      "step": 120
    },
    {
      "epoch": 0.19070133963750985,
      "grad_norm": 0.5051740407943726,
      "learning_rate": 3.779527559055118e-06,
      "log_odds_chosen": 0.7074211835861206,
      "log_odds_ratio": -0.4035435616970062,
      "logits/chosen": -0.1733936369419098,
      "logits/rejected": 0.12479071319103241,
      "logps/chosen": -1.961637020111084,
      "logps/rejected": -2.5950496196746826,
      "loss": 2.1988,
      "nll_loss": 2.158412218093872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19616371393203735,
      "rewards/margins": 0.06334124505519867,
      "rewards/rejected": -0.25950494408607483,
      "step": 121
    },
    {
      "epoch": 0.19227738376674547,
      "grad_norm": 0.5642458200454712,
      "learning_rate": 3.8110236220472436e-06,
      "log_odds_chosen": 0.5054293870925903,
      "log_odds_ratio": -0.4897348880767822,
      "logits/chosen": -0.2511146068572998,
      "logits/rejected": 0.25759318470954895,
      "logps/chosen": -1.9019826650619507,
      "logps/rejected": -2.3513503074645996,
      "loss": 2.144,
      "nll_loss": 2.0950286388397217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19019825756549835,
      "rewards/margins": 0.04493675008416176,
      "rewards/rejected": -0.2351350039243698,
      "step": 122
    },
    {
      "epoch": 0.1938534278959811,
      "grad_norm": 0.546146035194397,
      "learning_rate": 3.8425196850393695e-06,
      "log_odds_chosen": 0.424507737159729,
      "log_odds_ratio": -0.5058774352073669,
      "logits/chosen": 0.04497145488858223,
      "logits/rejected": 0.20520615577697754,
      "logps/chosen": -1.9872440099716187,
      "logps/rejected": -2.363257884979248,
      "loss": 2.2507,
      "nll_loss": 2.2000772953033447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1987243890762329,
      "rewards/margins": 0.037601370364427567,
      "rewards/rejected": -0.23632577061653137,
      "step": 123
    },
    {
      "epoch": 0.1954294720252167,
      "grad_norm": 0.496383935213089,
      "learning_rate": 3.874015748031496e-06,
      "log_odds_chosen": 0.43429940938949585,
      "log_odds_ratio": -0.5118638277053833,
      "logits/chosen": -0.17999807000160217,
      "logits/rejected": 0.2584393322467804,
      "logps/chosen": -1.8671247959136963,
      "logps/rejected": -2.2387583255767822,
      "loss": 2.1116,
      "nll_loss": 2.0604166984558105,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18671248853206635,
      "rewards/margins": 0.03716333210468292,
      "rewards/rejected": -0.22387583553791046,
      "step": 124
    },
    {
      "epoch": 0.19700551615445233,
      "grad_norm": 0.5127514600753784,
      "learning_rate": 3.905511811023622e-06,
      "log_odds_chosen": 0.47707968950271606,
      "log_odds_ratio": -0.49198096990585327,
      "logits/chosen": -0.09298758208751678,
      "logits/rejected": 0.04988850653171539,
      "logps/chosen": -1.9040323495864868,
      "logps/rejected": -2.3241515159606934,
      "loss": 2.1671,
      "nll_loss": 2.117922306060791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19040323793888092,
      "rewards/margins": 0.04201190173625946,
      "rewards/rejected": -0.23241513967514038,
      "step": 125
    },
    {
      "epoch": 0.19858156028368795,
      "grad_norm": 0.5426633358001709,
      "learning_rate": 3.937007874015748e-06,
      "log_odds_chosen": 0.6238756775856018,
      "log_odds_ratio": -0.4400244355201721,
      "logits/chosen": -0.0192108117043972,
      "logits/rejected": 0.3376918137073517,
      "logps/chosen": -1.9439184665679932,
      "logps/rejected": -2.5035502910614014,
      "loss": 2.1878,
      "nll_loss": 2.1438095569610596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1943918615579605,
      "rewards/margins": 0.0559631809592247,
      "rewards/rejected": -0.2503550350666046,
      "step": 126
    },
    {
      "epoch": 0.20015760441292357,
      "grad_norm": 0.5588484406471252,
      "learning_rate": 3.9685039370078736e-06,
      "log_odds_chosen": 0.567841112613678,
      "log_odds_ratio": -0.46976813673973083,
      "logits/chosen": 0.013626057654619217,
      "logits/rejected": 0.3258940875530243,
      "logps/chosen": -1.9778722524642944,
      "logps/rejected": -2.4898061752319336,
      "loss": 2.2598,
      "nll_loss": 2.21280574798584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19778724014759064,
      "rewards/margins": 0.05119336396455765,
      "rewards/rejected": -0.2489805966615677,
      "step": 127
    },
    {
      "epoch": 0.2017336485421592,
      "grad_norm": 0.5180553793907166,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.2811572551727295,
      "log_odds_ratio": -0.5696135759353638,
      "logits/chosen": 0.08110462874174118,
      "logits/rejected": 0.487108051776886,
      "logps/chosen": -2.0393309593200684,
      "logps/rejected": -2.288515090942383,
      "loss": 2.2809,
      "nll_loss": 2.2239131927490234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20393308997154236,
      "rewards/margins": 0.02491840347647667,
      "rewards/rejected": -0.22885149717330933,
      "step": 128
    },
    {
      "epoch": 0.2033096926713948,
      "grad_norm": 0.5022251009941101,
      "learning_rate": 3.999992445477635e-06,
      "log_odds_chosen": 0.4684014916419983,
      "log_odds_ratio": -0.49844875931739807,
      "logits/chosen": 0.03743650019168854,
      "logits/rejected": 0.18749424815177917,
      "logps/chosen": -2.027505874633789,
      "logps/rejected": -2.4474422931671143,
      "loss": 2.2515,
      "nll_loss": 2.2016143798828125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.202750563621521,
      "rewards/margins": 0.041993647813797,
      "rewards/rejected": -0.2447442263364792,
      "step": 129
    },
    {
      "epoch": 0.2048857368006304,
      "grad_norm": 0.49709925055503845,
      "learning_rate": 3.999969781967615e-06,
      "log_odds_chosen": 0.30073270201683044,
      "log_odds_ratio": -0.5600437521934509,
      "logits/chosen": 0.047653649002313614,
      "logits/rejected": 0.35839155316352844,
      "logps/chosen": -2.123664617538452,
      "logps/rejected": -2.3944878578186035,
      "loss": 2.3459,
      "nll_loss": 2.2898778915405273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21236644685268402,
      "rewards/margins": 0.027082331478595734,
      "rewards/rejected": -0.23944880068302155,
      "step": 130
    },
    {
      "epoch": 0.20646178092986603,
      "grad_norm": 0.44330260157585144,
      "learning_rate": 3.99993200964115e-06,
      "log_odds_chosen": 0.3670748472213745,
      "log_odds_ratio": -0.5292910933494568,
      "logits/chosen": -0.04872098192572594,
      "logits/rejected": 0.13453775644302368,
      "logps/chosen": -1.9400205612182617,
      "logps/rejected": -2.261190414428711,
      "loss": 2.1811,
      "nll_loss": 2.1281275749206543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19400206208229065,
      "rewards/margins": 0.03211698681116104,
      "rewards/rejected": -0.2261190563440323,
      "step": 131
    },
    {
      "epoch": 0.20803782505910165,
      "grad_norm": 0.5485524535179138,
      "learning_rate": 3.99987912878359e-06,
      "log_odds_chosen": 0.1374671310186386,
      "log_odds_ratio": -0.634438693523407,
      "logits/chosen": 0.07270228117704391,
      "logits/rejected": 0.4620732367038727,
      "logps/chosen": -1.9368000030517578,
      "logps/rejected": -2.0512306690216064,
      "loss": 2.2276,
      "nll_loss": 2.1641933917999268,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19368000328540802,
      "rewards/margins": 0.011443082243204117,
      "rewards/rejected": -0.20512308180332184,
      "step": 132
    },
    {
      "epoch": 0.20961386918833727,
      "grad_norm": 0.4629489481449127,
      "learning_rate": 3.999811139794429e-06,
      "log_odds_chosen": 0.40406906604766846,
      "log_odds_ratio": -0.5154622793197632,
      "logits/chosen": 0.050318021327257156,
      "logits/rejected": 0.28723451495170593,
      "logps/chosen": -1.9487131834030151,
      "logps/rejected": -2.304403305053711,
      "loss": 2.1784,
      "nll_loss": 2.1268701553344727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19487132132053375,
      "rewards/margins": 0.03556900843977928,
      "rewards/rejected": -0.23044034838676453,
      "step": 133
    },
    {
      "epoch": 0.2111899133175729,
      "grad_norm": 0.4633968472480774,
      "learning_rate": 3.999728043187288e-06,
      "log_odds_chosen": 0.35710862278938293,
      "log_odds_ratio": -0.5401404500007629,
      "logits/chosen": 0.08867964148521423,
      "logits/rejected": 0.26978808641433716,
      "logps/chosen": -1.9941068887710571,
      "logps/rejected": -2.3115949630737305,
      "loss": 2.2544,
      "nll_loss": 2.2004222869873047,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19941069185733795,
      "rewards/margins": 0.03174880892038345,
      "rewards/rejected": -0.231159508228302,
      "step": 134
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 0.45478522777557373,
      "learning_rate": 3.999629839589922e-06,
      "log_odds_chosen": 0.26061490178108215,
      "log_odds_ratio": -0.5747462511062622,
      "logits/chosen": 0.1319434493780136,
      "logits/rejected": 0.09344829618930817,
      "logps/chosen": -2.0818891525268555,
      "logps/rejected": -2.3141493797302246,
      "loss": 2.3355,
      "nll_loss": 2.2779908180236816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20818890631198883,
      "rewards/margins": 0.02322601154446602,
      "rewards/rejected": -0.23141492903232574,
      "step": 135
    },
    {
      "epoch": 0.21434200157604413,
      "grad_norm": 0.4266544282436371,
      "learning_rate": 3.999516529744215e-06,
      "log_odds_chosen": 0.3937363922595978,
      "log_odds_ratio": -0.5289265513420105,
      "logits/chosen": 0.1900571584701538,
      "logits/rejected": 0.15361058712005615,
      "logps/chosen": -1.9516725540161133,
      "logps/rejected": -2.301271438598633,
      "loss": 2.1982,
      "nll_loss": 2.1452577114105225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19516725838184357,
      "rewards/margins": 0.03495991230010986,
      "rewards/rejected": -0.23012715578079224,
      "step": 136
    },
    {
      "epoch": 0.21591804570527975,
      "grad_norm": 0.43369004130363464,
      "learning_rate": 3.999388114506166e-06,
      "log_odds_chosen": 0.4344315826892853,
      "log_odds_ratio": -0.5094961524009705,
      "logits/chosen": 0.058895863592624664,
      "logits/rejected": 0.163179412484169,
      "logps/chosen": -1.844701886177063,
      "logps/rejected": -2.226527690887451,
      "loss": 2.0858,
      "nll_loss": 2.0348169803619385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18447020649909973,
      "rewards/margins": 0.03818256035447121,
      "rewards/rejected": -0.22265274822711945,
      "step": 137
    },
    {
      "epoch": 0.21749408983451538,
      "grad_norm": 0.42203620076179504,
      "learning_rate": 3.999244594845892e-06,
      "log_odds_chosen": 0.37491941452026367,
      "log_odds_ratio": -0.5347194671630859,
      "logits/chosen": 0.09584309160709381,
      "logits/rejected": 0.18220439553260803,
      "logps/chosen": -1.8879350423812866,
      "logps/rejected": -2.219787120819092,
      "loss": 2.1374,
      "nll_loss": 2.0839574337005615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18879351019859314,
      "rewards/margins": 0.03318518400192261,
      "rewards/rejected": -0.22197869420051575,
      "step": 138
    },
    {
      "epoch": 0.219070133963751,
      "grad_norm": 0.4215342104434967,
      "learning_rate": 3.999085971847616e-06,
      "log_odds_chosen": 0.2666381895542145,
      "log_odds_ratio": -0.579487144947052,
      "logits/chosen": 0.12761875987052917,
      "logits/rejected": 0.00037235766649246216,
      "logps/chosen": -1.945099115371704,
      "logps/rejected": -2.181492805480957,
      "loss": 2.1896,
      "nll_loss": 2.1316585540771484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19450990855693817,
      "rewards/margins": 0.023639345541596413,
      "rewards/rejected": -0.21814927458763123,
      "step": 139
    },
    {
      "epoch": 0.22064617809298662,
      "grad_norm": 0.4192501902580261,
      "learning_rate": 3.998912246709658e-06,
      "log_odds_chosen": 0.47762030363082886,
      "log_odds_ratio": -0.5048978328704834,
      "logits/chosen": 0.03178280219435692,
      "logits/rejected": 0.035381902009248734,
      "logps/chosen": -1.8466227054595947,
      "logps/rejected": -2.268432855606079,
      "loss": 2.0897,
      "nll_loss": 2.039208173751831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18466226756572723,
      "rewards/margins": 0.04218100756406784,
      "rewards/rejected": -0.22684328258037567,
      "step": 140
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 0.4541744589805603,
      "learning_rate": 3.9987234207444295e-06,
      "log_odds_chosen": 0.39642781019210815,
      "log_odds_ratio": -0.5283527374267578,
      "logits/chosen": 0.0909489318728447,
      "logits/rejected": 0.18739546835422516,
      "logps/chosen": -1.9560786485671997,
      "logps/rejected": -2.3097827434539795,
      "loss": 2.2049,
      "nll_loss": 2.1520698070526123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19560788571834564,
      "rewards/margins": 0.03537040203809738,
      "rewards/rejected": -0.23097828030586243,
      "step": 141
    },
    {
      "epoch": 0.22379826635145783,
      "grad_norm": 0.4312290847301483,
      "learning_rate": 3.998519495378419e-06,
      "log_odds_chosen": 0.3546152114868164,
      "log_odds_ratio": -0.5386707782745361,
      "logits/chosen": 0.18091727793216705,
      "logits/rejected": 0.33730435371398926,
      "logps/chosen": -2.0447652339935303,
      "logps/rejected": -2.3624267578125,
      "loss": 2.2652,
      "nll_loss": 2.2112996578216553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2044765055179596,
      "rewards/margins": 0.03176615759730339,
      "rewards/rejected": -0.2362426519393921,
      "step": 142
    },
    {
      "epoch": 0.22537431048069345,
      "grad_norm": 0.38687628507614136,
      "learning_rate": 3.998300472152187e-06,
      "log_odds_chosen": 0.2729724645614624,
      "log_odds_ratio": -0.5763713121414185,
      "logits/chosen": 0.03658361732959747,
      "logits/rejected": -0.03532174229621887,
      "logps/chosen": -1.8041529655456543,
      "logps/rejected": -2.040553092956543,
      "loss": 2.0568,
      "nll_loss": 1.9991142749786377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18041527271270752,
      "rewards/margins": 0.023640040308237076,
      "rewards/rejected": -0.2040553241968155,
      "step": 143
    },
    {
      "epoch": 0.22695035460992907,
      "grad_norm": 0.42334818840026855,
      "learning_rate": 3.998066352720347e-06,
      "log_odds_chosen": 0.4992850124835968,
      "log_odds_ratio": -0.481873095035553,
      "logits/chosen": 0.238363116979599,
      "logits/rejected": 0.23998966813087463,
      "logps/chosen": -1.9472233057022095,
      "logps/rejected": -2.3916687965393066,
      "loss": 2.1797,
      "nll_loss": 2.131523847579956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19472233951091766,
      "rewards/margins": 0.04444453865289688,
      "rewards/rejected": -0.23916688561439514,
      "step": 144
    },
    {
      "epoch": 0.2285263987391647,
      "grad_norm": 0.4056847393512726,
      "learning_rate": 3.997817138851562e-06,
      "log_odds_chosen": 0.3399352431297302,
      "log_odds_ratio": -0.5512011051177979,
      "logits/chosen": 0.1913917064666748,
      "logits/rejected": 0.20310351252555847,
      "logps/chosen": -2.0232315063476562,
      "logps/rejected": -2.327948570251465,
      "loss": 2.2587,
      "nll_loss": 2.203575372695923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20232316851615906,
      "rewards/margins": 0.030471697449684143,
      "rewards/rejected": -0.2327948659658432,
      "step": 145
    },
    {
      "epoch": 0.23010244286840031,
      "grad_norm": 0.390330046415329,
      "learning_rate": 3.997552832428522e-06,
      "log_odds_chosen": 0.4053119421005249,
      "log_odds_ratio": -0.5154557228088379,
      "logits/chosen": 0.05899511277675629,
      "logits/rejected": 0.17203162610530853,
      "logps/chosen": -1.8117233514785767,
      "logps/rejected": -2.1613264083862305,
      "loss": 2.0757,
      "nll_loss": 2.0241761207580566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18117234110832214,
      "rewards/margins": 0.03496028482913971,
      "rewards/rejected": -0.21613261103630066,
      "step": 146
    },
    {
      "epoch": 0.23167848699763594,
      "grad_norm": 0.3808334171772003,
      "learning_rate": 3.9972734354479366e-06,
      "log_odds_chosen": 0.36333751678466797,
      "log_odds_ratio": -0.5398846864700317,
      "logits/chosen": 0.1746935397386551,
      "logits/rejected": -0.06057439744472504,
      "logps/chosen": -1.7978198528289795,
      "logps/rejected": -2.109157085418701,
      "loss": 2.0488,
      "nll_loss": 1.994767189025879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1797819882631302,
      "rewards/margins": 0.031133724376559258,
      "rewards/rejected": -0.2109157145023346,
      "step": 147
    },
    {
      "epoch": 0.23325453112687156,
      "grad_norm": 0.4227812588214874,
      "learning_rate": 3.996978950020517e-06,
      "log_odds_chosen": 0.23635518550872803,
      "log_odds_ratio": -0.602321207523346,
      "logits/chosen": 0.30303770303726196,
      "logits/rejected": 0.26232045888900757,
      "logps/chosen": -2.013943910598755,
      "logps/rejected": -2.220210313796997,
      "loss": 2.2625,
      "nll_loss": 2.2022464275360107,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20139439404010773,
      "rewards/margins": 0.020626649260520935,
      "rewards/rejected": -0.22202105820178986,
      "step": 148
    },
    {
      "epoch": 0.23483057525610718,
      "grad_norm": 0.36287862062454224,
      "learning_rate": 3.996669378370959e-06,
      "log_odds_chosen": 0.601196825504303,
      "log_odds_ratio": -0.4497024714946747,
      "logits/chosen": 0.16156260669231415,
      "logits/rejected": 0.15708498656749725,
      "logps/chosen": -1.7733263969421387,
      "logps/rejected": -2.2993738651275635,
      "loss": 2.017,
      "nll_loss": 1.9719862937927246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17733265459537506,
      "rewards/margins": 0.052604734897613525,
      "rewards/rejected": -0.2299373894929886,
      "step": 149
    },
    {
      "epoch": 0.2364066193853428,
      "grad_norm": 0.4217900335788727,
      "learning_rate": 3.996344722837929e-06,
      "log_odds_chosen": 0.4981028735637665,
      "log_odds_ratio": -0.48042774200439453,
      "logits/chosen": 0.14997795224189758,
      "logits/rejected": 0.20861023664474487,
      "logps/chosen": -1.9517134428024292,
      "logps/rejected": -2.393977403640747,
      "loss": 2.1797,
      "nll_loss": 2.1316077709198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19517134130001068,
      "rewards/margins": 0.04422639310359955,
      "rewards/rejected": -0.23939774930477142,
      "step": 150
    },
    {
      "epoch": 0.23798266351457842,
      "grad_norm": 0.3475205898284912,
      "learning_rate": 3.996004985874043e-06,
      "log_odds_chosen": 0.2393593043088913,
      "log_odds_ratio": -0.5837403535842896,
      "logits/chosen": 0.2634306252002716,
      "logits/rejected": 0.15408116579055786,
      "logps/chosen": -1.8215903043746948,
      "logps/rejected": -2.025744676589966,
      "loss": 2.0775,
      "nll_loss": 2.0191402435302734,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18215903639793396,
      "rewards/margins": 0.020415417850017548,
      "rewards/rejected": -0.2025744616985321,
      "step": 151
    },
    {
      "epoch": 0.23955870764381404,
      "grad_norm": 0.3991820216178894,
      "learning_rate": 3.995650170045855e-06,
      "log_odds_chosen": 0.27918171882629395,
      "log_odds_ratio": -0.5713668465614319,
      "logits/chosen": 0.19363005459308624,
      "logits/rejected": 0.11847929656505585,
      "logps/chosen": -1.8987635374069214,
      "logps/rejected": -2.141249656677246,
      "loss": 2.1421,
      "nll_loss": 2.0849664211273193,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18987634778022766,
      "rewards/margins": 0.024248624220490456,
      "rewards/rejected": -0.21412499248981476,
      "step": 152
    },
    {
      "epoch": 0.24113475177304963,
      "grad_norm": 0.39748474955558777,
      "learning_rate": 3.995280278033825e-06,
      "log_odds_chosen": 0.16328397393226624,
      "log_odds_ratio": -0.6251372694969177,
      "logits/chosen": 0.26617446541786194,
      "logits/rejected": 0.23933932185173035,
      "logps/chosen": -2.0087883472442627,
      "logps/rejected": -2.1514151096343994,
      "loss": 2.2561,
      "nll_loss": 2.1936352252960205,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20087884366512299,
      "rewards/margins": 0.014262663200497627,
      "rewards/rejected": -0.21514150500297546,
      "step": 153
    },
    {
      "epoch": 0.24271079590228525,
      "grad_norm": 0.3707138001918793,
      "learning_rate": 3.994895312632314e-06,
      "log_odds_chosen": 0.6531109809875488,
      "log_odds_ratio": -0.4291936159133911,
      "logits/chosen": 0.3400770127773285,
      "logits/rejected": -0.1956482231616974,
      "logps/chosen": -1.8738765716552734,
      "logps/rejected": -2.452014446258545,
      "loss": 2.1097,
      "nll_loss": 2.066800594329834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1873876452445984,
      "rewards/margins": 0.05781375616788864,
      "rewards/rejected": -0.24520139396190643,
      "step": 154
    },
    {
      "epoch": 0.24428684003152087,
      "grad_norm": 0.3588172495365143,
      "learning_rate": 3.994495276749549e-06,
      "log_odds_chosen": 0.5185203552246094,
      "log_odds_ratio": -0.47327202558517456,
      "logits/chosen": 0.23319880664348602,
      "logits/rejected": 0.1430215835571289,
      "logps/chosen": -1.8845221996307373,
      "logps/rejected": -2.341402053833008,
      "loss": 2.1094,
      "nll_loss": 2.062074661254883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18845221400260925,
      "rewards/margins": 0.045687995851039886,
      "rewards/rejected": -0.23414020240306854,
      "step": 155
    },
    {
      "epoch": 0.2458628841607565,
      "grad_norm": 0.38072529435157776,
      "learning_rate": 3.994080173407612e-06,
      "log_odds_chosen": 0.2687567174434662,
      "log_odds_ratio": -0.5712822079658508,
      "logits/chosen": 0.2882783114910126,
      "logits/rejected": 0.13753741979599,
      "logps/chosen": -1.923572301864624,
      "logps/rejected": -2.1562812328338623,
      "loss": 2.1602,
      "nll_loss": 2.10306453704834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19235721230506897,
      "rewards/margins": 0.023270905017852783,
      "rewards/rejected": -0.21562813222408295,
      "step": 156
    },
    {
      "epoch": 0.24743892828999212,
      "grad_norm": 0.37946414947509766,
      "learning_rate": 3.993650005742409e-06,
      "log_odds_chosen": 0.27943286299705505,
      "log_odds_ratio": -0.5697324275970459,
      "logits/chosen": 0.22721070051193237,
      "logits/rejected": 0.17388474941253662,
      "logps/chosen": -1.9122453927993774,
      "logps/rejected": -2.155641555786133,
      "loss": 2.151,
      "nll_loss": 2.0940608978271484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19122453033924103,
      "rewards/margins": 0.02433963678777218,
      "rewards/rejected": -0.21556417644023895,
      "step": 157
    },
    {
      "epoch": 0.24901497241922774,
      "grad_norm": 0.3543010652065277,
      "learning_rate": 3.993204777003652e-06,
      "log_odds_chosen": 0.3954124450683594,
      "log_odds_ratio": -0.5226207375526428,
      "logits/chosen": 0.22299621999263763,
      "logits/rejected": 0.03944576531648636,
      "logps/chosen": -1.8209710121154785,
      "logps/rejected": -2.1653072834014893,
      "loss": 2.0513,
      "nll_loss": 1.9990218877792358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18209709227085114,
      "rewards/margins": 0.034433621913194656,
      "rewards/rejected": -0.2165307253599167,
      "step": 158
    },
    {
      "epoch": 0.25059101654846333,
      "grad_norm": 0.3736223876476288,
      "learning_rate": 3.992744490554832e-06,
      "log_odds_chosen": 0.38352489471435547,
      "log_odds_ratio": -0.5264863967895508,
      "logits/chosen": 0.3949679136276245,
      "logits/rejected": 0.19439592957496643,
      "logps/chosen": -1.8858449459075928,
      "logps/rejected": -2.221181631088257,
      "loss": 2.1334,
      "nll_loss": 2.08070969581604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18858450651168823,
      "rewards/margins": 0.033533647656440735,
      "rewards/rejected": -0.22211815416812897,
      "step": 159
    },
    {
      "epoch": 0.25216706067769895,
      "grad_norm": 0.3478967845439911,
      "learning_rate": 3.992269149873192e-06,
      "log_odds_chosen": 0.2978421151638031,
      "log_odds_ratio": -0.5619373917579651,
      "logits/chosen": 0.1769585907459259,
      "logits/rejected": 0.05092085525393486,
      "logps/chosen": -1.8340367078781128,
      "logps/rejected": -2.087353467941284,
      "loss": 2.0725,
      "nll_loss": 2.016280174255371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18340367078781128,
      "rewards/margins": 0.02533169835805893,
      "rewards/rejected": -0.2087353765964508,
      "step": 160
    },
    {
      "epoch": 0.25374310480693457,
      "grad_norm": 0.34116631746292114,
      "learning_rate": 3.991778758549705e-06,
      "log_odds_chosen": 0.2225290983915329,
      "log_odds_ratio": -0.5962461829185486,
      "logits/chosen": 0.4033081829547882,
      "logits/rejected": 0.2022503912448883,
      "logps/chosen": -1.9565954208374023,
      "logps/rejected": -2.15372633934021,
      "loss": 2.1758,
      "nll_loss": 2.1161608695983887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1956595480442047,
      "rewards/margins": 0.019713083282113075,
      "rewards/rejected": -0.21537262201309204,
      "step": 161
    },
    {
      "epoch": 0.2553191489361702,
      "grad_norm": 0.36636391282081604,
      "learning_rate": 3.9912733202890415e-06,
      "log_odds_chosen": 0.3752463161945343,
      "log_odds_ratio": -0.5345346927642822,
      "logits/chosen": 0.28393152356147766,
      "logits/rejected": 0.05336347967386246,
      "logps/chosen": -1.8976832628250122,
      "logps/rejected": -2.226705312728882,
      "loss": 2.1285,
      "nll_loss": 2.0750443935394287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18976832926273346,
      "rewards/margins": 0.03290221095085144,
      "rewards/rejected": -0.2226705402135849,
      "step": 162
    },
    {
      "epoch": 0.2568951930654058,
      "grad_norm": 0.3403889834880829,
      "learning_rate": 3.990752838909548e-06,
      "log_odds_chosen": 0.20657047629356384,
      "log_odds_ratio": -0.6020048260688782,
      "logits/chosen": 0.36604052782058716,
      "logits/rejected": 0.1470409482717514,
      "logps/chosen": -1.9812164306640625,
      "logps/rejected": -2.162238121032715,
      "loss": 2.2041,
      "nll_loss": 2.143928289413452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19812165200710297,
      "rewards/margins": 0.018102185800671577,
      "rewards/rejected": -0.2162238508462906,
      "step": 163
    },
    {
      "epoch": 0.25847123719464143,
      "grad_norm": 0.3617730736732483,
      "learning_rate": 3.990217318343213e-06,
      "log_odds_chosen": 0.6242728233337402,
      "log_odds_ratio": -0.4343191981315613,
      "logits/chosen": 0.33031949400901794,
      "logits/rejected": -0.01348424144089222,
      "logps/chosen": -2.0581400394439697,
      "logps/rejected": -2.6227612495422363,
      "loss": 2.2678,
      "nll_loss": 2.22434139251709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20581400394439697,
      "rewards/margins": 0.056462109088897705,
      "rewards/rejected": -0.2622761130332947,
      "step": 164
    },
    {
      "epoch": 0.26004728132387706,
      "grad_norm": 0.3566831946372986,
      "learning_rate": 3.989666762635637e-06,
      "log_odds_chosen": 0.2842005491256714,
      "log_odds_ratio": -0.5646076798439026,
      "logits/chosen": 0.3204185962677002,
      "logits/rejected": 0.14812150597572327,
      "logps/chosen": -1.8993555307388306,
      "logps/rejected": -2.145724296569824,
      "loss": 2.1492,
      "nll_loss": 2.0927205085754395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18993555009365082,
      "rewards/margins": 0.024636875838041306,
      "rewards/rejected": -0.21457241475582123,
      "step": 165
    },
    {
      "epoch": 0.2616233254531127,
      "grad_norm": 0.34587928652763367,
      "learning_rate": 3.9891011759460056e-06,
      "log_odds_chosen": 0.38712218403816223,
      "log_odds_ratio": -0.5237506628036499,
      "logits/chosen": 0.30775323510169983,
      "logits/rejected": 0.1324882060289383,
      "logps/chosen": -1.8900407552719116,
      "logps/rejected": -2.2226169109344482,
      "loss": 2.1266,
      "nll_loss": 2.0742549896240234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1890040785074234,
      "rewards/margins": 0.03325759619474411,
      "rewards/rejected": -0.2222616821527481,
      "step": 166
    },
    {
      "epoch": 0.2631993695823483,
      "grad_norm": 0.32432565093040466,
      "learning_rate": 3.988520562547057e-06,
      "log_odds_chosen": 0.4879869818687439,
      "log_odds_ratio": -0.49468889832496643,
      "logits/chosen": 0.3148024082183838,
      "logits/rejected": 0.025401366874575615,
      "logps/chosen": -1.8358017206192017,
      "logps/rejected": -2.2611985206604004,
      "loss": 2.0526,
      "nll_loss": 2.003171682357788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1835801899433136,
      "rewards/margins": 0.04253965616226196,
      "rewards/rejected": -0.22611983120441437,
      "step": 167
    },
    {
      "epoch": 0.2647754137115839,
      "grad_norm": 0.3420664668083191,
      "learning_rate": 3.987924926825047e-06,
      "log_odds_chosen": 0.5040227174758911,
      "log_odds_ratio": -0.4986487627029419,
      "logits/chosen": 0.3197277784347534,
      "logits/rejected": 0.055843938142061234,
      "logps/chosen": -1.9207936525344849,
      "logps/rejected": -2.368576765060425,
      "loss": 2.1598,
      "nll_loss": 2.1099777221679688,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1920793652534485,
      "rewards/margins": 0.04477829486131668,
      "rewards/rejected": -0.23685765266418457,
      "step": 168
    },
    {
      "epoch": 0.26635145784081954,
      "grad_norm": 0.3396807909011841,
      "learning_rate": 3.98731427327972e-06,
      "log_odds_chosen": 0.4706634283065796,
      "log_odds_ratio": -0.4895702004432678,
      "logits/chosen": 0.2792813777923584,
      "logits/rejected": -0.0839325338602066,
      "logps/chosen": -1.8512392044067383,
      "logps/rejected": -2.2608284950256348,
      "loss": 2.0695,
      "nll_loss": 2.020576238632202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18512390553951263,
      "rewards/margins": 0.0409589484333992,
      "rewards/rejected": -0.22608286142349243,
      "step": 169
    },
    {
      "epoch": 0.26792750197005516,
      "grad_norm": 0.32379212975502014,
      "learning_rate": 3.986688606524273e-06,
      "log_odds_chosen": 0.3634461760520935,
      "log_odds_ratio": -0.543432354927063,
      "logits/chosen": 0.3800685703754425,
      "logits/rejected": -0.3125303387641907,
      "logps/chosen": -1.8691346645355225,
      "logps/rejected": -2.190423011779785,
      "loss": 2.0918,
      "nll_loss": 2.0374691486358643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18691349029541016,
      "rewards/margins": 0.032128822058439255,
      "rewards/rejected": -0.21904230117797852,
      "step": 170
    },
    {
      "epoch": 0.2695035460992908,
      "grad_norm": 0.3257015347480774,
      "learning_rate": 3.986047931285315e-06,
      "log_odds_chosen": 0.6643016338348389,
      "log_odds_ratio": -0.42536190152168274,
      "logits/chosen": 0.2640674412250519,
      "logits/rejected": -0.16231495141983032,
      "logps/chosen": -1.7854793071746826,
      "logps/rejected": -2.365164279937744,
      "loss": 1.9975,
      "nll_loss": 1.9549524784088135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1785479485988617,
      "rewards/margins": 0.057968489825725555,
      "rewards/rejected": -0.23651644587516785,
      "step": 171
    },
    {
      "epoch": 0.2710795902285264,
      "grad_norm": 0.3187534511089325,
      "learning_rate": 3.985392252402847e-06,
      "log_odds_chosen": 0.21815888583660126,
      "log_odds_ratio": -0.5928647518157959,
      "logits/chosen": 0.24203622341156006,
      "logits/rejected": -0.04228699579834938,
      "logps/chosen": -1.77311110496521,
      "logps/rejected": -1.9570823907852173,
      "loss": 2.0022,
      "nll_loss": 1.942939043045044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17731110751628876,
      "rewards/margins": 0.018397128209471703,
      "rewards/rejected": -0.1957082450389862,
      "step": 172
    },
    {
      "epoch": 0.272655634357762,
      "grad_norm": 0.32601413130760193,
      "learning_rate": 3.984721574830206e-06,
      "log_odds_chosen": 0.2795952558517456,
      "log_odds_ratio": -0.566702127456665,
      "logits/chosen": 0.32929977774620056,
      "logits/rejected": -0.1160498857498169,
      "logps/chosen": -1.8864023685455322,
      "logps/rejected": -2.128418445587158,
      "loss": 2.1073,
      "nll_loss": 2.050626039505005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18864023685455322,
      "rewards/margins": 0.02420159988105297,
      "rewards/rejected": -0.21284182369709015,
      "step": 173
    },
    {
      "epoch": 0.27423167848699764,
      "grad_norm": 0.30667030811309814,
      "learning_rate": 3.984035903634041e-06,
      "log_odds_chosen": 0.3065723478794098,
      "log_odds_ratio": -0.560809850692749,
      "logits/chosen": 0.37695005536079407,
      "logits/rejected": -0.13857263326644897,
      "logps/chosen": -1.8567309379577637,
      "logps/rejected": -2.1228272914886475,
      "loss": 2.0624,
      "nll_loss": 2.0063276290893555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1856730878353119,
      "rewards/margins": 0.026609646156430244,
      "rewards/rejected": -0.21228273212909698,
      "step": 174
    },
    {
      "epoch": 0.27580772261623326,
      "grad_norm": 0.32427388429641724,
      "learning_rate": 3.983335243994273e-06,
      "log_odds_chosen": 0.47853925824165344,
      "log_odds_ratio": -0.4986327290534973,
      "logits/chosen": 0.38745561242103577,
      "logits/rejected": -0.0722002163529396,
      "logps/chosen": -1.8594340085983276,
      "logps/rejected": -2.2790563106536865,
      "loss": 2.0772,
      "nll_loss": 2.0272879600524902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18594340980052948,
      "rewards/margins": 0.04196225106716156,
      "rewards/rejected": -0.22790564596652985,
      "step": 175
    },
    {
      "epoch": 0.2773837667454689,
      "grad_norm": 0.3393646776676178,
      "learning_rate": 3.982619601204049e-06,
      "log_odds_chosen": 0.3384026885032654,
      "log_odds_ratio": -0.5461087226867676,
      "logits/chosen": 0.32837018370628357,
      "logits/rejected": -0.0760173350572586,
      "logps/chosen": -1.928531289100647,
      "logps/rejected": -2.2252705097198486,
      "loss": 2.1483,
      "nll_loss": 2.0936667919158936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1928531527519226,
      "rewards/margins": 0.029673907905817032,
      "rewards/rejected": -0.22252705693244934,
      "step": 176
    },
    {
      "epoch": 0.2789598108747045,
      "grad_norm": 0.30985409021377563,
      "learning_rate": 3.9818889806697085e-06,
      "log_odds_chosen": 0.417074590921402,
      "log_odds_ratio": -0.5178275108337402,
      "logits/chosen": 0.44880637526512146,
      "logits/rejected": -0.11007529497146606,
      "logps/chosen": -1.9631050825119019,
      "logps/rejected": -2.3332266807556152,
      "loss": 2.1563,
      "nll_loss": 2.1044764518737793,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19631052017211914,
      "rewards/margins": 0.037012144923210144,
      "rewards/rejected": -0.2333226501941681,
      "step": 177
    },
    {
      "epoch": 0.2805358550039401,
      "grad_norm": 0.3177052438259125,
      "learning_rate": 3.98114338791074e-06,
      "log_odds_chosen": 0.545357882976532,
      "log_odds_ratio": -0.4829763174057007,
      "logits/chosen": 0.3017576038837433,
      "logits/rejected": -0.15456542372703552,
      "logps/chosen": -1.8234596252441406,
      "logps/rejected": -2.3049373626708984,
      "loss": 2.047,
      "nll_loss": 1.9987092018127441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18234597146511078,
      "rewards/margins": 0.0481477752327919,
      "rewards/rejected": -0.2304937243461609,
      "step": 178
    },
    {
      "epoch": 0.28211189913317575,
      "grad_norm": 0.31176838278770447,
      "learning_rate": 3.980382828559742e-06,
      "log_odds_chosen": 0.3622802197933197,
      "log_odds_ratio": -0.5349768400192261,
      "logits/chosen": 0.3076533377170563,
      "logits/rejected": -0.15830329060554504,
      "logps/chosen": -1.8622955083847046,
      "logps/rejected": -2.1759631633758545,
      "loss": 2.0629,
      "nll_loss": 2.009382963180542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18622955679893494,
      "rewards/margins": 0.03136677294969559,
      "rewards/rejected": -0.21759635210037231,
      "step": 179
    },
    {
      "epoch": 0.28368794326241137,
      "grad_norm": 0.3124650716781616,
      "learning_rate": 3.9796073083623774e-06,
      "log_odds_chosen": 0.37124893069267273,
      "log_odds_ratio": -0.5417748093605042,
      "logits/chosen": 0.32632479071617126,
      "logits/rejected": -0.19913263618946075,
      "logps/chosen": -1.9367098808288574,
      "logps/rejected": -2.2628095149993896,
      "loss": 2.1354,
      "nll_loss": 2.08120059967041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19367098808288574,
      "rewards/margins": 0.0326099693775177,
      "rewards/rejected": -0.22628095746040344,
      "step": 180
    },
    {
      "epoch": 0.285263987391647,
      "grad_norm": 0.3195948302745819,
      "learning_rate": 3.978816833177329e-06,
      "log_odds_chosen": 0.3703266978263855,
      "log_odds_ratio": -0.5306106805801392,
      "logits/chosen": 0.42076432704925537,
      "logits/rejected": -0.02614203840494156,
      "logps/chosen": -1.8599867820739746,
      "logps/rejected": -2.1812684535980225,
      "loss": 2.0678,
      "nll_loss": 2.014716386795044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18599866330623627,
      "rewards/margins": 0.03212819620966911,
      "rewards/rejected": -0.21812686324119568,
      "step": 181
    },
    {
      "epoch": 0.2868400315208826,
      "grad_norm": 0.3143973648548126,
      "learning_rate": 3.978011408976261e-06,
      "log_odds_chosen": 0.33120378851890564,
      "log_odds_ratio": -0.5492135286331177,
      "logits/chosen": 0.3848988711833954,
      "logits/rejected": -0.14226217567920685,
      "logps/chosen": -1.8885282278060913,
      "logps/rejected": -2.1768221855163574,
      "loss": 2.1045,
      "nll_loss": 2.049586296081543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18885281682014465,
      "rewards/margins": 0.02882939763367176,
      "rewards/rejected": -0.21768222749233246,
      "step": 182
    },
    {
      "epoch": 0.28841607565011823,
      "grad_norm": 0.36059871315956116,
      "learning_rate": 3.9771910418437674e-06,
      "log_odds_chosen": 0.2940455675125122,
      "log_odds_ratio": -0.5578598976135254,
      "logits/chosen": 0.38924431800842285,
      "logits/rejected": 0.05839519202709198,
      "logps/chosen": -1.926640272140503,
      "logps/rejected": -2.183753728866577,
      "loss": 2.1334,
      "nll_loss": 2.0775983333587646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1926640272140503,
      "rewards/margins": 0.02571135386824608,
      "rewards/rejected": -0.21837536990642548,
      "step": 183
    },
    {
      "epoch": 0.2899921197793538,
      "grad_norm": 0.31750741600990295,
      "learning_rate": 3.976355737977332e-06,
      "log_odds_chosen": 0.3702757656574249,
      "log_odds_ratio": -0.5352032780647278,
      "logits/chosen": 0.39942264556884766,
      "logits/rejected": -0.12641888856887817,
      "logps/chosen": -1.8630297183990479,
      "logps/rejected": -2.1844112873077393,
      "loss": 2.0737,
      "nll_loss": 2.0201988220214844,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18630295991897583,
      "rewards/margins": 0.032138168811798096,
      "rewards/rejected": -0.2184411585330963,
      "step": 184
    },
    {
      "epoch": 0.2915681639085894,
      "grad_norm": 0.2944638133049011,
      "learning_rate": 3.975505503687274e-06,
      "log_odds_chosen": 0.31336691975593567,
      "log_odds_ratio": -0.5550580620765686,
      "logits/chosen": 0.43454480171203613,
      "logits/rejected": 0.02699423022568226,
      "logps/chosen": -1.8394372463226318,
      "logps/rejected": -2.110544204711914,
      "loss": 2.0454,
      "nll_loss": 1.9898747205734253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1839437335729599,
      "rewards/margins": 0.027110688388347626,
      "rewards/rejected": -0.21105441451072693,
      "step": 185
    },
    {
      "epoch": 0.29314420803782504,
      "grad_norm": 0.28209826350212097,
      "learning_rate": 3.974640345396708e-06,
      "log_odds_chosen": 0.5129460096359253,
      "log_odds_ratio": -0.4778994023799896,
      "logits/chosen": 0.2777142822742462,
      "logits/rejected": -0.47065022587776184,
      "logps/chosen": -1.806014060974121,
      "logps/rejected": -2.2508716583251953,
      "loss": 2.0079,
      "nll_loss": 1.9601199626922607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18060140311717987,
      "rewards/margins": 0.04448575899004936,
      "rewards/rejected": -0.22508715093135834,
      "step": 186
    },
    {
      "epoch": 0.29472025216706066,
      "grad_norm": 0.2899785339832306,
      "learning_rate": 3.9737602696414925e-06,
      "log_odds_chosen": 0.14901578426361084,
      "log_odds_ratio": -0.6323056221008301,
      "logits/chosen": 0.3966515362262726,
      "logits/rejected": -0.2466234564781189,
      "logps/chosen": -1.9014618396759033,
      "logps/rejected": -2.0296077728271484,
      "loss": 2.0863,
      "nll_loss": 2.0230603218078613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19014619290828705,
      "rewards/margins": 0.012814579531550407,
      "rewards/rejected": -0.2029607594013214,
      "step": 187
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 0.2821260690689087,
      "learning_rate": 3.972865283070179e-06,
      "log_odds_chosen": 0.5185416340827942,
      "log_odds_ratio": -0.4854019284248352,
      "logits/chosen": 0.35249727964401245,
      "logits/rejected": -0.3178330361843109,
      "logps/chosen": -1.7990273237228394,
      "logps/rejected": -2.2565367221832275,
      "loss": 2.0084,
      "nll_loss": 1.9598151445388794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17990271747112274,
      "rewards/margins": 0.0457509309053421,
      "rewards/rejected": -0.22565364837646484,
      "step": 188
    },
    {
      "epoch": 0.2978723404255319,
      "grad_norm": 0.2892349660396576,
      "learning_rate": 3.971955392443965e-06,
      "log_odds_chosen": 0.3767319917678833,
      "log_odds_ratio": -0.528398871421814,
      "logits/chosen": 0.39535295963287354,
      "logits/rejected": -0.14428094029426575,
      "logps/chosen": -1.8341965675354004,
      "logps/rejected": -2.162943124771118,
      "loss": 2.0321,
      "nll_loss": 1.9792832136154175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18341965973377228,
      "rewards/margins": 0.03287467360496521,
      "rewards/rejected": -0.21629434823989868,
      "step": 189
    },
    {
      "epoch": 0.2994483845547675,
      "grad_norm": 0.5001341104507446,
      "learning_rate": 3.971030604636637e-06,
      "log_odds_chosen": 0.2465718537569046,
      "log_odds_ratio": -0.5827715992927551,
      "logits/chosen": 0.4445749819278717,
      "logits/rejected": -0.11869044601917267,
      "logps/chosen": -1.8594155311584473,
      "logps/rejected": -2.070815324783325,
      "loss": 2.0563,
      "nll_loss": 1.9980722665786743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18594154715538025,
      "rewards/margins": 0.021139992401003838,
      "rewards/rejected": -0.20708155632019043,
      "step": 190
    },
    {
      "epoch": 0.30102442868400314,
      "grad_norm": 0.3018903434276581,
      "learning_rate": 3.970090926634526e-06,
      "log_odds_chosen": 0.31170162558555603,
      "log_odds_ratio": -0.5536643862724304,
      "logits/chosen": 0.49709048867225647,
      "logits/rejected": -0.3552049696445465,
      "logps/chosen": -1.9083786010742188,
      "logps/rejected": -2.1798431873321533,
      "loss": 2.1132,
      "nll_loss": 2.0578255653381348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19083787500858307,
      "rewards/margins": 0.027146445587277412,
      "rewards/rejected": -0.21798431873321533,
      "step": 191
    },
    {
      "epoch": 0.30260047281323876,
      "grad_norm": 0.3082922399044037,
      "learning_rate": 3.9691363655364526e-06,
      "log_odds_chosen": 0.41405007243156433,
      "log_odds_ratio": -0.5227676630020142,
      "logits/chosen": 0.4238288998603821,
      "logits/rejected": -0.18818999826908112,
      "logps/chosen": -1.8893095254898071,
      "logps/rejected": -2.254439353942871,
      "loss": 2.0958,
      "nll_loss": 2.0435335636138916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1889309585094452,
      "rewards/margins": 0.036512985825538635,
      "rewards/rejected": -0.22544392943382263,
      "step": 192
    },
    {
      "epoch": 0.3041765169424744,
      "grad_norm": 0.2771205008029938,
      "learning_rate": 3.968166928553666e-06,
      "log_odds_chosen": 0.32098525762557983,
      "log_odds_ratio": -0.5535095930099487,
      "logits/chosen": 0.44575732946395874,
      "logits/rejected": -0.3498935103416443,
      "logps/chosen": -1.8364524841308594,
      "logps/rejected": -2.111985445022583,
      "loss": 2.0581,
      "nll_loss": 2.002781391143799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18364526331424713,
      "rewards/margins": 0.02755330130457878,
      "rewards/rejected": -0.21119855344295502,
      "step": 193
    },
    {
      "epoch": 0.30575256107171,
      "grad_norm": 0.28915470838546753,
      "learning_rate": 3.967182623009804e-06,
      "log_odds_chosen": 0.4584325850009918,
      "log_odds_ratio": -0.4974234998226166,
      "logits/chosen": 0.39246881008148193,
      "logits/rejected": -0.22741518914699554,
      "logps/chosen": -1.8555591106414795,
      "logps/rejected": -2.257932424545288,
      "loss": 2.0439,
      "nll_loss": 1.9941781759262085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18555592000484467,
      "rewards/margins": 0.040237318724393845,
      "rewards/rejected": -0.22579322755336761,
      "step": 194
    },
    {
      "epoch": 0.3073286052009456,
      "grad_norm": 0.31561747193336487,
      "learning_rate": 3.966183456340821e-06,
      "log_odds_chosen": 0.20420649647712708,
      "log_odds_ratio": -0.5991290211677551,
      "logits/chosen": 0.4658154547214508,
      "logits/rejected": -0.2021852731704712,
      "logps/chosen": -1.907456874847412,
      "logps/rejected": -2.082449197769165,
      "loss": 2.1109,
      "nll_loss": 2.0510265827178955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19074568152427673,
      "rewards/margins": 0.01749921590089798,
      "rewards/rejected": -0.2082449197769165,
      "step": 195
    },
    {
      "epoch": 0.30890464933018125,
      "grad_norm": 0.31182610988616943,
      "learning_rate": 3.965169436094947e-06,
      "log_odds_chosen": 0.3316318392753601,
      "log_odds_ratio": -0.554113507270813,
      "logits/chosen": 0.46127960085868835,
      "logits/rejected": -0.33384498953819275,
      "logps/chosen": -1.7970610857009888,
      "logps/rejected": -2.084744453430176,
      "loss": 2.0109,
      "nll_loss": 1.955501914024353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17970611155033112,
      "rewards/margins": 0.028768306598067284,
      "rewards/rejected": -0.20847442746162415,
      "step": 196
    },
    {
      "epoch": 0.31048069345941687,
      "grad_norm": 0.2999548017978668,
      "learning_rate": 3.964140569932618e-06,
      "log_odds_chosen": 0.3349445164203644,
      "log_odds_ratio": -0.5604900121688843,
      "logits/chosen": 0.3906029164791107,
      "logits/rejected": -0.4700361490249634,
      "logps/chosen": -1.8970091342926025,
      "logps/rejected": -2.1866891384124756,
      "loss": 2.0889,
      "nll_loss": 2.032803773880005,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1897009164094925,
      "rewards/margins": 0.028968006372451782,
      "rewards/rejected": -0.21866890788078308,
      "step": 197
    },
    {
      "epoch": 0.3120567375886525,
      "grad_norm": 0.28372958302497864,
      "learning_rate": 3.9630968656264285e-06,
      "log_odds_chosen": 0.43718868494033813,
      "log_odds_ratio": -0.5063585042953491,
      "logits/chosen": 0.41493895649909973,
      "logits/rejected": -0.177265927195549,
      "logps/chosen": -1.820644497871399,
      "logps/rejected": -2.1998634338378906,
      "loss": 2.0096,
      "nll_loss": 1.959004521369934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1820644587278366,
      "rewards/margins": 0.037921883165836334,
      "rewards/rejected": -0.21998633444309235,
      "step": 198
    },
    {
      "epoch": 0.3136327817178881,
      "grad_norm": 0.30846837162971497,
      "learning_rate": 3.962038331061065e-06,
      "log_odds_chosen": 0.19350767135620117,
      "log_odds_ratio": -0.623175859451294,
      "logits/chosen": 0.5208650231361389,
      "logits/rejected": -0.31634849309921265,
      "logps/chosen": -1.888583779335022,
      "logps/rejected": -2.0645432472229004,
      "loss": 2.1024,
      "nll_loss": 2.0401272773742676,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18885837495326996,
      "rewards/margins": 0.017595931887626648,
      "rewards/rejected": -0.2064543068408966,
      "step": 199
    },
    {
      "epoch": 0.31520882584712373,
      "grad_norm": 0.30154353380203247,
      "learning_rate": 3.96096497423325e-06,
      "log_odds_chosen": 0.21430566906929016,
      "log_odds_ratio": -0.5975645184516907,
      "logits/chosen": 0.5608594417572021,
      "logits/rejected": 0.0020670928061008453,
      "logps/chosen": -1.7950149774551392,
      "logps/rejected": -1.978040099143982,
      "loss": 2.0287,
      "nll_loss": 1.9689255952835083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1795014888048172,
      "rewards/margins": 0.01830250397324562,
      "rewards/rejected": -0.19780398905277252,
      "step": 200
    },
    {
      "epoch": 0.31678486997635935,
      "grad_norm": 0.28481966257095337,
      "learning_rate": 3.959876803251684e-06,
      "log_odds_chosen": 0.3561375141143799,
      "log_odds_ratio": -0.5370092988014221,
      "logits/chosen": 0.3803846538066864,
      "logits/rejected": -0.2256171554327011,
      "logps/chosen": -1.7965466976165771,
      "logps/rejected": -2.102306365966797,
      "loss": 1.9801,
      "nll_loss": 1.9264239072799683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17965465784072876,
      "rewards/margins": 0.030575988814234734,
      "rewards/rejected": -0.21023064851760864,
      "step": 201
    },
    {
      "epoch": 0.31836091410559497,
      "grad_norm": 0.28623512387275696,
      "learning_rate": 3.958773826336977e-06,
      "log_odds_chosen": 0.23698343336582184,
      "log_odds_ratio": -0.5838424563407898,
      "logits/chosen": 0.49230071902275085,
      "logits/rejected": -0.10177897661924362,
      "logps/chosen": -1.84354829788208,
      "logps/rejected": -2.0470781326293945,
      "loss": 2.048,
      "nll_loss": 1.989640235900879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18435484170913696,
      "rewards/margins": 0.02035299874842167,
      "rewards/rejected": -0.20470783114433289,
      "step": 202
    },
    {
      "epoch": 0.3199369582348306,
      "grad_norm": 0.29585060477256775,
      "learning_rate": 3.957656051821592e-06,
      "log_odds_chosen": 0.25145307183265686,
      "log_odds_ratio": -0.5859741568565369,
      "logits/chosen": 0.539005696773529,
      "logits/rejected": -0.3879539668560028,
      "logps/chosen": -1.8778821229934692,
      "logps/rejected": -2.1008167266845703,
      "loss": 2.0589,
      "nll_loss": 2.0003015995025635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1877882331609726,
      "rewards/margins": 0.02229345589876175,
      "rewards/rejected": -0.21008168160915375,
      "step": 203
    },
    {
      "epoch": 0.3215130023640662,
      "grad_norm": 0.3141014575958252,
      "learning_rate": 3.956523488149783e-06,
      "log_odds_chosen": 0.35386207699775696,
      "log_odds_ratio": -0.5435526371002197,
      "logits/chosen": 0.5516231060028076,
      "logits/rejected": -0.11498698592185974,
      "logps/chosen": -1.9893256425857544,
      "logps/rejected": -2.303192138671875,
      "loss": 2.1781,
      "nll_loss": 2.1237025260925293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19893257319927216,
      "rewards/margins": 0.03138667345046997,
      "rewards/rejected": -0.23031923174858093,
      "step": 204
    },
    {
      "epoch": 0.32308904649330183,
      "grad_norm": 0.30985650420188904,
      "learning_rate": 3.9553761438775285e-06,
      "log_odds_chosen": 0.5383609533309937,
      "log_odds_ratio": -0.4667017161846161,
      "logits/chosen": 0.4218091368675232,
      "logits/rejected": -0.1754085123538971,
      "logps/chosen": -1.719486951828003,
      "logps/rejected": -2.184528112411499,
      "loss": 1.9139,
      "nll_loss": 1.8672513961791992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17194871604442596,
      "rewards/margins": 0.046504102647304535,
      "rewards/rejected": -0.2184528112411499,
      "step": 205
    },
    {
      "epoch": 0.32466509062253746,
      "grad_norm": 0.2680577039718628,
      "learning_rate": 3.954214027672465e-06,
      "log_odds_chosen": 0.5561447143554688,
      "log_odds_ratio": -0.46768665313720703,
      "logits/chosen": 0.3986890912055969,
      "logits/rejected": -0.5810946226119995,
      "logps/chosen": -1.7915854454040527,
      "logps/rejected": -2.280474901199341,
      "loss": 1.9719,
      "nll_loss": 1.9251141548156738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.179158553481102,
      "rewards/margins": 0.048888955265283585,
      "rewards/rejected": -0.22804749011993408,
      "step": 206
    },
    {
      "epoch": 0.3262411347517731,
      "grad_norm": 0.29800331592559814,
      "learning_rate": 3.953037148313825e-06,
      "log_odds_chosen": 0.2825692296028137,
      "log_odds_ratio": -0.5700769424438477,
      "logits/chosen": 0.5266653895378113,
      "logits/rejected": -0.22400972247123718,
      "logps/chosen": -1.8013321161270142,
      "logps/rejected": -2.048236131668091,
      "loss": 2.0184,
      "nll_loss": 1.9614076614379883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18013322353363037,
      "rewards/margins": 0.024690402671694756,
      "rewards/rejected": -0.20482362806797028,
      "step": 207
    },
    {
      "epoch": 0.32781717888100864,
      "grad_norm": 0.30844083428382874,
      "learning_rate": 3.951845514692371e-06,
      "log_odds_chosen": 0.36232998967170715,
      "log_odds_ratio": -0.5381972789764404,
      "logits/chosen": 0.4263172149658203,
      "logits/rejected": -0.17897063493728638,
      "logps/chosen": -1.856035828590393,
      "logps/rejected": -2.1717522144317627,
      "loss": 2.0606,
      "nll_loss": 2.0067667961120605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18560358881950378,
      "rewards/margins": 0.031571630388498306,
      "rewards/rejected": -0.2171752154827118,
      "step": 208
    },
    {
      "epoch": 0.32939322301024426,
      "grad_norm": 0.31168684363365173,
      "learning_rate": 3.950639135810325e-06,
      "log_odds_chosen": 0.36213281750679016,
      "log_odds_ratio": -0.5349076390266418,
      "logits/chosen": 0.5617266893386841,
      "logits/rejected": -0.08795854449272156,
      "logps/chosen": -1.9983832836151123,
      "logps/rejected": -2.31766414642334,
      "loss": 2.1698,
      "nll_loss": 2.1163430213928223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1998383104801178,
      "rewards/margins": 0.031928107142448425,
      "rewards/rejected": -0.23176643252372742,
      "step": 209
    },
    {
      "epoch": 0.3309692671394799,
      "grad_norm": 0.2974774241447449,
      "learning_rate": 3.9494180207813044e-06,
      "log_odds_chosen": 0.28252044320106506,
      "log_odds_ratio": -0.5777407884597778,
      "logits/chosen": 0.5435987710952759,
      "logits/rejected": -0.44823166728019714,
      "logps/chosen": -1.8489923477172852,
      "logps/rejected": -2.093599796295166,
      "loss": 2.0489,
      "nll_loss": 1.9911445379257202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1848992556333542,
      "rewards/margins": 0.024460751563310623,
      "rewards/rejected": -0.20935998857021332,
      "step": 210
    },
    {
      "epoch": 0.3325453112687155,
      "grad_norm": 0.29235726594924927,
      "learning_rate": 3.948182178830249e-06,
      "log_odds_chosen": 0.3494931757450104,
      "log_odds_ratio": -0.5623037815093994,
      "logits/chosen": 0.5011661648750305,
      "logits/rejected": -0.5379094481468201,
      "logps/chosen": -1.8723922967910767,
      "logps/rejected": -2.184168577194214,
      "loss": 2.0515,
      "nll_loss": 1.9952374696731567,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18723922967910767,
      "rewards/margins": 0.03117763251066208,
      "rewards/rejected": -0.21841685473918915,
      "step": 211
    },
    {
      "epoch": 0.3341213553979511,
      "grad_norm": 0.30690592527389526,
      "learning_rate": 3.9469316192933545e-06,
      "log_odds_chosen": 0.5762660503387451,
      "log_odds_ratio": -0.4586283564567566,
      "logits/chosen": 0.40061119198799133,
      "logits/rejected": -0.19122405350208282,
      "logps/chosen": -1.7368558645248413,
      "logps/rejected": -2.236180305480957,
      "loss": 1.934,
      "nll_loss": 1.8881280422210693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17368561029434204,
      "rewards/margins": 0.04993244633078575,
      "rewards/rejected": -0.2236180454492569,
      "step": 212
    },
    {
      "epoch": 0.33569739952718675,
      "grad_norm": 0.2981858253479004,
      "learning_rate": 3.945666351618001e-06,
      "log_odds_chosen": 0.24784377217292786,
      "log_odds_ratio": -0.5805960893630981,
      "logits/chosen": 0.4917218089103699,
      "logits/rejected": -0.2779148519039154,
      "logps/chosen": -1.7779290676116943,
      "logps/rejected": -1.9896657466888428,
      "loss": 1.9798,
      "nll_loss": 1.921708583831787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17779293656349182,
      "rewards/margins": 0.021173639222979546,
      "rewards/rejected": -0.19896656274795532,
      "step": 213
    },
    {
      "epoch": 0.33727344365642237,
      "grad_norm": 0.30391067266464233,
      "learning_rate": 3.9443863853626825e-06,
      "log_odds_chosen": 0.34453117847442627,
      "log_odds_ratio": -0.5483933091163635,
      "logits/chosen": 0.5546002388000488,
      "logits/rejected": -0.3275584876537323,
      "logps/chosen": -1.7622315883636475,
      "logps/rejected": -2.0517117977142334,
      "loss": 1.9813,
      "nll_loss": 1.9264605045318604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17622317373752594,
      "rewards/margins": 0.028948020190000534,
      "rewards/rejected": -0.20517119765281677,
      "step": 214
    },
    {
      "epoch": 0.338849487785658,
      "grad_norm": 0.2940407991409302,
      "learning_rate": 3.943091730196931e-06,
      "log_odds_chosen": 0.24225011467933655,
      "log_odds_ratio": -0.5915079712867737,
      "logits/chosen": 0.5921890139579773,
      "logits/rejected": -0.13432368636131287,
      "logps/chosen": -1.7511825561523438,
      "logps/rejected": -1.9615026712417603,
      "loss": 1.9606,
      "nll_loss": 1.9014678001403809,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17511828243732452,
      "rewards/margins": 0.02103200927376747,
      "rewards/rejected": -0.1961502581834793,
      "step": 215
    },
    {
      "epoch": 0.3404255319148936,
      "grad_norm": 0.3325921893119812,
      "learning_rate": 3.941782395901249e-06,
      "log_odds_chosen": 0.30871960520744324,
      "log_odds_ratio": -0.5597559213638306,
      "logits/chosen": 0.4807589054107666,
      "logits/rejected": -0.254393994808197,
      "logps/chosen": -1.8524887561798096,
      "logps/rejected": -2.1190786361694336,
      "loss": 2.0559,
      "nll_loss": 1.9999738931655884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18524888157844543,
      "rewards/margins": 0.026658987626433372,
      "rewards/rejected": -0.21190787851810455,
      "step": 216
    },
    {
      "epoch": 0.34200157604412923,
      "grad_norm": 0.29487061500549316,
      "learning_rate": 3.940458392367032e-06,
      "log_odds_chosen": 0.3391064703464508,
      "log_odds_ratio": -0.5431881546974182,
      "logits/chosen": 0.48744064569473267,
      "logits/rejected": -0.5134267807006836,
      "logps/chosen": -1.8746438026428223,
      "logps/rejected": -2.1684770584106445,
      "loss": 2.0436,
      "nll_loss": 1.989285945892334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1874643713235855,
      "rewards/margins": 0.029383335262537003,
      "rewards/rejected": -0.2168477326631546,
      "step": 217
    },
    {
      "epoch": 0.34357762017336485,
      "grad_norm": 0.29633522033691406,
      "learning_rate": 3.939119729596493e-06,
      "log_odds_chosen": 0.3622989058494568,
      "log_odds_ratio": -0.5331794619560242,
      "logits/chosen": 0.5500465631484985,
      "logits/rejected": -0.39332813024520874,
      "logps/chosen": -1.8827656507492065,
      "logps/rejected": -2.198401927947998,
      "loss": 2.0648,
      "nll_loss": 2.0114388465881348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18827655911445618,
      "rewards/margins": 0.0315636545419693,
      "rewards/rejected": -0.21984019875526428,
      "step": 218
    },
    {
      "epoch": 0.34515366430260047,
      "grad_norm": 0.2902742326259613,
      "learning_rate": 3.93776641770259e-06,
      "log_odds_chosen": 0.2742522060871124,
      "log_odds_ratio": -0.5714613199234009,
      "logits/chosen": 0.5202993154525757,
      "logits/rejected": -0.45154300332069397,
      "logps/chosen": -1.848337173461914,
      "logps/rejected": -2.0857441425323486,
      "loss": 2.0285,
      "nll_loss": 1.9713159799575806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18483375012874603,
      "rewards/margins": 0.023740684613585472,
      "rewards/rejected": -0.20857441425323486,
      "step": 219
    },
    {
      "epoch": 0.3467297084318361,
      "grad_norm": 0.29342028498649597,
      "learning_rate": 3.93639846690895e-06,
      "log_odds_chosen": 0.46571236848831177,
      "log_odds_ratio": -0.4907868206501007,
      "logits/chosen": 0.5409021973609924,
      "logits/rejected": -0.6227868795394897,
      "logps/chosen": -1.8621845245361328,
      "logps/rejected": -2.268977165222168,
      "loss": 2.0331,
      "nll_loss": 1.9839848279953003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1862184703350067,
      "rewards/margins": 0.040679264813661575,
      "rewards/rejected": -0.226897731423378,
      "step": 220
    },
    {
      "epoch": 0.3483057525610717,
      "grad_norm": 0.2885148823261261,
      "learning_rate": 3.9350158875497855e-06,
      "log_odds_chosen": 0.5439132452011108,
      "log_odds_ratio": -0.46580708026885986,
      "logits/chosen": 0.5373093485832214,
      "logits/rejected": -0.4585607051849365,
      "logps/chosen": -1.767594575881958,
      "logps/rejected": -2.2380287647247314,
      "loss": 1.956,
      "nll_loss": 1.909372329711914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17675945162773132,
      "rewards/margins": 0.04704343155026436,
      "rewards/rejected": -0.22380287945270538,
      "step": 221
    },
    {
      "epoch": 0.34988179669030733,
      "grad_norm": 0.2833883762359619,
      "learning_rate": 3.933618690069824e-06,
      "log_odds_chosen": 0.48881855607032776,
      "log_odds_ratio": -0.4842972159385681,
      "logits/chosen": 0.39756086468696594,
      "logits/rejected": -0.41850200295448303,
      "logps/chosen": -1.7153518199920654,
      "logps/rejected": -2.129936933517456,
      "loss": 1.9156,
      "nll_loss": 1.8671796321868896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1715351641178131,
      "rewards/margins": 0.04145851731300354,
      "rewards/rejected": -0.21299368143081665,
      "step": 222
    },
    {
      "epoch": 0.35145784081954295,
      "grad_norm": 0.2946211099624634,
      "learning_rate": 3.932206885024226e-06,
      "log_odds_chosen": 0.2477441281080246,
      "log_odds_ratio": -0.5846911072731018,
      "logits/chosen": 0.466760516166687,
      "logits/rejected": -0.3636370003223419,
      "logps/chosen": -1.8313201665878296,
      "logps/rejected": -2.0461394786834717,
      "loss": 2.005,
      "nll_loss": 1.946488857269287,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18313202261924744,
      "rewards/margins": 0.021481947973370552,
      "rewards/rejected": -0.20461396872997284,
      "step": 223
    },
    {
      "epoch": 0.3530338849487786,
      "grad_norm": 0.2853477895259857,
      "learning_rate": 3.930780483078502e-06,
      "log_odds_chosen": 0.34601038694381714,
      "log_odds_ratio": -0.5424416661262512,
      "logits/chosen": 0.43299296498298645,
      "logits/rejected": -0.43704044818878174,
      "logps/chosen": -1.7491978406906128,
      "logps/rejected": -2.0430784225463867,
      "loss": 1.9483,
      "nll_loss": 1.8940699100494385,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17491979897022247,
      "rewards/margins": 0.02938804030418396,
      "rewards/rejected": -0.20430783927440643,
      "step": 224
    },
    {
      "epoch": 0.3546099290780142,
      "grad_norm": 0.3010624647140503,
      "learning_rate": 3.92933949500844e-06,
      "log_odds_chosen": 0.3569672405719757,
      "log_odds_ratio": -0.548423171043396,
      "logits/chosen": 0.555142879486084,
      "logits/rejected": -0.394178569316864,
      "logps/chosen": -1.7384213209152222,
      "logps/rejected": -2.036893367767334,
      "loss": 1.9405,
      "nll_loss": 1.8856838941574097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17384213209152222,
      "rewards/margins": 0.029847197234630585,
      "rewards/rejected": -0.2036893218755722,
      "step": 225
    },
    {
      "epoch": 0.3561859732072498,
      "grad_norm": 0.28172364830970764,
      "learning_rate": 3.9278839317000155e-06,
      "log_odds_chosen": 0.4323083162307739,
      "log_odds_ratio": -0.5072457790374756,
      "logits/chosen": 0.5468685030937195,
      "logits/rejected": -0.37213990092277527,
      "logps/chosen": -1.7517664432525635,
      "logps/rejected": -2.123047113418579,
      "loss": 1.9389,
      "nll_loss": 1.888132929801941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1751766800880432,
      "rewards/margins": 0.03712804615497589,
      "rewards/rejected": -0.2123047113418579,
      "step": 226
    },
    {
      "epoch": 0.35776201733648544,
      "grad_norm": 0.28558024764060974,
      "learning_rate": 3.926413804149314e-06,
      "log_odds_chosen": 0.12638477981090546,
      "log_odds_ratio": -0.64253169298172,
      "logits/chosen": 0.5238491296768188,
      "logits/rejected": -0.41080769896507263,
      "logps/chosen": -1.8467971086502075,
      "logps/rejected": -1.955765724182129,
      "loss": 2.03,
      "nll_loss": 1.9657707214355469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18467971682548523,
      "rewards/margins": 0.010896885767579079,
      "rewards/rejected": -0.19557659327983856,
      "step": 227
    },
    {
      "epoch": 0.35933806146572106,
      "grad_norm": 0.2915712594985962,
      "learning_rate": 3.924929123462447e-06,
      "log_odds_chosen": 0.3050719201564789,
      "log_odds_ratio": -0.5605703592300415,
      "logits/chosen": 0.5805931687355042,
      "logits/rejected": -0.4595981538295746,
      "logps/chosen": -1.8690290451049805,
      "logps/rejected": -2.13301157951355,
      "loss": 2.0441,
      "nll_loss": 1.9880114793777466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18690288066864014,
      "rewards/margins": 0.026398256421089172,
      "rewards/rejected": -0.2133011519908905,
      "step": 228
    },
    {
      "epoch": 0.3609141055949567,
      "grad_norm": 0.29803889989852905,
      "learning_rate": 3.923429900855468e-06,
      "log_odds_chosen": 0.19842049479484558,
      "log_odds_ratio": -0.6108622550964355,
      "logits/chosen": 0.5528815984725952,
      "logits/rejected": -0.23056240379810333,
      "logps/chosen": -1.9518077373504639,
      "logps/rejected": -2.122875213623047,
      "loss": 2.1321,
      "nll_loss": 2.0710208415985107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1951807588338852,
      "rewards/margins": 0.01710677333176136,
      "rewards/rejected": -0.2122875452041626,
      "step": 229
    },
    {
      "epoch": 0.3624901497241923,
      "grad_norm": 0.2763267457485199,
      "learning_rate": 3.921916147654287e-06,
      "log_odds_chosen": 0.4757264256477356,
      "log_odds_ratio": -0.49002140760421753,
      "logits/chosen": 0.4894074499607086,
      "logits/rejected": -0.5041885375976562,
      "logps/chosen": -1.774259328842163,
      "logps/rejected": -2.182471513748169,
      "loss": 1.9376,
      "nll_loss": 1.8885548114776611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17742595076560974,
      "rewards/margins": 0.04082119092345238,
      "rewards/rejected": -0.21824713051319122,
      "step": 230
    },
    {
      "epoch": 0.3640661938534279,
      "grad_norm": 0.2874124348163605,
      "learning_rate": 3.920387875294588e-06,
      "log_odds_chosen": 0.30937978625297546,
      "log_odds_ratio": -0.5523663759231567,
      "logits/chosen": 0.5325709581375122,
      "logits/rejected": -0.34745681285858154,
      "logps/chosen": -1.7468411922454834,
      "logps/rejected": -2.0082056522369385,
      "loss": 1.9317,
      "nll_loss": 1.8764588832855225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17468410730361938,
      "rewards/margins": 0.026136459782719612,
      "rewards/rejected": -0.20082058012485504,
      "step": 231
    },
    {
      "epoch": 0.3656422379826635,
      "grad_norm": 0.2740110456943512,
      "learning_rate": 3.918845095321737e-06,
      "log_odds_chosen": 0.2758645713329315,
      "log_odds_ratio": -0.576305627822876,
      "logits/chosen": 0.45111507177352905,
      "logits/rejected": -0.5447308421134949,
      "logps/chosen": -1.7711262702941895,
      "logps/rejected": -1.999793529510498,
      "loss": 1.9468,
      "nll_loss": 1.8891706466674805,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1771126240491867,
      "rewards/margins": 0.02286672033369541,
      "rewards/rejected": -0.19997934997081757,
      "step": 232
    },
    {
      "epoch": 0.3672182821118991,
      "grad_norm": 0.30315297842025757,
      "learning_rate": 3.9172878193907004e-06,
      "log_odds_chosen": 0.24979126453399658,
      "log_odds_ratio": -0.5865897536277771,
      "logits/chosen": 0.4408178925514221,
      "logits/rejected": -0.430459588766098,
      "logps/chosen": -1.8613289594650269,
      "logps/rejected": -2.078932762145996,
      "loss": 2.0549,
      "nll_loss": 1.9962174892425537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18613289296627045,
      "rewards/margins": 0.02176038548350334,
      "rewards/rejected": -0.2078932672739029,
      "step": 233
    },
    {
      "epoch": 0.36879432624113473,
      "grad_norm": 0.28406012058258057,
      "learning_rate": 3.915716059265955e-06,
      "log_odds_chosen": 0.3215256929397583,
      "log_odds_ratio": -0.5549464225769043,
      "logits/chosen": 0.502613365650177,
      "logits/rejected": -0.5350136160850525,
      "logps/chosen": -1.8017627000808716,
      "logps/rejected": -2.079495668411255,
      "loss": 1.98,
      "nll_loss": 1.9245139360427856,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.180176243185997,
      "rewards/margins": 0.027773309499025345,
      "rewards/rejected": -0.20794956386089325,
      "step": 234
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 0.26154837012290955,
      "learning_rate": 3.9141298268213966e-06,
      "log_odds_chosen": 0.474556028842926,
      "log_odds_ratio": -0.4884706139564514,
      "logits/chosen": 0.5216464996337891,
      "logits/rejected": -0.4089938998222351,
      "logps/chosen": -1.7237547636032104,
      "logps/rejected": -2.131652355194092,
      "loss": 1.9058,
      "nll_loss": 1.8569673299789429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17237547039985657,
      "rewards/margins": 0.04078977555036545,
      "rewards/rejected": -0.21316523849964142,
      "step": 235
    },
    {
      "epoch": 0.37194641449960597,
      "grad_norm": 0.28945106267929077,
      "learning_rate": 3.912529134040255e-06,
      "log_odds_chosen": 0.2693881094455719,
      "log_odds_ratio": -0.5723139643669128,
      "logits/chosen": 0.5206422805786133,
      "logits/rejected": -0.3318684697151184,
      "logps/chosen": -1.8110253810882568,
      "logps/rejected": -2.0432167053222656,
      "loss": 1.9982,
      "nll_loss": 1.9410051107406616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18110254406929016,
      "rewards/margins": 0.023219116032123566,
      "rewards/rejected": -0.20432165265083313,
      "step": 236
    },
    {
      "epoch": 0.3735224586288416,
      "grad_norm": 0.2875734567642212,
      "learning_rate": 3.910913993014998e-06,
      "log_odds_chosen": 0.4889007806777954,
      "log_odds_ratio": -0.4840152859687805,
      "logits/chosen": 0.6125221252441406,
      "logits/rejected": -0.47014880180358887,
      "logps/chosen": -1.7982161045074463,
      "logps/rejected": -2.2228915691375732,
      "loss": 1.9734,
      "nll_loss": 1.9249656200408936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17982161045074463,
      "rewards/margins": 0.042467549443244934,
      "rewards/rejected": -0.22228917479515076,
      "step": 237
    },
    {
      "epoch": 0.3750985027580772,
      "grad_norm": 0.2845603823661804,
      "learning_rate": 3.909284415947246e-06,
      "log_odds_chosen": 0.4321085214614868,
      "log_odds_ratio": -0.508188009262085,
      "logits/chosen": 0.5699516534805298,
      "logits/rejected": -0.634801983833313,
      "logps/chosen": -1.8531954288482666,
      "logps/rejected": -2.2272119522094727,
      "loss": 2.013,
      "nll_loss": 1.9621832370758057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18531955778598785,
      "rewards/margins": 0.03740163892507553,
      "rewards/rejected": -0.2227211892604828,
      "step": 238
    },
    {
      "epoch": 0.37667454688731283,
      "grad_norm": 0.27208924293518066,
      "learning_rate": 3.907640415147674e-06,
      "log_odds_chosen": 0.5407861471176147,
      "log_odds_ratio": -0.4670700430870056,
      "logits/chosen": 0.45632970333099365,
      "logits/rejected": -0.6743687391281128,
      "logps/chosen": -1.7466856241226196,
      "logps/rejected": -2.2121920585632324,
      "loss": 1.9239,
      "nll_loss": 1.8772082328796387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1746685802936554,
      "rewards/margins": 0.046550631523132324,
      "rewards/rejected": -0.22121921181678772,
      "step": 239
    },
    {
      "epoch": 0.37825059101654845,
      "grad_norm": 0.28472790122032166,
      "learning_rate": 3.905982003035924e-06,
      "log_odds_chosen": 0.37944096326828003,
      "log_odds_ratio": -0.5271102786064148,
      "logits/chosen": 0.4610236883163452,
      "logits/rejected": -0.6050642132759094,
      "logps/chosen": -1.8401498794555664,
      "logps/rejected": -2.1687233448028564,
      "loss": 2.004,
      "nll_loss": 1.9512531757354736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18401499092578888,
      "rewards/margins": 0.03285733237862587,
      "rewards/rejected": -0.21687231957912445,
      "step": 240
    },
    {
      "epoch": 0.3798266351457841,
      "grad_norm": 0.28115785121917725,
      "learning_rate": 3.904309192140506e-06,
      "log_odds_chosen": 0.21689362823963165,
      "log_odds_ratio": -0.6001635789871216,
      "logits/chosen": 0.5072639584541321,
      "logits/rejected": -0.26205065846443176,
      "logps/chosen": -1.7706879377365112,
      "logps/rejected": -1.9564616680145264,
      "loss": 1.9545,
      "nll_loss": 1.8944413661956787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1770688146352768,
      "rewards/margins": 0.018577365204691887,
      "rewards/rejected": -0.19564618170261383,
      "step": 241
    },
    {
      "epoch": 0.3814026792750197,
      "grad_norm": 0.2975656986236572,
      "learning_rate": 3.90262199509871e-06,
      "log_odds_chosen": 0.26215988397598267,
      "log_odds_ratio": -0.5752044916152954,
      "logits/chosen": 0.4758797287940979,
      "logits/rejected": -0.33018162846565247,
      "logps/chosen": -1.8034093379974365,
      "logps/rejected": -2.0274529457092285,
      "loss": 1.9887,
      "nll_loss": 1.9311902523040771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1803409308195114,
      "rewards/margins": 0.022404363378882408,
      "rewards/rejected": -0.20274530351161957,
      "step": 242
    },
    {
      "epoch": 0.3829787234042553,
      "grad_norm": 0.2805575728416443,
      "learning_rate": 3.900920424656501e-06,
      "log_odds_chosen": 0.4157733917236328,
      "log_odds_ratio": -0.5154089331626892,
      "logits/chosen": 0.4855855107307434,
      "logits/rejected": -0.49067410826683044,
      "logps/chosen": -1.7054016590118408,
      "logps/rejected": -2.058197021484375,
      "loss": 1.8792,
      "nll_loss": 1.8276593685150146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17054018378257751,
      "rewards/margins": 0.0352795347571373,
      "rewards/rejected": -0.20581969618797302,
      "step": 243
    },
    {
      "epoch": 0.38455476753349094,
      "grad_norm": 0.32197999954223633,
      "learning_rate": 3.899204493668432e-06,
      "log_odds_chosen": 0.3415714502334595,
      "log_odds_ratio": -0.5509850978851318,
      "logits/chosen": 0.5429471731185913,
      "logits/rejected": -0.42629846930503845,
      "logps/chosen": -1.8829054832458496,
      "logps/rejected": -2.185755491256714,
      "loss": 2.0453,
      "nll_loss": 1.9901872873306274,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1882905513048172,
      "rewards/margins": 0.030284997075796127,
      "rewards/rejected": -0.21857555210590363,
      "step": 244
    },
    {
      "epoch": 0.38613081166272656,
      "grad_norm": 0.2792482376098633,
      "learning_rate": 3.897474215097542e-06,
      "log_odds_chosen": 0.3295517563819885,
      "log_odds_ratio": -0.5488483309745789,
      "logits/chosen": 0.5338771939277649,
      "logits/rejected": -0.7156805396080017,
      "logps/chosen": -1.8040363788604736,
      "logps/rejected": -2.0901222229003906,
      "loss": 1.99,
      "nll_loss": 1.9351005554199219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18040362000465393,
      "rewards/margins": 0.028608599677681923,
      "rewards/rejected": -0.2090122103691101,
      "step": 245
    },
    {
      "epoch": 0.3877068557919622,
      "grad_norm": 0.2792900502681732,
      "learning_rate": 3.8957296020152596e-06,
      "log_odds_chosen": 0.4721974730491638,
      "log_odds_ratio": -0.49195337295532227,
      "logits/chosen": 0.5625882148742676,
      "logits/rejected": -0.5468290448188782,
      "logps/chosen": -1.7749059200286865,
      "logps/rejected": -2.1837642192840576,
      "loss": 1.953,
      "nll_loss": 1.9038406610488892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17749059200286865,
      "rewards/margins": 0.04088583216071129,
      "rewards/rejected": -0.21837644279003143,
      "step": 246
    },
    {
      "epoch": 0.3892828999211978,
      "grad_norm": 0.29311904311180115,
      "learning_rate": 3.893970667601303e-06,
      "log_odds_chosen": 0.3301757276058197,
      "log_odds_ratio": -0.5501846075057983,
      "logits/chosen": 0.45822829008102417,
      "logits/rejected": -0.7882699966430664,
      "logps/chosen": -1.8771990537643433,
      "logps/rejected": -2.1641597747802734,
      "loss": 2.0396,
      "nll_loss": 1.984574317932129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18771988153457642,
      "rewards/margins": 0.028696071356534958,
      "rewards/rejected": -0.21641597151756287,
      "step": 247
    },
    {
      "epoch": 0.3908589440504334,
      "grad_norm": 0.2894313931465149,
      "learning_rate": 3.892197425143581e-06,
      "log_odds_chosen": 0.5370165705680847,
      "log_odds_ratio": -0.4752396047115326,
      "logits/chosen": 0.5151894092559814,
      "logits/rejected": -0.387676477432251,
      "logps/chosen": -1.799578309059143,
      "logps/rejected": -2.2680466175079346,
      "loss": 1.9568,
      "nll_loss": 1.9092282056808472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17995783686637878,
      "rewards/margins": 0.046846821904182434,
      "rewards/rejected": -0.22680465877056122,
      "step": 248
    },
    {
      "epoch": 0.39243498817966904,
      "grad_norm": 0.3017396926879883,
      "learning_rate": 3.890409888038094e-06,
      "log_odds_chosen": 0.3672383725643158,
      "log_odds_ratio": -0.5369208455085754,
      "logits/chosen": 0.5536250472068787,
      "logits/rejected": -0.3620547950267792,
      "logps/chosen": -1.8479626178741455,
      "logps/rejected": -2.171208143234253,
      "loss": 2.0219,
      "nll_loss": 1.9681875705718994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1847962588071823,
      "rewards/margins": 0.03232455253601074,
      "rewards/rejected": -0.21712081134319305,
      "step": 249
    },
    {
      "epoch": 0.39401103230890466,
      "grad_norm": 0.28757238388061523,
      "learning_rate": 3.888608069788831e-06,
      "log_odds_chosen": 0.28784364461898804,
      "log_odds_ratio": -0.575218141078949,
      "logits/chosen": 0.4429545998573303,
      "logits/rejected": -0.35244786739349365,
      "logps/chosen": -1.7926141023635864,
      "logps/rejected": -2.0435385704040527,
      "loss": 1.9702,
      "nll_loss": 1.912689208984375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17926141619682312,
      "rewards/margins": 0.02509242296218872,
      "rewards/rejected": -0.20435382425785065,
      "step": 250
    },
    {
      "epoch": 0.3955870764381403,
      "grad_norm": 0.26302191615104675,
      "learning_rate": 3.8867919840076685e-06,
      "log_odds_chosen": 0.4418867826461792,
      "log_odds_ratio": -0.5003976225852966,
      "logits/chosen": 0.46342819929122925,
      "logits/rejected": -0.6135963797569275,
      "logps/chosen": -1.7101153135299683,
      "logps/rejected": -2.0860018730163574,
      "loss": 1.8719,
      "nll_loss": 1.8218750953674316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17101150751113892,
      "rewards/margins": 0.0375886932015419,
      "rewards/rejected": -0.2086002230644226,
      "step": 251
    },
    {
      "epoch": 0.3971631205673759,
      "grad_norm": 0.28215491771698,
      "learning_rate": 3.884961644414267e-06,
      "log_odds_chosen": 0.33040478825569153,
      "log_odds_ratio": -0.5436782836914062,
      "logits/chosen": 0.4993625581264496,
      "logits/rejected": -0.1604970395565033,
      "logps/chosen": -1.8361539840698242,
      "logps/rejected": -2.1208314895629883,
      "loss": 1.98,
      "nll_loss": 1.9255832433700562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18361540138721466,
      "rewards/margins": 0.028467737138271332,
      "rewards/rejected": -0.2120831459760666,
      "step": 252
    },
    {
      "epoch": 0.3987391646966115,
      "grad_norm": 0.280631959438324,
      "learning_rate": 3.883117064835967e-06,
      "log_odds_chosen": 0.36852777004241943,
      "log_odds_ratio": -0.5398227572441101,
      "logits/chosen": 0.45098748803138733,
      "logits/rejected": -0.4938051402568817,
      "logps/chosen": -1.7849329710006714,
      "logps/rejected": -2.1051712036132812,
      "loss": 1.9618,
      "nll_loss": 1.9077905416488647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17849330604076385,
      "rewards/margins": 0.0320238396525383,
      "rewards/rejected": -0.21051713824272156,
      "step": 253
    },
    {
      "epoch": 0.40031520882584715,
      "grad_norm": 0.2820970416069031,
      "learning_rate": 3.881258259207688e-06,
      "log_odds_chosen": 0.30618369579315186,
      "log_odds_ratio": -0.5557839870452881,
      "logits/chosen": 0.549262523651123,
      "logits/rejected": -0.5373132228851318,
      "logps/chosen": -1.7485055923461914,
      "logps/rejected": -2.0079431533813477,
      "loss": 1.9015,
      "nll_loss": 1.845925211906433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17485055327415466,
      "rewards/margins": 0.025943776592612267,
      "rewards/rejected": -0.20079432427883148,
      "step": 254
    },
    {
      "epoch": 0.40189125295508277,
      "grad_norm": 0.26481202244758606,
      "learning_rate": 3.8793852415718165e-06,
      "log_odds_chosen": 0.44486790895462036,
      "log_odds_ratio": -0.49916309118270874,
      "logits/chosen": 0.4596560597419739,
      "logits/rejected": -0.7777143716812134,
      "logps/chosen": -1.6682631969451904,
      "logps/rejected": -2.0447134971618652,
      "loss": 1.8254,
      "nll_loss": 1.7754442691802979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16682633757591248,
      "rewards/margins": 0.03764501214027405,
      "rewards/rejected": -0.20447133481502533,
      "step": 255
    },
    {
      "epoch": 0.4034672970843184,
      "grad_norm": 0.27511125802993774,
      "learning_rate": 3.877498026078107e-06,
      "log_odds_chosen": 0.2643274664878845,
      "log_odds_ratio": -0.5784630179405212,
      "logits/chosen": 0.4659520089626312,
      "logits/rejected": -0.6729861497879028,
      "logps/chosen": -1.8191360235214233,
      "logps/rejected": -2.0481255054473877,
      "loss": 1.9784,
      "nll_loss": 1.9205960035324097,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1819136142730713,
      "rewards/margins": 0.02289893850684166,
      "rewards/rejected": -0.20481254160404205,
      "step": 256
    },
    {
      "epoch": 0.40504334121355395,
      "grad_norm": 0.28119486570358276,
      "learning_rate": 3.875596626983573e-06,
      "log_odds_chosen": 0.4398841857910156,
      "log_odds_ratio": -0.5096471309661865,
      "logits/chosen": 0.48892465233802795,
      "logits/rejected": -0.37292981147766113,
      "logps/chosen": -1.7355577945709229,
      "logps/rejected": -2.1096043586730957,
      "loss": 1.8938,
      "nll_loss": 1.842874526977539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17355579137802124,
      "rewards/margins": 0.037404656410217285,
      "rewards/rejected": -0.21096044778823853,
      "step": 257
    },
    {
      "epoch": 0.4066193853427896,
      "grad_norm": 0.2830592691898346,
      "learning_rate": 3.873681058652374e-06,
      "log_odds_chosen": 0.21046969294548035,
      "log_odds_ratio": -0.6032708883285522,
      "logits/chosen": 0.47122278809547424,
      "logits/rejected": -0.4699457287788391,
      "logps/chosen": -1.8599016666412354,
      "logps/rejected": -2.043604850769043,
      "loss": 2.02,
      "nll_loss": 1.9596831798553467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1859901398420334,
      "rewards/margins": 0.01837034709751606,
      "rewards/rejected": -0.2043604850769043,
      "step": 258
    },
    {
      "epoch": 0.4081954294720252,
      "grad_norm": 0.2882324457168579,
      "learning_rate": 3.871751335555715e-06,
      "log_odds_chosen": 0.3207879066467285,
      "log_odds_ratio": -0.551344633102417,
      "logits/chosen": 0.4617171585559845,
      "logits/rejected": -0.5876399874687195,
      "logps/chosen": -1.7937874794006348,
      "logps/rejected": -2.069718360900879,
      "loss": 1.9415,
      "nll_loss": 1.8863192796707153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17937874794006348,
      "rewards/margins": 0.027593078091740608,
      "rewards/rejected": -0.20697180926799774,
      "step": 259
    },
    {
      "epoch": 0.4097714736012608,
      "grad_norm": 0.26368269324302673,
      "learning_rate": 3.869807472271731e-06,
      "log_odds_chosen": 0.3248588442802429,
      "log_odds_ratio": -0.5475046634674072,
      "logits/chosen": 0.4813993573188782,
      "logits/rejected": -0.5130388140678406,
      "logps/chosen": -1.7483863830566406,
      "logps/rejected": -2.0266005992889404,
      "loss": 1.9345,
      "nll_loss": 1.8797162771224976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17483864724636078,
      "rewards/margins": 0.02782142162322998,
      "rewards/rejected": -0.20266005396842957,
      "step": 260
    },
    {
      "epoch": 0.41134751773049644,
      "grad_norm": 0.27178314328193665,
      "learning_rate": 3.8678494834853826e-06,
      "log_odds_chosen": 0.3267313838005066,
      "log_odds_ratio": -0.5527905821800232,
      "logits/chosen": 0.463585764169693,
      "logits/rejected": -0.4223080277442932,
      "logps/chosen": -1.7635301351547241,
      "logps/rejected": -2.0397844314575195,
      "loss": 1.9186,
      "nll_loss": 1.863335132598877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17635302245616913,
      "rewards/margins": 0.027625398710370064,
      "rewards/rejected": -0.20397843420505524,
      "step": 261
    },
    {
      "epoch": 0.41292356185973206,
      "grad_norm": 0.29288655519485474,
      "learning_rate": 3.865877383988339e-06,
      "log_odds_chosen": 0.3967463970184326,
      "log_odds_ratio": -0.5207533836364746,
      "logits/chosen": 0.4554799199104309,
      "logits/rejected": -0.4541538953781128,
      "logps/chosen": -1.8009213209152222,
      "logps/rejected": -2.139960765838623,
      "loss": 1.9765,
      "nll_loss": 1.9244519472122192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18009214103221893,
      "rewards/margins": 0.03390393778681755,
      "rewards/rejected": -0.21399609744548798,
      "step": 262
    },
    {
      "epoch": 0.4144996059889677,
      "grad_norm": 0.2784212827682495,
      "learning_rate": 3.863891188678869e-06,
      "log_odds_chosen": 0.14312618970870972,
      "log_odds_ratio": -0.6421398520469666,
      "logits/chosen": 0.4103483259677887,
      "logits/rejected": -0.6072329878807068,
      "logps/chosen": -1.6957128047943115,
      "logps/rejected": -1.8229939937591553,
      "loss": 1.865,
      "nll_loss": 1.8007937669754028,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16957129538059235,
      "rewards/margins": 0.012728110887110233,
      "rewards/rejected": -0.1822993904352188,
      "step": 263
    },
    {
      "epoch": 0.4160756501182033,
      "grad_norm": 0.2715873122215271,
      "learning_rate": 3.8618909125617305e-06,
      "log_odds_chosen": 0.5240032076835632,
      "log_odds_ratio": -0.4674418270587921,
      "logits/chosen": 0.4032454192638397,
      "logits/rejected": -0.8130543828010559,
      "logps/chosen": -1.6866666078567505,
      "logps/rejected": -2.1323814392089844,
      "loss": 1.8611,
      "nll_loss": 1.8143792152404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16866666078567505,
      "rewards/margins": 0.044571489095687866,
      "rewards/rejected": -0.21323813498020172,
      "step": 264
    },
    {
      "epoch": 0.4176516942474389,
      "grad_norm": 0.2634105682373047,
      "learning_rate": 3.859876570748054e-06,
      "log_odds_chosen": 0.3052539527416229,
      "log_odds_ratio": -0.555004894733429,
      "logits/chosen": 0.484012246131897,
      "logits/rejected": -0.15861767530441284,
      "logps/chosen": -1.7536225318908691,
      "logps/rejected": -2.012622833251953,
      "loss": 1.9283,
      "nll_loss": 1.8727929592132568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1753622442483902,
      "rewards/margins": 0.025900062173604965,
      "rewards/rejected": -0.20126229524612427,
      "step": 265
    },
    {
      "epoch": 0.41922773837667454,
      "grad_norm": 0.26199325919151306,
      "learning_rate": 3.857848178455231e-06,
      "log_odds_chosen": 0.10959838330745697,
      "log_odds_ratio": -0.6467946767807007,
      "logits/chosen": 0.48704051971435547,
      "logits/rejected": -0.8094627261161804,
      "logps/chosen": -1.7569371461868286,
      "logps/rejected": -1.8549176454544067,
      "loss": 1.9459,
      "nll_loss": 1.88121497631073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17569372057914734,
      "rewards/margins": 0.00979805551469326,
      "rewards/rejected": -0.18549177050590515,
      "step": 266
    },
    {
      "epoch": 0.42080378250591016,
      "grad_norm": 0.29430001974105835,
      "learning_rate": 3.855805751006794e-06,
      "log_odds_chosen": 0.4646510183811188,
      "log_odds_ratio": -0.5037495493888855,
      "logits/chosen": 0.45664528012275696,
      "logits/rejected": -0.7876089215278625,
      "logps/chosen": -1.7912521362304688,
      "logps/rejected": -2.192134380340576,
      "loss": 1.9399,
      "nll_loss": 1.8894766569137573,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17912520468235016,
      "rewards/margins": 0.04008820280432701,
      "rewards/rejected": -0.21921342611312866,
      "step": 267
    },
    {
      "epoch": 0.4223798266351458,
      "grad_norm": 0.24849706888198853,
      "learning_rate": 3.853749303832308e-06,
      "log_odds_chosen": 0.3170139491558075,
      "log_odds_ratio": -0.5517860651016235,
      "logits/chosen": 0.43677279353141785,
      "logits/rejected": -0.38730910420417786,
      "logps/chosen": -1.7149559259414673,
      "logps/rejected": -1.982513427734375,
      "loss": 1.8752,
      "nll_loss": 1.8200231790542603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17149560153484344,
      "rewards/margins": 0.026755744591355324,
      "rewards/rejected": -0.19825133681297302,
      "step": 268
    },
    {
      "epoch": 0.4239558707643814,
      "grad_norm": 0.2742369771003723,
      "learning_rate": 3.8516788524672495e-06,
      "log_odds_chosen": 0.1780942976474762,
      "log_odds_ratio": -0.6158064603805542,
      "logits/chosen": 0.4654761552810669,
      "logits/rejected": -0.6833764314651489,
      "logps/chosen": -1.7883644104003906,
      "logps/rejected": -1.942136526107788,
      "loss": 1.9458,
      "nll_loss": 1.8842130899429321,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17883646488189697,
      "rewards/margins": 0.015377204865217209,
      "rewards/rejected": -0.1942136585712433,
      "step": 269
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 0.2591206133365631,
      "learning_rate": 3.849594412552889e-06,
      "log_odds_chosen": 0.3254651725292206,
      "log_odds_ratio": -0.5489966869354248,
      "logits/chosen": 0.47052422165870667,
      "logits/rejected": -0.4670095443725586,
      "logps/chosen": -1.6865018606185913,
      "logps/rejected": -1.9543278217315674,
      "loss": 1.8487,
      "nll_loss": 1.7938035726547241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16865020990371704,
      "rewards/margins": 0.026782585307955742,
      "rewards/rejected": -0.19543276727199554,
      "step": 270
    },
    {
      "epoch": 0.42710795902285265,
      "grad_norm": 0.26700544357299805,
      "learning_rate": 3.847495999836175e-06,
      "log_odds_chosen": 0.3362584114074707,
      "log_odds_ratio": -0.5431700944900513,
      "logits/chosen": 0.5047026872634888,
      "logits/rejected": -0.694282054901123,
      "logps/chosen": -1.7739330530166626,
      "logps/rejected": -2.061260461807251,
      "loss": 1.92,
      "nll_loss": 1.8657115697860718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17739331722259521,
      "rewards/margins": 0.028732730075716972,
      "rewards/rejected": -0.20612604916095734,
      "step": 271
    },
    {
      "epoch": 0.42868400315208827,
      "grad_norm": 0.25737646222114563,
      "learning_rate": 3.845383630169613e-06,
      "log_odds_chosen": 0.33631449937820435,
      "log_odds_ratio": -0.5460928678512573,
      "logits/chosen": 0.3562855124473572,
      "logits/rejected": -0.6789432168006897,
      "logps/chosen": -1.6831501722335815,
      "logps/rejected": -1.9663141965866089,
      "loss": 1.8306,
      "nll_loss": 1.7760382890701294,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16831502318382263,
      "rewards/margins": 0.028316402807831764,
      "rewards/rejected": -0.19663141667842865,
      "step": 272
    },
    {
      "epoch": 0.4302600472813239,
      "grad_norm": 0.27558234333992004,
      "learning_rate": 3.843257319511147e-06,
      "log_odds_chosen": 0.5057035684585571,
      "log_odds_ratio": -0.480516254901886,
      "logits/chosen": 0.4118584394454956,
      "logits/rejected": -0.8097151517868042,
      "logps/chosen": -1.6787090301513672,
      "logps/rejected": -2.1040146350860596,
      "loss": 1.8281,
      "nll_loss": 1.7800368070602417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1678709089756012,
      "rewards/margins": 0.04253055900335312,
      "rewards/rejected": -0.21040146052837372,
      "step": 273
    },
    {
      "epoch": 0.4318360914105595,
      "grad_norm": 0.26741060614585876,
      "learning_rate": 3.841117083924039e-06,
      "log_odds_chosen": 0.3235999643802643,
      "log_odds_ratio": -0.5523552298545837,
      "logits/chosen": 0.397009938955307,
      "logits/rejected": -0.8522763252258301,
      "logps/chosen": -1.7239158153533936,
      "logps/rejected": -2.0000529289245605,
      "loss": 1.874,
      "nll_loss": 1.8187570571899414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17239157855510712,
      "rewards/margins": 0.027613701298832893,
      "rewards/rejected": -0.20000529289245605,
      "step": 274
    },
    {
      "epoch": 0.43341213553979513,
      "grad_norm": 0.26848408579826355,
      "learning_rate": 3.838962939576746e-06,
      "log_odds_chosen": 0.36745256185531616,
      "log_odds_ratio": -0.5347477793693542,
      "logits/chosen": 0.34434112906455994,
      "logits/rejected": -0.6846289038658142,
      "logps/chosen": -1.6550366878509521,
      "logps/rejected": -1.963791847229004,
      "loss": 1.8302,
      "nll_loss": 1.7767248153686523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16550366580486298,
      "rewards/margins": 0.030875524505972862,
      "rewards/rejected": -0.19637919962406158,
      "step": 275
    },
    {
      "epoch": 0.43498817966903075,
      "grad_norm": 0.27719369530677795,
      "learning_rate": 3.8367949027427985e-06,
      "log_odds_chosen": 0.24599358439445496,
      "log_odds_ratio": -0.581466794013977,
      "logits/chosen": 0.41534924507141113,
      "logits/rejected": -0.5528399348258972,
      "logps/chosen": -1.7868237495422363,
      "logps/rejected": -1.9973784685134888,
      "loss": 1.9663,
      "nll_loss": 1.908132791519165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1786823868751526,
      "rewards/margins": 0.02105545997619629,
      "rewards/rejected": -0.19973784685134888,
      "step": 276
    },
    {
      "epoch": 0.43656422379826637,
      "grad_norm": 0.2863655984401703,
      "learning_rate": 3.834612989800681e-06,
      "log_odds_chosen": 0.6205421686172485,
      "log_odds_ratio": -0.43973320722579956,
      "logits/chosen": 0.4383569359779358,
      "logits/rejected": -1.0248867273330688,
      "logps/chosen": -1.768141269683838,
      "logps/rejected": -2.3065080642700195,
      "loss": 1.9028,
      "nll_loss": 1.8588356971740723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17681412398815155,
      "rewards/margins": 0.05383666604757309,
      "rewards/rejected": -0.23065082728862762,
      "step": 277
    },
    {
      "epoch": 0.438140267927502,
      "grad_norm": 0.2790682315826416,
      "learning_rate": 3.832417217233703e-06,
      "log_odds_chosen": 0.23550176620483398,
      "log_odds_ratio": -0.5852302312850952,
      "logits/chosen": 0.39655160903930664,
      "logits/rejected": -0.5482574105262756,
      "logps/chosen": -1.7667053937911987,
      "logps/rejected": -1.9674649238586426,
      "loss": 1.925,
      "nll_loss": 1.8664298057556152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17667050659656525,
      "rewards/margins": 0.020075976848602295,
      "rewards/rejected": -0.19674648344516754,
      "step": 278
    },
    {
      "epoch": 0.4397163120567376,
      "grad_norm": 0.26800537109375,
      "learning_rate": 3.8302076016298775e-06,
      "log_odds_chosen": 0.39774730801582336,
      "log_odds_ratio": -0.5189967751502991,
      "logits/chosen": 0.33551838994026184,
      "logits/rejected": -0.7844797372817993,
      "logps/chosen": -1.6954327821731567,
      "logps/rejected": -2.031496524810791,
      "loss": 1.8556,
      "nll_loss": 1.8036519289016724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16954326629638672,
      "rewards/margins": 0.03360636159777641,
      "rewards/rejected": -0.20314966142177582,
      "step": 279
    },
    {
      "epoch": 0.44129235618597323,
      "grad_norm": 0.26829519867897034,
      "learning_rate": 3.827984159681796e-06,
      "log_odds_chosen": 0.3000229299068451,
      "log_odds_ratio": -0.5594607591629028,
      "logits/chosen": 0.31995582580566406,
      "logits/rejected": -0.6465460062026978,
      "logps/chosen": -1.7473139762878418,
      "logps/rejected": -2.0010228157043457,
      "loss": 1.8941,
      "nll_loss": 1.838153600692749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17473138868808746,
      "rewards/margins": 0.025370875373482704,
      "rewards/rejected": -0.20010226964950562,
      "step": 280
    },
    {
      "epoch": 0.4428684003152088,
      "grad_norm": 0.2701958417892456,
      "learning_rate": 3.825746908186498e-06,
      "log_odds_chosen": 0.38087302446365356,
      "log_odds_ratio": -0.5301374793052673,
      "logits/chosen": 0.4656312167644501,
      "logits/rejected": -0.8502053618431091,
      "logps/chosen": -1.7905412912368774,
      "logps/rejected": -2.118225336074829,
      "loss": 1.9211,
      "nll_loss": 1.8680615425109863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1790541410446167,
      "rewards/margins": 0.032768405973911285,
      "rewards/rejected": -0.2118225395679474,
      "step": 281
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.28369569778442383,
      "learning_rate": 3.823495864045352e-06,
      "log_odds_chosen": 0.41946643590927124,
      "log_odds_ratio": -0.5092715620994568,
      "logits/chosen": 0.4426696002483368,
      "logits/rejected": -0.2964620292186737,
      "logps/chosen": -1.7443768978118896,
      "logps/rejected": -2.0997912883758545,
      "loss": 1.8958,
      "nll_loss": 1.8448481559753418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17443770170211792,
      "rewards/margins": 0.03554143011569977,
      "rewards/rejected": -0.2099791169166565,
      "step": 282
    },
    {
      "epoch": 0.44602048857368004,
      "grad_norm": 0.24753610789775848,
      "learning_rate": 3.8212310442639205e-06,
      "log_odds_chosen": 0.3292975425720215,
      "log_odds_ratio": -0.5465047359466553,
      "logits/chosen": 0.4356715977191925,
      "logits/rejected": -0.4361802339553833,
      "logps/chosen": -1.6899651288986206,
      "logps/rejected": -1.9682831764221191,
      "loss": 1.8516,
      "nll_loss": 1.7969615459442139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16899652779102325,
      "rewards/margins": 0.027831794694066048,
      "rewards/rejected": -0.19682832062244415,
      "step": 283
    },
    {
      "epoch": 0.44759653270291566,
      "grad_norm": 0.249053955078125,
      "learning_rate": 3.8189524659518355e-06,
      "log_odds_chosen": 0.38190656900405884,
      "log_odds_ratio": -0.5236896276473999,
      "logits/chosen": 0.48760396242141724,
      "logits/rejected": -0.6940497756004333,
      "logps/chosen": -1.7510186433792114,
      "logps/rejected": -2.0749480724334717,
      "loss": 1.8959,
      "nll_loss": 1.8434971570968628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1751018613576889,
      "rewards/margins": 0.0323929525911808,
      "rewards/rejected": -0.2074948251247406,
      "step": 284
    },
    {
      "epoch": 0.4491725768321513,
      "grad_norm": 0.26106294989585876,
      "learning_rate": 3.816660146322667e-06,
      "log_odds_chosen": 0.31609034538269043,
      "log_odds_ratio": -0.5512754917144775,
      "logits/chosen": 0.42169806361198425,
      "logits/rejected": -0.4507894217967987,
      "logps/chosen": -1.6568574905395508,
      "logps/rejected": -1.9196453094482422,
      "loss": 1.8096,
      "nll_loss": 1.7544704675674438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1656857579946518,
      "rewards/margins": 0.026278777047991753,
      "rewards/rejected": -0.1919645369052887,
      "step": 285
    },
    {
      "epoch": 0.4507486209613869,
      "grad_norm": 0.24721866846084595,
      "learning_rate": 3.814354102693797e-06,
      "log_odds_chosen": 0.4020289182662964,
      "log_odds_ratio": -0.519049882888794,
      "logits/chosen": 0.4439999759197235,
      "logits/rejected": -0.6853295564651489,
      "logps/chosen": -1.718071699142456,
      "logps/rejected": -2.0572900772094727,
      "loss": 1.8677,
      "nll_loss": 1.8157658576965332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1718071699142456,
      "rewards/margins": 0.03392184525728226,
      "rewards/rejected": -0.20572902262210846,
      "step": 286
    },
    {
      "epoch": 0.4523246650906225,
      "grad_norm": 0.2539658844470978,
      "learning_rate": 3.8120343524862814e-06,
      "log_odds_chosen": 0.2967430651187897,
      "log_odds_ratio": -0.5581727027893066,
      "logits/chosen": 0.40530580282211304,
      "logits/rejected": -0.6203088760375977,
      "logps/chosen": -1.7594120502471924,
      "logps/rejected": -2.0090889930725098,
      "loss": 1.8925,
      "nll_loss": 1.8366661071777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17594121396541595,
      "rewards/margins": 0.024967655539512634,
      "rewards/rejected": -0.2009088695049286,
      "step": 287
    },
    {
      "epoch": 0.45390070921985815,
      "grad_norm": 0.24290117621421814,
      "learning_rate": 3.809700913224726e-06,
      "log_odds_chosen": 0.41434168815612793,
      "log_odds_ratio": -0.5204272866249084,
      "logits/chosen": 0.367045521736145,
      "logits/rejected": -1.0038524866104126,
      "logps/chosen": -1.6730784177780151,
      "logps/rejected": -2.0234670639038086,
      "loss": 1.8168,
      "nll_loss": 1.764800786972046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16730786859989166,
      "rewards/margins": 0.03503885120153427,
      "rewards/rejected": -0.20234668254852295,
      "step": 288
    },
    {
      "epoch": 0.45547675334909377,
      "grad_norm": 0.2513137757778168,
      "learning_rate": 3.8073538025371494e-06,
      "log_odds_chosen": 0.49752530455589294,
      "log_odds_ratio": -0.4798320233821869,
      "logits/chosen": 0.4955059885978699,
      "logits/rejected": -0.8152831792831421,
      "logps/chosen": -1.6548831462860107,
      "logps/rejected": -2.0759389400482178,
      "loss": 1.8087,
      "nll_loss": 1.7607529163360596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16548830270767212,
      "rewards/margins": 0.04210559278726578,
      "rewards/rejected": -0.2075939029455185,
      "step": 289
    },
    {
      "epoch": 0.4570527974783294,
      "grad_norm": 0.2759801745414734,
      "learning_rate": 3.804993038154852e-06,
      "log_odds_chosen": 0.37791135907173157,
      "log_odds_ratio": -0.5382211208343506,
      "logits/chosen": 0.4199434518814087,
      "logits/rejected": -0.7131822109222412,
      "logps/chosen": -1.695268154144287,
      "logps/rejected": -2.019188165664673,
      "loss": 1.8613,
      "nll_loss": 1.8074455261230469,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1695268154144287,
      "rewards/margins": 0.03239201009273529,
      "rewards/rejected": -0.201918825507164,
      "step": 290
    },
    {
      "epoch": 0.458628841607565,
      "grad_norm": 0.2639031708240509,
      "learning_rate": 3.8026186379122816e-06,
      "log_odds_chosen": 0.2410442978143692,
      "log_odds_ratio": -0.5826308727264404,
      "logits/chosen": 0.36750051379203796,
      "logits/rejected": -0.7549840807914734,
      "logps/chosen": -1.7680672407150269,
      "logps/rejected": -1.972318410873413,
      "loss": 1.9077,
      "nll_loss": 1.8494325876235962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1768067330121994,
      "rewards/margins": 0.02042512409389019,
      "rewards/rejected": -0.19723184406757355,
      "step": 291
    },
    {
      "epoch": 0.46020488573680063,
      "grad_norm": 0.24375998973846436,
      "learning_rate": 3.8002306197468983e-06,
      "log_odds_chosen": 0.20314940810203552,
      "log_odds_ratio": -0.6030868291854858,
      "logits/chosen": 0.3567318320274353,
      "logits/rejected": -0.7969244122505188,
      "logps/chosen": -1.7306976318359375,
      "logps/rejected": -1.901548981666565,
      "loss": 1.8857,
      "nll_loss": 1.8254362344741821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17306974530220032,
      "rewards/margins": 0.01708514243364334,
      "rewards/rejected": -0.19015489518642426,
      "step": 292
    },
    {
      "epoch": 0.46178092986603625,
      "grad_norm": 0.24072591960430145,
      "learning_rate": 3.7978290016990367e-06,
      "log_odds_chosen": 0.4728235602378845,
      "log_odds_ratio": -0.48956677317619324,
      "logits/chosen": 0.346945583820343,
      "logits/rejected": -0.7346411943435669,
      "logps/chosen": -1.7421035766601562,
      "logps/rejected": -2.148958444595337,
      "loss": 1.8835,
      "nll_loss": 1.8345091342926025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17421036958694458,
      "rewards/margins": 0.040685467422008514,
      "rewards/rejected": -0.2148958295583725,
      "step": 293
    },
    {
      "epoch": 0.46335697399527187,
      "grad_norm": 0.2703053951263428,
      "learning_rate": 3.795413801911776e-06,
      "log_odds_chosen": 0.6493099927902222,
      "log_odds_ratio": -0.43461745977401733,
      "logits/chosen": 0.34954923391342163,
      "logits/rejected": -0.8253506422042847,
      "logps/chosen": -1.6791515350341797,
      "logps/rejected": -2.2405524253845215,
      "loss": 1.8362,
      "nll_loss": 1.7926918268203735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16791515052318573,
      "rewards/margins": 0.05614008009433746,
      "rewards/rejected": -0.2240552306175232,
      "step": 294
    },
    {
      "epoch": 0.4649330181245075,
      "grad_norm": 0.2824196219444275,
      "learning_rate": 3.7929850386307965e-06,
      "log_odds_chosen": 0.31321728229522705,
      "log_odds_ratio": -0.5547051429748535,
      "logits/chosen": 0.4384803771972656,
      "logits/rejected": -0.619215726852417,
      "logps/chosen": -1.8919639587402344,
      "logps/rejected": -2.1647329330444336,
      "loss": 2.0217,
      "nll_loss": 1.9662539958953857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1891964077949524,
      "rewards/margins": 0.027276910841464996,
      "rewards/rejected": -0.21647332608699799,
      "step": 295
    },
    {
      "epoch": 0.4665090622537431,
      "grad_norm": 0.25732651352882385,
      "learning_rate": 3.790542730204245e-06,
      "log_odds_chosen": 0.3414647579193115,
      "log_odds_ratio": -0.5392407178878784,
      "logits/chosen": 0.3852652311325073,
      "logits/rejected": -0.8155044913291931,
      "logps/chosen": -1.8048094511032104,
      "logps/rejected": -2.098794460296631,
      "loss": 1.9317,
      "nll_loss": 1.877764105796814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1804809719324112,
      "rewards/margins": 0.02939850091934204,
      "rewards/rejected": -0.20987947285175323,
      "step": 296
    },
    {
      "epoch": 0.46808510638297873,
      "grad_norm": 0.26373788714408875,
      "learning_rate": 3.7880868950825935e-06,
      "log_odds_chosen": 0.4434778094291687,
      "log_odds_ratio": -0.4981243908405304,
      "logits/chosen": 0.3620328903198242,
      "logits/rejected": -0.8764305114746094,
      "logps/chosen": -1.7774946689605713,
      "logps/rejected": -2.1578266620635986,
      "loss": 1.8923,
      "nll_loss": 1.842448353767395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17774948477745056,
      "rewards/margins": 0.03803320974111557,
      "rewards/rejected": -0.21578267216682434,
      "step": 297
    },
    {
      "epoch": 0.46966115051221435,
      "grad_norm": 0.26903581619262695,
      "learning_rate": 3.7856175518185058e-06,
      "log_odds_chosen": 0.38129091262817383,
      "log_odds_ratio": -0.5281710028648376,
      "logits/chosen": 0.20757347345352173,
      "logits/rejected": -0.9698464870452881,
      "logps/chosen": -1.742034912109375,
      "logps/rejected": -2.0659961700439453,
      "loss": 1.888,
      "nll_loss": 1.8351514339447021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17420348525047302,
      "rewards/margins": 0.032396137714385986,
      "rewards/rejected": -0.206599622964859,
      "step": 298
    },
    {
      "epoch": 0.47123719464145,
      "grad_norm": 0.2571781575679779,
      "learning_rate": 3.7831347190666883e-06,
      "log_odds_chosen": 0.4503501057624817,
      "log_odds_ratio": -0.49767959117889404,
      "logits/chosen": 0.29682573676109314,
      "logits/rejected": -0.8833941221237183,
      "logps/chosen": -1.7364448308944702,
      "logps/rejected": -2.1207523345947266,
      "loss": 1.8836,
      "nll_loss": 1.8338348865509033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17364446818828583,
      "rewards/margins": 0.03843075782060623,
      "rewards/rejected": -0.21207523345947266,
      "step": 299
    },
    {
      "epoch": 0.4728132387706856,
      "grad_norm": 0.2652571499347687,
      "learning_rate": 3.780638415583759e-06,
      "log_odds_chosen": 0.2772998809814453,
      "log_odds_ratio": -0.5754559636116028,
      "logits/chosen": 0.41935643553733826,
      "logits/rejected": -0.6864033341407776,
      "logps/chosen": -1.7198681831359863,
      "logps/rejected": -1.9582655429840088,
      "loss": 1.8772,
      "nll_loss": 1.819682240486145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17198681831359863,
      "rewards/margins": 0.02383972890675068,
      "rewards/rejected": -0.19582654535770416,
      "step": 300
    },
    {
      "epoch": 0.4743892828999212,
      "grad_norm": 0.26339584589004517,
      "learning_rate": 3.7781286602280967e-06,
      "log_odds_chosen": 0.1517128050327301,
      "log_odds_ratio": -0.6284233331680298,
      "logits/chosen": 0.3332933187484741,
      "logits/rejected": -0.9037990570068359,
      "logps/chosen": -1.7000682353973389,
      "logps/rejected": -1.8223541975021362,
      "loss": 1.8654,
      "nll_loss": 1.802597165107727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17000681161880493,
      "rewards/margins": 0.012228596955537796,
      "rewards/rejected": -0.18223541975021362,
      "step": 301
    },
    {
      "epoch": 0.47596532702915684,
      "grad_norm": 0.24838051199913025,
      "learning_rate": 3.7756054719597044e-06,
      "log_odds_chosen": 0.23710456490516663,
      "log_odds_ratio": -0.617565393447876,
      "logits/chosen": 0.23182198405265808,
      "logits/rejected": -0.9376928806304932,
      "logps/chosen": -1.7347861528396606,
      "logps/rejected": -1.9488537311553955,
      "loss": 1.8729,
      "nll_loss": 1.8111293315887451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1734786182641983,
      "rewards/margins": 0.021406756713986397,
      "rewards/rejected": -0.19488537311553955,
      "step": 302
    },
    {
      "epoch": 0.47754137115839246,
      "grad_norm": 0.3430563509464264,
      "learning_rate": 3.773068869840066e-06,
      "log_odds_chosen": 0.21702063083648682,
      "log_odds_ratio": -0.5988757014274597,
      "logits/chosen": 0.4508088231086731,
      "logits/rejected": -0.9070717096328735,
      "logps/chosen": -1.7604243755340576,
      "logps/rejected": -1.9423820972442627,
      "loss": 1.8963,
      "nll_loss": 1.8363980054855347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17604243755340576,
      "rewards/margins": 0.018195772543549538,
      "rewards/rejected": -0.19423823058605194,
      "step": 303
    },
    {
      "epoch": 0.4791174152876281,
      "grad_norm": 0.23839880526065826,
      "learning_rate": 3.770518873031997e-06,
      "log_odds_chosen": 0.4319533109664917,
      "log_odds_ratio": -0.5028316378593445,
      "logits/chosen": 0.21307101845741272,
      "logits/rejected": -0.6253250241279602,
      "logps/chosen": -1.619141936302185,
      "logps/rejected": -1.9784443378448486,
      "loss": 1.7785,
      "nll_loss": 1.7282602787017822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16191419959068298,
      "rewards/margins": 0.03593023866415024,
      "rewards/rejected": -0.197844460606575,
      "step": 304
    },
    {
      "epoch": 0.4806934594168637,
      "grad_norm": 0.2817673683166504,
      "learning_rate": 3.7679555007995065e-06,
      "log_odds_chosen": 0.4497647285461426,
      "log_odds_ratio": -0.5144132375717163,
      "logits/chosen": 0.3390055298805237,
      "logits/rejected": -0.7776145935058594,
      "logps/chosen": -1.7621970176696777,
      "logps/rejected": -2.1498844623565674,
      "loss": 1.908,
      "nll_loss": 1.856536865234375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17621970176696777,
      "rewards/margins": 0.03876873850822449,
      "rewards/rejected": -0.21498845517635345,
      "step": 305
    },
    {
      "epoch": 0.48226950354609927,
      "grad_norm": 0.26427942514419556,
      "learning_rate": 3.7653787725076464e-06,
      "log_odds_chosen": 0.2555605173110962,
      "log_odds_ratio": -0.5800157785415649,
      "logits/chosen": 0.3456537425518036,
      "logits/rejected": -0.738465428352356,
      "logps/chosen": -1.7156627178192139,
      "logps/rejected": -1.9342702627182007,
      "loss": 1.8576,
      "nll_loss": 1.7995985746383667,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17156627774238586,
      "rewards/margins": 0.02186075784265995,
      "rewards/rejected": -0.19342704117298126,
      "step": 306
    },
    {
      "epoch": 0.4838455476753349,
      "grad_norm": 0.2824785113334656,
      "learning_rate": 3.7627887076223685e-06,
      "log_odds_chosen": 0.3448987305164337,
      "log_odds_ratio": -0.5420399904251099,
      "logits/chosen": 0.3761395215988159,
      "logits/rejected": -0.5656124353408813,
      "logps/chosen": -1.7869853973388672,
      "logps/rejected": -2.0822932720184326,
      "loss": 1.9128,
      "nll_loss": 1.8585689067840576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17869853973388672,
      "rewards/margins": 0.02953079342842102,
      "rewards/rejected": -0.20822934806346893,
      "step": 307
    },
    {
      "epoch": 0.4854215918045705,
      "grad_norm": 0.2356492429971695,
      "learning_rate": 3.7601853257103765e-06,
      "log_odds_chosen": 0.17670860886573792,
      "log_odds_ratio": -0.6152773499488831,
      "logits/chosen": 0.26213836669921875,
      "logits/rejected": -0.832030177116394,
      "logps/chosen": -1.6793239116668701,
      "logps/rejected": -1.829579472541809,
      "loss": 1.81,
      "nll_loss": 1.7484831809997559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.167932391166687,
      "rewards/margins": 0.015025541186332703,
      "rewards/rejected": -0.18295791745185852,
      "step": 308
    },
    {
      "epoch": 0.48699763593380613,
      "grad_norm": 0.24143275618553162,
      "learning_rate": 3.7575686464389767e-06,
      "log_odds_chosen": 0.32946810126304626,
      "log_odds_ratio": -0.5521824359893799,
      "logits/chosen": 0.3027976453304291,
      "logits/rejected": -0.8635303378105164,
      "logps/chosen": -1.655510663986206,
      "logps/rejected": -1.9322257041931152,
      "loss": 1.8093,
      "nll_loss": 1.754052996635437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1655510514974594,
      "rewards/margins": 0.027671512216329575,
      "rewards/rejected": -0.19322258234024048,
      "step": 309
    },
    {
      "epoch": 0.48857368006304175,
      "grad_norm": 0.24138718843460083,
      "learning_rate": 3.7549386895759315e-06,
      "log_odds_chosen": 0.364417165517807,
      "log_odds_ratio": -0.5298891663551331,
      "logits/chosen": 0.26288849115371704,
      "logits/rejected": -0.7961046099662781,
      "logps/chosen": -1.605391025543213,
      "logps/rejected": -1.9066240787506104,
      "loss": 1.7459,
      "nll_loss": 1.692944049835205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16053910553455353,
      "rewards/margins": 0.030123300850391388,
      "rewards/rejected": -0.19066239893436432,
      "step": 310
    },
    {
      "epoch": 0.49014972419227737,
      "grad_norm": 0.4169044494628906,
      "learning_rate": 3.7522954749893086e-06,
      "log_odds_chosen": 0.609459638595581,
      "log_odds_ratio": -0.4481661021709442,
      "logits/chosen": 0.1874232292175293,
      "logits/rejected": -0.9124564528465271,
      "logps/chosen": -1.6414852142333984,
      "logps/rejected": -2.157813549041748,
      "loss": 1.7921,
      "nll_loss": 1.7472877502441406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1641485095024109,
      "rewards/margins": 0.051632851362228394,
      "rewards/rejected": -0.21578137576580048,
      "step": 311
    },
    {
      "epoch": 0.491725768321513,
      "grad_norm": 0.2586691379547119,
      "learning_rate": 3.749639022647332e-06,
      "log_odds_chosen": 0.3325248658657074,
      "log_odds_ratio": -0.5451020002365112,
      "logits/chosen": 0.24192464351654053,
      "logits/rejected": -0.8050363063812256,
      "logps/chosen": -1.7278075218200684,
      "logps/rejected": -2.010347366333008,
      "loss": 1.8693,
      "nll_loss": 1.814815878868103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17278075218200684,
      "rewards/margins": 0.02825397439301014,
      "rewards/rejected": -0.20103472471237183,
      "step": 312
    },
    {
      "epoch": 0.4933018124507486,
      "grad_norm": 0.22876015305519104,
      "learning_rate": 3.7469693526182304e-06,
      "log_odds_chosen": 0.4910639524459839,
      "log_odds_ratio": -0.4973025321960449,
      "logits/chosen": 0.18789926171302795,
      "logits/rejected": -1.2326761484146118,
      "logps/chosen": -1.6729106903076172,
      "logps/rejected": -2.0879974365234375,
      "loss": 1.7994,
      "nll_loss": 1.749713659286499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.167291060090065,
      "rewards/margins": 0.041508689522743225,
      "rewards/rejected": -0.20879974961280823,
      "step": 313
    },
    {
      "epoch": 0.49487785657998423,
      "grad_norm": 0.23891504108905792,
      "learning_rate": 3.744286485070085e-06,
      "log_odds_chosen": 0.46477484703063965,
      "log_odds_ratio": -0.49088478088378906,
      "logits/chosen": 0.2151736468076706,
      "logits/rejected": -0.9330959916114807,
      "logps/chosen": -1.6210802793502808,
      "logps/rejected": -2.0096402168273926,
      "loss": 1.754,
      "nll_loss": 1.7048873901367188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16210801899433136,
      "rewards/margins": 0.038856007158756256,
      "rewards/rejected": -0.20096401870250702,
      "step": 314
    },
    {
      "epoch": 0.49645390070921985,
      "grad_norm": 0.2731998562812805,
      "learning_rate": 3.7415904402706795e-06,
      "log_odds_chosen": 0.3603336811065674,
      "log_odds_ratio": -0.5388191342353821,
      "logits/chosen": 0.2712251543998718,
      "logits/rejected": -0.8804117441177368,
      "logps/chosen": -1.8475456237792969,
      "logps/rejected": -2.1596999168395996,
      "loss": 1.9762,
      "nll_loss": 1.9222995042800903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18475456535816193,
      "rewards/margins": 0.031215444207191467,
      "rewards/rejected": -0.2159700095653534,
      "step": 315
    },
    {
      "epoch": 0.4980299448384555,
      "grad_norm": 0.24663853645324707,
      "learning_rate": 3.7388812385873435e-06,
      "log_odds_chosen": 0.321816086769104,
      "log_odds_ratio": -0.5476392507553101,
      "logits/chosen": 0.21599149703979492,
      "logits/rejected": -0.9269928336143494,
      "logps/chosen": -1.6557197570800781,
      "logps/rejected": -1.9236208200454712,
      "loss": 1.7899,
      "nll_loss": 1.7351197004318237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16557197272777557,
      "rewards/margins": 0.026790114119648933,
      "rewards/rejected": -0.19236209988594055,
      "step": 316
    },
    {
      "epoch": 0.4996059889676911,
      "grad_norm": 0.25920945405960083,
      "learning_rate": 3.7361589004868033e-06,
      "log_odds_chosen": 0.549557626247406,
      "log_odds_ratio": -0.46774470806121826,
      "logits/chosen": 0.33838802576065063,
      "logits/rejected": -0.9401962757110596,
      "logps/chosen": -1.7451956272125244,
      "logps/rejected": -2.222694158554077,
      "loss": 1.8613,
      "nll_loss": 1.8144901990890503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17451956868171692,
      "rewards/margins": 0.047749847173690796,
      "rewards/rejected": -0.22226941585540771,
      "step": 317
    },
    {
      "epoch": 0.5011820330969267,
      "grad_norm": 0.2366023063659668,
      "learning_rate": 3.733423446535022e-06,
      "log_odds_chosen": 0.5207498669624329,
      "log_odds_ratio": -0.4805837869644165,
      "logits/chosen": 0.30363941192626953,
      "logits/rejected": -1.0254158973693848,
      "logps/chosen": -1.738085150718689,
      "logps/rejected": -2.1922197341918945,
      "loss": 1.8581,
      "nll_loss": 1.8100310564041138,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17380854487419128,
      "rewards/margins": 0.04541344195604324,
      "rewards/rejected": -0.21922197937965393,
      "step": 318
    },
    {
      "epoch": 0.5027580772261623,
      "grad_norm": 0.23470795154571533,
      "learning_rate": 3.7306748973970476e-06,
      "log_odds_chosen": 0.34376969933509827,
      "log_odds_ratio": -0.5430240035057068,
      "logits/chosen": 0.20498515665531158,
      "logits/rejected": -1.1637543439865112,
      "logps/chosen": -1.5948607921600342,
      "logps/rejected": -1.8776003122329712,
      "loss": 1.7581,
      "nll_loss": 1.7037681341171265,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1594860851764679,
      "rewards/margins": 0.02827395498752594,
      "rewards/rejected": -0.18776002526283264,
      "step": 319
    },
    {
      "epoch": 0.5043341213553979,
      "grad_norm": 0.23327548801898956,
      "learning_rate": 3.7279132738368564e-06,
      "log_odds_chosen": 0.17803671956062317,
      "log_odds_ratio": -0.6121765375137329,
      "logits/chosen": 0.2815796136856079,
      "logits/rejected": -0.9793424606323242,
      "logps/chosen": -1.7961714267730713,
      "logps/rejected": -1.9467929601669312,
      "loss": 1.9245,
      "nll_loss": 1.863244652748108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17961712181568146,
      "rewards/margins": 0.015062162652611732,
      "rewards/rejected": -0.19467930495738983,
      "step": 320
    },
    {
      "epoch": 0.5059101654846335,
      "grad_norm": 0.24164979159832,
      "learning_rate": 3.725138596717195e-06,
      "log_odds_chosen": 0.5048070549964905,
      "log_odds_ratio": -0.4779382348060608,
      "logits/chosen": 0.20476730167865753,
      "logits/rejected": -0.9783819913864136,
      "logps/chosen": -1.6493675708770752,
      "logps/rejected": -2.0773749351501465,
      "loss": 1.7734,
      "nll_loss": 1.7255733013153076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16493676602840424,
      "rewards/margins": 0.04280070960521698,
      "rewards/rejected": -0.2077374905347824,
      "step": 321
    },
    {
      "epoch": 0.5074862096138691,
      "grad_norm": 0.23102499544620514,
      "learning_rate": 3.7223508869994244e-06,
      "log_odds_chosen": 0.42121315002441406,
      "log_odds_ratio": -0.5144806504249573,
      "logits/chosen": 0.23960447311401367,
      "logits/rejected": -1.09214448928833,
      "logps/chosen": -1.702633261680603,
      "logps/rejected": -2.0583620071411133,
      "loss": 1.833,
      "nll_loss": 1.7815495729446411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17026333510875702,
      "rewards/margins": 0.035572879016399384,
      "rewards/rejected": -0.2058362066745758,
      "step": 322
    },
    {
      "epoch": 0.5090622537431048,
      "grad_norm": 0.2813505530357361,
      "learning_rate": 3.7195501657433594e-06,
      "log_odds_chosen": 0.33818790316581726,
      "log_odds_ratio": -0.5427064895629883,
      "logits/chosen": 0.1689353883266449,
      "logits/rejected": -1.0303304195404053,
      "logps/chosen": -1.7009472846984863,
      "logps/rejected": -1.984174132347107,
      "loss": 1.8396,
      "nll_loss": 1.7853001356124878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17009472846984863,
      "rewards/margins": 0.028322694823145866,
      "rewards/rejected": -0.19841741025447845,
      "step": 323
    },
    {
      "epoch": 0.5106382978723404,
      "grad_norm": 0.22637781500816345,
      "learning_rate": 3.716736454107111e-06,
      "log_odds_chosen": 0.5035085082054138,
      "log_odds_ratio": -0.4778319299221039,
      "logits/chosen": 0.1796778440475464,
      "logits/rejected": -0.8279569149017334,
      "logps/chosen": -1.5848971605300903,
      "logps/rejected": -2.0021541118621826,
      "loss": 1.7142,
      "nll_loss": 1.6663726568222046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15848970413208008,
      "rewards/margins": 0.04172568395733833,
      "rewards/rejected": -0.2002153992652893,
      "step": 324
    },
    {
      "epoch": 0.512214342001576,
      "grad_norm": 0.2479918897151947,
      "learning_rate": 3.7139097733469277e-06,
      "log_odds_chosen": 0.5620511770248413,
      "log_odds_ratio": -0.4595773220062256,
      "logits/chosen": 0.20734019577503204,
      "logits/rejected": -0.9722301363945007,
      "logps/chosen": -1.644300937652588,
      "logps/rejected": -2.121246099472046,
      "loss": 1.7752,
      "nll_loss": 1.729271411895752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16443009674549103,
      "rewards/margins": 0.047694504261016846,
      "rewards/rejected": -0.21212461590766907,
      "step": 325
    },
    {
      "epoch": 0.5137903861308116,
      "grad_norm": 0.25066426396369934,
      "learning_rate": 3.711070144817032e-06,
      "log_odds_chosen": 0.3923027515411377,
      "log_odds_ratio": -0.5236109495162964,
      "logits/chosen": 0.18746960163116455,
      "logits/rejected": -0.8730261921882629,
      "logps/chosen": -1.737861156463623,
      "logps/rejected": -2.0755648612976074,
      "loss": 1.8758,
      "nll_loss": 1.8234347105026245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17378610372543335,
      "rewards/margins": 0.03377038985490799,
      "rewards/rejected": -0.20755648612976074,
      "step": 326
    },
    {
      "epoch": 0.5153664302600472,
      "grad_norm": 0.25296109914779663,
      "learning_rate": 3.708217589969461e-06,
      "log_odds_chosen": 0.49952542781829834,
      "log_odds_ratio": -0.4828339219093323,
      "logits/chosen": 0.13944938778877258,
      "logits/rejected": -1.0598782300949097,
      "logps/chosen": -1.7016721963882446,
      "logps/rejected": -2.1262779235839844,
      "loss": 1.8419,
      "nll_loss": 1.793623447418213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1701672375202179,
      "rewards/margins": 0.04246056079864502,
      "rewards/rejected": -0.21262778341770172,
      "step": 327
    },
    {
      "epoch": 0.5169424743892829,
      "grad_norm": 0.223394513130188,
      "learning_rate": 3.705352130353904e-06,
      "log_odds_chosen": 0.4204602837562561,
      "log_odds_ratio": -0.519094705581665,
      "logits/chosen": 0.20659029483795166,
      "logits/rejected": -1.0534716844558716,
      "logps/chosen": -1.6077661514282227,
      "logps/rejected": -1.9581828117370605,
      "loss": 1.7557,
      "nll_loss": 1.7038142681121826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16077663004398346,
      "rewards/margins": 0.03504165634512901,
      "rewards/rejected": -0.19581827521324158,
      "step": 328
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 0.2492428869009018,
      "learning_rate": 3.7024737876175404e-06,
      "log_odds_chosen": 0.39084863662719727,
      "log_odds_ratio": -0.5200874209403992,
      "logits/chosen": 0.17618009448051453,
      "logits/rejected": -0.8343906998634338,
      "logps/chosen": -1.6858640909194946,
      "logps/rejected": -2.01192307472229,
      "loss": 1.8105,
      "nll_loss": 1.7585185766220093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16858640313148499,
      "rewards/margins": 0.03260589390993118,
      "rewards/rejected": -0.20119231939315796,
      "step": 329
    },
    {
      "epoch": 0.5200945626477541,
      "grad_norm": 0.24973885715007782,
      "learning_rate": 3.699582583504874e-06,
      "log_odds_chosen": 0.31327661871910095,
      "log_odds_ratio": -0.5593623518943787,
      "logits/chosen": 0.20105372369289398,
      "logits/rejected": -0.9870561361312866,
      "logps/chosen": -1.8093860149383545,
      "logps/rejected": -2.079317569732666,
      "loss": 1.9242,
      "nll_loss": 1.8682433366775513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18093861639499664,
      "rewards/margins": 0.02699316293001175,
      "rewards/rejected": -0.2079317718744278,
      "step": 330
    },
    {
      "epoch": 0.5216706067769897,
      "grad_norm": 0.24848176538944244,
      "learning_rate": 3.696678539857571e-06,
      "log_odds_chosen": 0.4067533314228058,
      "log_odds_ratio": -0.5259693264961243,
      "logits/chosen": 0.15454429388046265,
      "logits/rejected": -1.0274417400360107,
      "logps/chosen": -1.783982515335083,
      "logps/rejected": -2.1373672485351562,
      "loss": 1.9076,
      "nll_loss": 1.8550231456756592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1783982515335083,
      "rewards/margins": 0.03533848002552986,
      "rewards/rejected": -0.21373675763607025,
      "step": 331
    },
    {
      "epoch": 0.5232466509062254,
      "grad_norm": 0.24895605444908142,
      "learning_rate": 3.6937616786142956e-06,
      "log_odds_chosen": 0.4905010759830475,
      "log_odds_ratio": -0.4821808338165283,
      "logits/chosen": 0.19289091229438782,
      "logits/rejected": -0.7941063642501831,
      "logps/chosen": -1.6132020950317383,
      "logps/rejected": -2.0248632431030273,
      "loss": 1.7617,
      "nll_loss": 1.7134675979614258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16132019460201263,
      "rewards/margins": 0.04116611182689667,
      "rewards/rejected": -0.2024863064289093,
      "step": 332
    },
    {
      "epoch": 0.524822695035461,
      "grad_norm": 0.2128501832485199,
      "learning_rate": 3.6908320218105393e-06,
      "log_odds_chosen": 0.28758326172828674,
      "log_odds_ratio": -0.5683539509773254,
      "logits/chosen": 0.1749098002910614,
      "logits/rejected": -0.9845055341720581,
      "logps/chosen": -1.6016740798950195,
      "logps/rejected": -1.8421180248260498,
      "loss": 1.7326,
      "nll_loss": 1.6757293939590454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1601674109697342,
      "rewards/margins": 0.02404441311955452,
      "rewards/rejected": -0.1842118203639984,
      "step": 333
    },
    {
      "epoch": 0.5263987391646966,
      "grad_norm": 0.21799606084823608,
      "learning_rate": 3.6878895915784607e-06,
      "log_odds_chosen": 0.4461507797241211,
      "log_odds_ratio": -0.4998866617679596,
      "logits/chosen": 0.20331019163131714,
      "logits/rejected": -0.9468520283699036,
      "logps/chosen": -1.565372109413147,
      "logps/rejected": -1.9329590797424316,
      "loss": 1.694,
      "nll_loss": 1.644019603729248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15653720498085022,
      "rewards/margins": 0.036758698523044586,
      "rewards/rejected": -0.1932959109544754,
      "step": 334
    },
    {
      "epoch": 0.5279747832939322,
      "grad_norm": 0.2156113237142563,
      "learning_rate": 3.6849344101467147e-06,
      "log_odds_chosen": 0.38997355103492737,
      "log_odds_ratio": -0.5264524221420288,
      "logits/chosen": 0.19628331065177917,
      "logits/rejected": -1.0862376689910889,
      "logps/chosen": -1.646598219871521,
      "logps/rejected": -1.9763703346252441,
      "loss": 1.7834,
      "nll_loss": 1.730759859085083,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16465981304645538,
      "rewards/margins": 0.03297720104455948,
      "rewards/rejected": -0.19763700664043427,
      "step": 335
    },
    {
      "epoch": 0.5295508274231678,
      "grad_norm": 0.23609226942062378,
      "learning_rate": 3.6819664998402857e-06,
      "log_odds_chosen": 0.3619554936885834,
      "log_odds_ratio": -0.5322998762130737,
      "logits/chosen": 0.17890314757823944,
      "logits/rejected": -0.9327296018600464,
      "logps/chosen": -1.6979272365570068,
      "logps/rejected": -2.0048635005950928,
      "loss": 1.8282,
      "nll_loss": 1.7749249935150146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1697927564382553,
      "rewards/margins": 0.030693601816892624,
      "rewards/rejected": -0.20048634707927704,
      "step": 336
    },
    {
      "epoch": 0.5311268715524035,
      "grad_norm": 0.24500414729118347,
      "learning_rate": 3.6789858830803186e-06,
      "log_odds_chosen": 0.3548503518104553,
      "log_odds_ratio": -0.5371453762054443,
      "logits/chosen": 0.19275718927383423,
      "logits/rejected": -0.961004912853241,
      "logps/chosen": -1.751190423965454,
      "logps/rejected": -2.051436424255371,
      "loss": 1.8707,
      "nll_loss": 1.8169537782669067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17511902749538422,
      "rewards/margins": 0.03002461977303028,
      "rewards/rejected": -0.20514366030693054,
      "step": 337
    },
    {
      "epoch": 0.5327029156816391,
      "grad_norm": 0.2349151223897934,
      "learning_rate": 3.6759925823839486e-06,
      "log_odds_chosen": 0.2844861149787903,
      "log_odds_ratio": -0.5660067200660706,
      "logits/chosen": 0.14869461953639984,
      "logits/rejected": -0.836445689201355,
      "logps/chosen": -1.6251897811889648,
      "logps/rejected": -1.8610213994979858,
      "loss": 1.7789,
      "nll_loss": 1.722307562828064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1625189632177353,
      "rewards/margins": 0.023583177477121353,
      "rewards/rejected": -0.18610213696956635,
      "step": 338
    },
    {
      "epoch": 0.5342789598108747,
      "grad_norm": 0.21934184432029724,
      "learning_rate": 3.672986620364134e-06,
      "log_odds_chosen": 0.4246414601802826,
      "log_odds_ratio": -0.5082186460494995,
      "logits/chosen": 0.21185454726219177,
      "logits/rejected": -0.8254536986351013,
      "logps/chosen": -1.7098535299301147,
      "logps/rejected": -2.0699710845947266,
      "loss": 1.8338,
      "nll_loss": 1.7829303741455078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1709853559732437,
      "rewards/margins": 0.036011770367622375,
      "rewards/rejected": -0.2069971263408661,
      "step": 339
    },
    {
      "epoch": 0.5358550039401103,
      "grad_norm": 0.22029125690460205,
      "learning_rate": 3.669968019729481e-06,
      "log_odds_chosen": 0.4750698208808899,
      "log_odds_ratio": -0.4959007501602173,
      "logits/chosen": 0.14572562277317047,
      "logits/rejected": -1.0968286991119385,
      "logps/chosen": -1.5743448734283447,
      "logps/rejected": -1.9716068506240845,
      "loss": 1.7095,
      "nll_loss": 1.6599397659301758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15743449330329895,
      "rewards/margins": 0.039726193994283676,
      "rewards/rejected": -0.19716067612171173,
      "step": 340
    },
    {
      "epoch": 0.5374310480693459,
      "grad_norm": 0.2289605438709259,
      "learning_rate": 3.666936803284076e-06,
      "log_odds_chosen": 0.4306865632534027,
      "log_odds_ratio": -0.5031254887580872,
      "logits/chosen": 0.1732909083366394,
      "logits/rejected": -0.9109163880348206,
      "logps/chosen": -1.721662998199463,
      "logps/rejected": -2.0888686180114746,
      "loss": 1.8312,
      "nll_loss": 1.780855417251587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17216630280017853,
      "rewards/margins": 0.03672058507800102,
      "rewards/rejected": -0.20888687670230865,
      "step": 341
    },
    {
      "epoch": 0.5390070921985816,
      "grad_norm": 0.23886913061141968,
      "learning_rate": 3.663892993927312e-06,
      "log_odds_chosen": 0.5287318825721741,
      "log_odds_ratio": -0.46768540143966675,
      "logits/chosen": 0.10965054482221603,
      "logits/rejected": -1.0303406715393066,
      "logps/chosen": -1.671059012413025,
      "logps/rejected": -2.1188931465148926,
      "loss": 1.8029,
      "nll_loss": 1.7561312913894653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16710591316223145,
      "rewards/margins": 0.044783420860767365,
      "rewards/rejected": -0.2118893265724182,
      "step": 342
    },
    {
      "epoch": 0.5405831363278172,
      "grad_norm": 0.2293994128704071,
      "learning_rate": 3.6608366146537136e-06,
      "log_odds_chosen": 0.552444577217102,
      "log_odds_ratio": -0.47390323877334595,
      "logits/chosen": 0.09345658123493195,
      "logits/rejected": -0.875726580619812,
      "logps/chosen": -1.5994176864624023,
      "logps/rejected": -2.0705385208129883,
      "loss": 1.7346,
      "nll_loss": 1.6872093677520752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15994176268577576,
      "rewards/margins": 0.047112107276916504,
      "rewards/rejected": -0.20705386996269226,
      "step": 343
    },
    {
      "epoch": 0.5421591804570528,
      "grad_norm": 0.2236412763595581,
      "learning_rate": 3.6577676885527674e-06,
      "log_odds_chosen": 0.46793222427368164,
      "log_odds_ratio": -0.4905901551246643,
      "logits/chosen": 0.10005174577236176,
      "logits/rejected": -1.1285041570663452,
      "logps/chosen": -1.649950623512268,
      "logps/rejected": -2.043297290802002,
      "loss": 1.7655,
      "nll_loss": 1.716444492340088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16499507427215576,
      "rewards/margins": 0.03933466970920563,
      "rewards/rejected": -0.2043297290802002,
      "step": 344
    },
    {
      "epoch": 0.5437352245862884,
      "grad_norm": 0.21764251589775085,
      "learning_rate": 3.654686238808744e-06,
      "log_odds_chosen": 0.4737105667591095,
      "log_odds_ratio": -0.4936067461967468,
      "logits/chosen": 0.0544014573097229,
      "logits/rejected": -1.042222023010254,
      "logps/chosen": -1.746705174446106,
      "logps/rejected": -2.1518256664276123,
      "loss": 1.863,
      "nll_loss": 1.8135954141616821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1746705323457718,
      "rewards/margins": 0.04051202908158302,
      "rewards/rejected": -0.21518255770206451,
      "step": 345
    },
    {
      "epoch": 0.545311268715524,
      "grad_norm": 0.1986258327960968,
      "learning_rate": 3.6515922887005245e-06,
      "log_odds_chosen": 0.6257790923118591,
      "log_odds_ratio": -0.4334947466850281,
      "logits/chosen": -0.02682601287961006,
      "logits/rejected": -1.096382737159729,
      "logps/chosen": -1.5822004079818726,
      "logps/rejected": -2.1046130657196045,
      "loss": 1.7103,
      "nll_loss": 1.6669962406158447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1582200527191162,
      "rewards/margins": 0.052241258323192596,
      "rewards/rejected": -0.2104613035917282,
      "step": 346
    },
    {
      "epoch": 0.5468873128447597,
      "grad_norm": 0.20813308656215668,
      "learning_rate": 3.6484858616014236e-06,
      "log_odds_chosen": 0.256540983915329,
      "log_odds_ratio": -0.5760852098464966,
      "logits/chosen": 0.12902772426605225,
      "logits/rejected": -1.1435496807098389,
      "logps/chosen": -1.707273244857788,
      "logps/rejected": -1.9220017194747925,
      "loss": 1.8215,
      "nll_loss": 1.763900876045227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17072734236717224,
      "rewards/margins": 0.02147284336388111,
      "rewards/rejected": -0.19220015406608582,
      "step": 347
    },
    {
      "epoch": 0.5484633569739953,
      "grad_norm": 0.3381063640117645,
      "learning_rate": 3.6453669809790154e-06,
      "log_odds_chosen": 0.2898363769054413,
      "log_odds_ratio": -0.5690465569496155,
      "logits/chosen": 0.1767110675573349,
      "logits/rejected": -0.8112865090370178,
      "logps/chosen": -1.6394524574279785,
      "logps/rejected": -1.8821648359298706,
      "loss": 1.765,
      "nll_loss": 1.7080632448196411,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1639452427625656,
      "rewards/margins": 0.02427123486995697,
      "rewards/rejected": -0.18821649253368378,
      "step": 348
    },
    {
      "epoch": 0.5500394011032309,
      "grad_norm": 0.21555578708648682,
      "learning_rate": 3.642235670394952e-06,
      "log_odds_chosen": 0.3761124312877655,
      "log_odds_ratio": -0.528813362121582,
      "logits/chosen": 0.13704705238342285,
      "logits/rejected": -0.8061936497688293,
      "logps/chosen": -1.6551097631454468,
      "logps/rejected": -1.9656535387039185,
      "loss": 1.7871,
      "nll_loss": 1.7342618703842163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16551096737384796,
      "rewards/margins": 0.03105437569320202,
      "rewards/rejected": -0.19656534492969513,
      "step": 349
    },
    {
      "epoch": 0.5516154452324665,
      "grad_norm": 0.20423917472362518,
      "learning_rate": 3.63909195350479e-06,
      "log_odds_chosen": 0.3855099081993103,
      "log_odds_ratio": -0.5282479524612427,
      "logits/chosen": 0.2054780125617981,
      "logits/rejected": -0.9389649629592896,
      "logps/chosen": -1.7321782112121582,
      "logps/rejected": -2.0566914081573486,
      "loss": 1.8494,
      "nll_loss": 1.7965946197509766,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17321781814098358,
      "rewards/margins": 0.032451316714286804,
      "rewards/rejected": -0.20566913485527039,
      "step": 350
    },
    {
      "epoch": 0.5531914893617021,
      "grad_norm": 0.22335051000118256,
      "learning_rate": 3.635935854057809e-06,
      "log_odds_chosen": 0.3637022376060486,
      "log_odds_ratio": -0.5370145440101624,
      "logits/chosen": 0.021777212619781494,
      "logits/rejected": -0.6934305429458618,
      "logps/chosen": -1.7031822204589844,
      "logps/rejected": -2.01242733001709,
      "loss": 1.8245,
      "nll_loss": 1.770749568939209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17031821608543396,
      "rewards/margins": 0.03092453069984913,
      "rewards/rejected": -0.20124275982379913,
      "step": 351
    },
    {
      "epoch": 0.5547675334909378,
      "grad_norm": 0.21424627304077148,
      "learning_rate": 3.6327673958968327e-06,
      "log_odds_chosen": 0.5423768758773804,
      "log_odds_ratio": -0.4632525146007538,
      "logits/chosen": 0.051188066601753235,
      "logits/rejected": -0.9619206190109253,
      "logps/chosen": -1.5644475221633911,
      "logps/rejected": -2.0145986080169678,
      "loss": 1.7063,
      "nll_loss": 1.6599864959716797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1564447581768036,
      "rewards/margins": 0.045015111565589905,
      "rewards/rejected": -0.2014598697423935,
      "step": 352
    },
    {
      "epoch": 0.5563435776201734,
      "grad_norm": 0.2026013284921646,
      "learning_rate": 3.6295866029580483e-06,
      "log_odds_chosen": 0.4790729284286499,
      "log_odds_ratio": -0.4863028824329376,
      "logits/chosen": 0.06050941348075867,
      "logits/rejected": -1.0654728412628174,
      "logps/chosen": -1.605035424232483,
      "logps/rejected": -2.004734516143799,
      "loss": 1.7168,
      "nll_loss": 1.6681344509124756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.160503551363945,
      "rewards/margins": 0.039969902485609055,
      "rewards/rejected": -0.20047345757484436,
      "step": 353
    },
    {
      "epoch": 0.557919621749409,
      "grad_norm": 0.20405229926109314,
      "learning_rate": 3.626393499270829e-06,
      "log_odds_chosen": 0.4320348799228668,
      "log_odds_ratio": -0.5096400380134583,
      "logits/chosen": 0.07103009521961212,
      "logits/rejected": -1.0413506031036377,
      "logps/chosen": -1.5444334745407104,
      "logps/rejected": -1.9000470638275146,
      "loss": 1.6895,
      "nll_loss": 1.6385679244995117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15444333851337433,
      "rewards/margins": 0.03556136414408684,
      "rewards/rejected": -0.19000470638275146,
      "step": 354
    },
    {
      "epoch": 0.5594956658786446,
      "grad_norm": 0.2170814573764801,
      "learning_rate": 3.6231881089575466e-06,
      "log_odds_chosen": 0.5878276228904724,
      "log_odds_ratio": -0.4456479847431183,
      "logits/chosen": 0.027034137398004532,
      "logits/rejected": -1.0812389850616455,
      "logps/chosen": -1.6104373931884766,
      "logps/rejected": -2.1062474250793457,
      "loss": 1.7061,
      "nll_loss": 1.6615161895751953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16104374825954437,
      "rewards/margins": 0.04958099126815796,
      "rewards/rejected": -0.21062473952770233,
      "step": 355
    },
    {
      "epoch": 0.5610717100078803,
      "grad_norm": 0.2183144986629486,
      "learning_rate": 3.6199704562333945e-06,
      "log_odds_chosen": 0.40854716300964355,
      "log_odds_ratio": -0.5191488265991211,
      "logits/chosen": 0.029446076601743698,
      "logits/rejected": -1.0460023880004883,
      "logps/chosen": -1.5693233013153076,
      "logps/rejected": -1.903488278388977,
      "loss": 1.6957,
      "nll_loss": 1.643739938735962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15693233907222748,
      "rewards/margins": 0.0334165021777153,
      "rewards/rejected": -0.19034883379936218,
      "step": 356
    },
    {
      "epoch": 0.5626477541371159,
      "grad_norm": 0.2159062922000885,
      "learning_rate": 3.6167405654062024e-06,
      "log_odds_chosen": 0.4419677257537842,
      "log_odds_ratio": -0.5147860050201416,
      "logits/chosen": 0.08764218538999557,
      "logits/rejected": -0.9715548157691956,
      "logps/chosen": -1.6319985389709473,
      "logps/rejected": -2.0011050701141357,
      "loss": 1.7458,
      "nll_loss": 1.694358229637146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16319985687732697,
      "rewards/margins": 0.03691064938902855,
      "rewards/rejected": -0.2001105099916458,
      "step": 357
    },
    {
      "epoch": 0.5642237982663515,
      "grad_norm": 0.19851958751678467,
      "learning_rate": 3.6134984608762515e-06,
      "log_odds_chosen": 0.4827573299407959,
      "log_odds_ratio": -0.4849916100502014,
      "logits/chosen": 0.073295459151268,
      "logits/rejected": -1.0616111755371094,
      "logps/chosen": -1.5443633794784546,
      "logps/rejected": -1.9410005807876587,
      "loss": 1.6782,
      "nll_loss": 1.6297074556350708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15443634986877441,
      "rewards/margins": 0.03966371342539787,
      "rewards/rejected": -0.19410006701946259,
      "step": 358
    },
    {
      "epoch": 0.5657998423955871,
      "grad_norm": 0.23549191653728485,
      "learning_rate": 3.6102441671360945e-06,
      "log_odds_chosen": 0.4817807078361511,
      "log_odds_ratio": -0.49621692299842834,
      "logits/chosen": 0.05556647107005119,
      "logits/rejected": -0.9252523183822632,
      "logps/chosen": -1.7002850770950317,
      "logps/rejected": -2.1108858585357666,
      "loss": 1.8078,
      "nll_loss": 1.7582156658172607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17002852261066437,
      "rewards/margins": 0.04106004908680916,
      "rewards/rejected": -0.21108856797218323,
      "step": 359
    },
    {
      "epoch": 0.5673758865248227,
      "grad_norm": 0.2164311408996582,
      "learning_rate": 3.6069777087703654e-06,
      "log_odds_chosen": 0.47711795568466187,
      "log_odds_ratio": -0.4905511140823364,
      "logits/chosen": 0.06711583584547043,
      "logits/rejected": -0.9263973832130432,
      "logps/chosen": -1.6224663257598877,
      "logps/rejected": -2.021437168121338,
      "loss": 1.7555,
      "nll_loss": 1.7064476013183594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16224664449691772,
      "rewards/margins": 0.039897091686725616,
      "rewards/rejected": -0.20214374363422394,
      "step": 360
    },
    {
      "epoch": 0.5689519306540584,
      "grad_norm": 0.21710002422332764,
      "learning_rate": 3.6036991104555973e-06,
      "log_odds_chosen": 0.407501757144928,
      "log_odds_ratio": -0.5239275693893433,
      "logits/chosen": 0.07704886794090271,
      "logits/rejected": -0.9694811105728149,
      "logps/chosen": -1.7658026218414307,
      "logps/rejected": -2.117172956466675,
      "loss": 1.8712,
      "nll_loss": 1.8187607526779175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1765802651643753,
      "rewards/margins": 0.03513704985380173,
      "rewards/rejected": -0.21171732246875763,
      "step": 361
    },
    {
      "epoch": 0.570527974783294,
      "grad_norm": 0.20836122334003448,
      "learning_rate": 3.600408396960034e-06,
      "log_odds_chosen": 0.47454649209976196,
      "log_odds_ratio": -0.49302300810813904,
      "logits/chosen": 0.09529343247413635,
      "logits/rejected": -0.6717502474784851,
      "logps/chosen": -1.5604360103607178,
      "logps/rejected": -1.9463016986846924,
      "loss": 1.6685,
      "nll_loss": 1.6191602945327759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15604360401630402,
      "rewards/margins": 0.038586560636758804,
      "rewards/rejected": -0.19463016092777252,
      "step": 362
    },
    {
      "epoch": 0.5721040189125296,
      "grad_norm": 0.22342000901699066,
      "learning_rate": 3.5971055931434447e-06,
      "log_odds_chosen": 0.42324817180633545,
      "log_odds_ratio": -0.5131744742393494,
      "logits/chosen": 0.11431422084569931,
      "logits/rejected": -0.8911008834838867,
      "logps/chosen": -1.6157538890838623,
      "logps/rejected": -1.9717456102371216,
      "loss": 1.7379,
      "nll_loss": 1.6865862607955933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16157537698745728,
      "rewards/margins": 0.035599179565906525,
      "rewards/rejected": -0.1971745789051056,
      "step": 363
    },
    {
      "epoch": 0.5736800630417652,
      "grad_norm": 0.20487506687641144,
      "learning_rate": 3.5937907239569343e-06,
      "log_odds_chosen": 0.46312034130096436,
      "log_odds_ratio": -0.5019612908363342,
      "logits/chosen": 0.022356808185577393,
      "logits/rejected": -0.989889919757843,
      "logps/chosen": -1.667398452758789,
      "logps/rejected": -2.0568132400512695,
      "loss": 1.7903,
      "nll_loss": 1.7401193380355835,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16673985123634338,
      "rewards/margins": 0.038941483944654465,
      "rewards/rejected": -0.20568133890628815,
      "step": 364
    },
    {
      "epoch": 0.5752561071710008,
      "grad_norm": 0.23341180384159088,
      "learning_rate": 3.5904638144427572e-06,
      "log_odds_chosen": 0.2531343400478363,
      "log_odds_ratio": -0.5829412937164307,
      "logits/chosen": 0.07688252627849579,
      "logits/rejected": -0.7990673780441284,
      "logps/chosen": -1.73116135597229,
      "logps/rejected": -1.9438457489013672,
      "loss": 1.8433,
      "nll_loss": 1.7850035429000854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17311613261699677,
      "rewards/margins": 0.021268434822559357,
      "rewards/rejected": -0.19438455998897552,
      "step": 365
    },
    {
      "epoch": 0.5768321513002365,
      "grad_norm": 0.2151210755109787,
      "learning_rate": 3.5871248897341246e-06,
      "log_odds_chosen": 0.4656566381454468,
      "log_odds_ratio": -0.493457555770874,
      "logits/chosen": 0.033955443650484085,
      "logits/rejected": -0.7494514584541321,
      "logps/chosen": -1.5631694793701172,
      "logps/rejected": -1.954725742340088,
      "loss": 1.7017,
      "nll_loss": 1.6523396968841553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15631696581840515,
      "rewards/margins": 0.03915562108159065,
      "rewards/rejected": -0.1954725980758667,
      "step": 366
    },
    {
      "epoch": 0.578408195429472,
      "grad_norm": 0.19448134303092957,
      "learning_rate": 3.5837739750550182e-06,
      "log_odds_chosen": 0.4710557460784912,
      "log_odds_ratio": -0.4958794116973877,
      "logits/chosen": -0.001226938795298338,
      "logits/rejected": -1.270828127861023,
      "logps/chosen": -1.6350599527359009,
      "logps/rejected": -2.034780740737915,
      "loss": 1.7501,
      "nll_loss": 1.7005350589752197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16350601613521576,
      "rewards/margins": 0.03997207060456276,
      "rewards/rejected": -0.20347808301448822,
      "step": 367
    },
    {
      "epoch": 0.5799842395587076,
      "grad_norm": 0.19283808767795563,
      "learning_rate": 3.5804110957199977e-06,
      "log_odds_chosen": 0.48457401990890503,
      "log_odds_ratio": -0.48585668206214905,
      "logits/chosen": 0.1169048473238945,
      "logits/rejected": -0.8879725337028503,
      "logps/chosen": -1.61647367477417,
      "logps/rejected": -2.023709297180176,
      "loss": 1.7294,
      "nll_loss": 1.6807751655578613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16164736449718475,
      "rewards/margins": 0.04072355851531029,
      "rewards/rejected": -0.20237095654010773,
      "step": 368
    },
    {
      "epoch": 0.5815602836879432,
      "grad_norm": 0.19826547801494598,
      "learning_rate": 3.577036277134011e-06,
      "log_odds_chosen": 0.5813598036766052,
      "log_odds_ratio": -0.4480295777320862,
      "logits/chosen": 0.03684981167316437,
      "logits/rejected": -1.0105152130126953,
      "logps/chosen": -1.6038864850997925,
      "logps/rejected": -2.0928397178649902,
      "loss": 1.7299,
      "nll_loss": 1.6851170063018799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16038864850997925,
      "rewards/margins": 0.04889533668756485,
      "rewards/rejected": -0.2092839926481247,
      "step": 369
    },
    {
      "epoch": 0.5831363278171788,
      "grad_norm": 0.21686360239982605,
      "learning_rate": 3.5736495447922e-06,
      "log_odds_chosen": 0.3633495569229126,
      "log_odds_ratio": -0.5332463383674622,
      "logits/chosen": 0.037289127707481384,
      "logits/rejected": -1.074998140335083,
      "logps/chosen": -1.7311910390853882,
      "logps/rejected": -2.037515878677368,
      "loss": 1.847,
      "nll_loss": 1.7936969995498657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17311908304691315,
      "rewards/margins": 0.03063248284161091,
      "rewards/rejected": -0.2037515640258789,
      "step": 370
    },
    {
      "epoch": 0.5847123719464145,
      "grad_norm": 0.18624477088451385,
      "learning_rate": 3.5702509242797096e-06,
      "log_odds_chosen": 0.6746554970741272,
      "log_odds_ratio": -0.4293067753314972,
      "logits/chosen": 0.031168414279818535,
      "logits/rejected": -1.0391783714294434,
      "logps/chosen": -1.5503772497177124,
      "logps/rejected": -2.1166985034942627,
      "loss": 1.6782,
      "nll_loss": 1.6352722644805908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15503773093223572,
      "rewards/margins": 0.05663213133811951,
      "rewards/rejected": -0.21166983246803284,
      "step": 371
    },
    {
      "epoch": 0.5862884160756501,
      "grad_norm": 0.19968333840370178,
      "learning_rate": 3.566840441271495e-06,
      "log_odds_chosen": 0.630666196346283,
      "log_odds_ratio": -0.44081687927246094,
      "logits/chosen": 0.06830377876758575,
      "logits/rejected": -1.0593217611312866,
      "logps/chosen": -1.5753437280654907,
      "logps/rejected": -2.1008994579315186,
      "loss": 1.6789,
      "nll_loss": 1.634864091873169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1575343906879425,
      "rewards/margins": 0.05255556106567383,
      "rewards/rejected": -0.21008995175361633,
      "step": 372
    },
    {
      "epoch": 0.5878644602048857,
      "grad_norm": 0.20025794208049774,
      "learning_rate": 3.5634181215321265e-06,
      "log_odds_chosen": 0.663584291934967,
      "log_odds_ratio": -0.4243214428424835,
      "logits/chosen": 0.07222753763198853,
      "logits/rejected": -0.9715753793716431,
      "logps/chosen": -1.6212844848632812,
      "logps/rejected": -2.1870157718658447,
      "loss": 1.7468,
      "nll_loss": 1.7043309211730957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16212844848632812,
      "rewards/margins": 0.05657312646508217,
      "rewards/rejected": -0.2187015861272812,
      "step": 373
    },
    {
      "epoch": 0.5894405043341213,
      "grad_norm": 0.19907920062541962,
      "learning_rate": 3.5599839909155947e-06,
      "log_odds_chosen": 0.4552556872367859,
      "log_odds_ratio": -0.5016735196113586,
      "logits/chosen": 0.0466487854719162,
      "logits/rejected": -1.0045429468154907,
      "logps/chosen": -1.584489107131958,
      "logps/rejected": -1.9633792638778687,
      "loss": 1.7012,
      "nll_loss": 1.6509945392608643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15844891965389252,
      "rewards/margins": 0.03788900747895241,
      "rewards/rejected": -0.19633793830871582,
      "step": 374
    },
    {
      "epoch": 0.5910165484633569,
      "grad_norm": 0.19784437119960785,
      "learning_rate": 3.556538075365116e-06,
      "log_odds_chosen": 0.5365445017814636,
      "log_odds_ratio": -0.4729233384132385,
      "logits/chosen": 0.10310062021017075,
      "logits/rejected": -1.0228643417358398,
      "logps/chosen": -1.5829472541809082,
      "logps/rejected": -2.030022144317627,
      "loss": 1.6987,
      "nll_loss": 1.651432752609253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15829472243785858,
      "rewards/margins": 0.04470750689506531,
      "rewards/rejected": -0.2030022293329239,
      "step": 375
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 0.2056477963924408,
      "learning_rate": 3.5530804009129367e-06,
      "log_odds_chosen": 0.5238441228866577,
      "log_odds_ratio": -0.47168248891830444,
      "logits/chosen": 0.05509530007839203,
      "logits/rejected": -0.8633387684822083,
      "logps/chosen": -1.6203933954238892,
      "logps/rejected": -2.0622262954711914,
      "loss": 1.7216,
      "nll_loss": 1.6743903160095215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16203933954238892,
      "rewards/margins": 0.04418327659368515,
      "rewards/rejected": -0.20622262358665466,
      "step": 376
    },
    {
      "epoch": 0.5941686367218282,
      "grad_norm": 0.20296871662139893,
      "learning_rate": 3.5496109936801368e-06,
      "log_odds_chosen": 0.47061851620674133,
      "log_odds_ratio": -0.4923834204673767,
      "logits/chosen": -0.022969774901866913,
      "logits/rejected": -1.096977710723877,
      "logps/chosen": -1.6428031921386719,
      "logps/rejected": -2.04349946975708,
      "loss": 1.7551,
      "nll_loss": 1.7058473825454712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16428032517433167,
      "rewards/margins": 0.04006962105631828,
      "rewards/rejected": -0.20434993505477905,
      "step": 377
    },
    {
      "epoch": 0.5957446808510638,
      "grad_norm": 0.21032318472862244,
      "learning_rate": 3.546129879876429e-06,
      "log_odds_chosen": 0.3223990797996521,
      "log_odds_ratio": -0.5593502521514893,
      "logits/chosen": 0.11473214626312256,
      "logits/rejected": -0.7919275760650635,
      "logps/chosen": -1.7105305194854736,
      "logps/rejected": -1.983834981918335,
      "loss": 1.8197,
      "nll_loss": 1.763751745223999,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17105305194854736,
      "rewards/margins": 0.027330443263053894,
      "rewards/rejected": -0.19838349521160126,
      "step": 378
    },
    {
      "epoch": 0.5973207249802994,
      "grad_norm": 0.20299871265888214,
      "learning_rate": 3.5426370857999662e-06,
      "log_odds_chosen": 0.32067781686782837,
      "log_odds_ratio": -0.5509010553359985,
      "logits/chosen": 0.04862087219953537,
      "logits/rejected": -0.9239585399627686,
      "logps/chosen": -1.6904906034469604,
      "logps/rejected": -1.9611345529556274,
      "loss": 1.8102,
      "nll_loss": 1.7551449537277222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16904906928539276,
      "rewards/margins": 0.027064397931098938,
      "rewards/rejected": -0.1961134672164917,
      "step": 379
    },
    {
      "epoch": 0.598896769109535,
      "grad_norm": 0.20678678154945374,
      "learning_rate": 3.53913263783714e-06,
      "log_odds_chosen": 0.37589550018310547,
      "log_odds_ratio": -0.5370999574661255,
      "logits/chosen": 0.007449129596352577,
      "logits/rejected": -1.0229346752166748,
      "logps/chosen": -1.633695363998413,
      "logps/rejected": -1.9534492492675781,
      "loss": 1.7419,
      "nll_loss": 1.6882388591766357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1633695513010025,
      "rewards/margins": 0.0319753997027874,
      "rewards/rejected": -0.195344939827919,
      "step": 380
    },
    {
      "epoch": 0.6004728132387707,
      "grad_norm": 0.19402244687080383,
      "learning_rate": 3.53561656246238e-06,
      "log_odds_chosen": 0.6378045678138733,
      "log_odds_ratio": -0.42818355560302734,
      "logits/chosen": -0.08074290305376053,
      "logits/rejected": -1.2157189846038818,
      "logps/chosen": -1.6589109897613525,
      "logps/rejected": -2.201770782470703,
      "loss": 1.7652,
      "nll_loss": 1.7223517894744873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1658911108970642,
      "rewards/margins": 0.05428597331047058,
      "rewards/rejected": -0.2201770842075348,
      "step": 381
    },
    {
      "epoch": 0.6020488573680063,
      "grad_norm": 0.19140584766864777,
      "learning_rate": 3.532088886237956e-06,
      "log_odds_chosen": 0.5052012801170349,
      "log_odds_ratio": -0.4754614233970642,
      "logits/chosen": -0.016120534390211105,
      "logits/rejected": -1.1966180801391602,
      "logps/chosen": -1.6766316890716553,
      "logps/rejected": -2.1046011447906494,
      "loss": 1.7709,
      "nll_loss": 1.7233370542526245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16766318678855896,
      "rewards/margins": 0.042796939611434937,
      "rewards/rejected": -0.2104601114988327,
      "step": 382
    },
    {
      "epoch": 0.6036249014972419,
      "grad_norm": 0.18412470817565918,
      "learning_rate": 3.528549635813778e-06,
      "log_odds_chosen": 0.5283522605895996,
      "log_odds_ratio": -0.4716714024543762,
      "logits/chosen": -0.04352571442723274,
      "logits/rejected": -1.0171538591384888,
      "logps/chosen": -1.5115892887115479,
      "logps/rejected": -1.9484082460403442,
      "loss": 1.6296,
      "nll_loss": 1.5823882818222046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15115892887115479,
      "rewards/margins": 0.043681904673576355,
      "rewards/rejected": -0.19484084844589233,
      "step": 383
    },
    {
      "epoch": 0.6052009456264775,
      "grad_norm": 0.2047756463289261,
      "learning_rate": 3.524998837927192e-06,
      "log_odds_chosen": 0.6116353869438171,
      "log_odds_ratio": -0.43900421261787415,
      "logits/chosen": 0.008262881077826023,
      "logits/rejected": -0.9327568411827087,
      "logps/chosen": -1.645035982131958,
      "logps/rejected": -2.1674892902374268,
      "loss": 1.7375,
      "nll_loss": 1.693565845489502,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16450358927249908,
      "rewards/margins": 0.05224531143903732,
      "rewards/rejected": -0.2167489230632782,
      "step": 384
    },
    {
      "epoch": 0.6067769897557131,
      "grad_norm": 0.18797335028648376,
      "learning_rate": 3.5214365194027797e-06,
      "log_odds_chosen": 0.5875496864318848,
      "log_odds_ratio": -0.4458829462528229,
      "logits/chosen": -0.019994519650936127,
      "logits/rejected": -1.0720854997634888,
      "logps/chosen": -1.5490386486053467,
      "logps/rejected": -2.0399489402770996,
      "loss": 1.656,
      "nll_loss": 1.6113654375076294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15490387380123138,
      "rewards/margins": 0.04909101501107216,
      "rewards/rejected": -0.20399489998817444,
      "step": 385
    },
    {
      "epoch": 0.6083530338849488,
      "grad_norm": 0.21445293724536896,
      "learning_rate": 3.517862707152157e-06,
      "log_odds_chosen": 0.45695003867149353,
      "log_odds_ratio": -0.5020439624786377,
      "logits/chosen": 0.08733657747507095,
      "logits/rejected": -0.7828851938247681,
      "logps/chosen": -1.7037779092788696,
      "logps/rejected": -2.0940959453582764,
      "loss": 1.7985,
      "nll_loss": 1.7482857704162598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17037779092788696,
      "rewards/margins": 0.03903179615736008,
      "rewards/rejected": -0.20940957963466644,
      "step": 386
    },
    {
      "epoch": 0.6099290780141844,
      "grad_norm": 0.2248964160680771,
      "learning_rate": 3.5142774281737674e-06,
      "log_odds_chosen": 0.6219518184661865,
      "log_odds_ratio": -0.43292707204818726,
      "logits/chosen": -0.0042352513410151005,
      "logits/rejected": -0.9333048462867737,
      "logps/chosen": -1.695968747138977,
      "logps/rejected": -2.2276930809020996,
      "loss": 1.7847,
      "nll_loss": 1.741443395614624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1695968508720398,
      "rewards/margins": 0.05317245051264763,
      "rewards/rejected": -0.22276932001113892,
      "step": 387
    },
    {
      "epoch": 0.61150512214342,
      "grad_norm": 0.18177717924118042,
      "learning_rate": 3.5106807095526817e-06,
      "log_odds_chosen": 0.6358584761619568,
      "log_odds_ratio": -0.43654751777648926,
      "logits/chosen": -0.039142899215221405,
      "logits/rejected": -1.1646403074264526,
      "logps/chosen": -1.6416778564453125,
      "logps/rejected": -2.176908493041992,
      "loss": 1.7464,
      "nll_loss": 1.7027642726898193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16416779160499573,
      "rewards/margins": 0.05352305620908737,
      "rewards/rejected": -0.2176908403635025,
      "step": 388
    },
    {
      "epoch": 0.6130811662726556,
      "grad_norm": 0.19595922529697418,
      "learning_rate": 3.5070725784603905e-06,
      "log_odds_chosen": 0.49472156167030334,
      "log_odds_ratio": -0.48498183488845825,
      "logits/chosen": -0.10919392853975296,
      "logits/rejected": -0.9267551898956299,
      "logps/chosen": -1.5107898712158203,
      "logps/rejected": -1.9186828136444092,
      "loss": 1.6134,
      "nll_loss": 1.5649324655532837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1510789841413498,
      "rewards/margins": 0.04078929126262665,
      "rewards/rejected": -0.19186827540397644,
      "step": 389
    },
    {
      "epoch": 0.6146572104018913,
      "grad_norm": 0.21639399230480194,
      "learning_rate": 3.503453062154602e-06,
      "log_odds_chosen": 0.42183130979537964,
      "log_odds_ratio": -0.5190197825431824,
      "logits/chosen": -0.023847615346312523,
      "logits/rejected": -0.8754997253417969,
      "logps/chosen": -1.6942306756973267,
      "logps/rejected": -2.06078839302063,
      "loss": 1.8063,
      "nll_loss": 1.754384160041809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16942308843135834,
      "rewards/margins": 0.036655765026807785,
      "rewards/rejected": -0.20607884228229523,
      "step": 390
    },
    {
      "epoch": 0.6162332545311269,
      "grad_norm": 0.1975736767053604,
      "learning_rate": 3.499822187979032e-06,
      "log_odds_chosen": 0.40866145491600037,
      "log_odds_ratio": -0.5196468830108643,
      "logits/chosen": 0.0417475588619709,
      "logits/rejected": -1.139929175376892,
      "logps/chosen": -1.6336414813995361,
      "logps/rejected": -1.9767978191375732,
      "loss": 1.7518,
      "nll_loss": 1.6998686790466309,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16336414217948914,
      "rewards/margins": 0.03431565314531326,
      "rewards/rejected": -0.1976797729730606,
      "step": 391
    },
    {
      "epoch": 0.6178092986603625,
      "grad_norm": 0.19061535596847534,
      "learning_rate": 3.496179983363202e-06,
      "log_odds_chosen": 0.37159737944602966,
      "log_odds_ratio": -0.5294693112373352,
      "logits/chosen": 0.022184930741786957,
      "logits/rejected": -0.9958785176277161,
      "logps/chosen": -1.6446539163589478,
      "logps/rejected": -1.9568779468536377,
      "loss": 1.7428,
      "nll_loss": 1.6898627281188965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16446539759635925,
      "rewards/margins": 0.031222393736243248,
      "rewards/rejected": -0.19568778574466705,
      "step": 392
    },
    {
      "epoch": 0.6193853427895981,
      "grad_norm": 0.2155338078737259,
      "learning_rate": 3.4925264758222268e-06,
      "log_odds_chosen": 0.5748679637908936,
      "log_odds_ratio": -0.45070305466651917,
      "logits/chosen": -0.02329150401055813,
      "logits/rejected": -0.8651071786880493,
      "logps/chosen": -1.5768085718154907,
      "logps/rejected": -2.059037208557129,
      "loss": 1.6901,
      "nll_loss": 1.645035982131958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15768086910247803,
      "rewards/margins": 0.0482228547334671,
      "rewards/rejected": -0.20590370893478394,
      "step": 393
    },
    {
      "epoch": 0.6209613869188337,
      "grad_norm": 0.19670893251895905,
      "learning_rate": 3.488861692956611e-06,
      "log_odds_chosen": 0.5193027257919312,
      "log_odds_ratio": -0.4729624092578888,
      "logits/chosen": -0.062314629554748535,
      "logits/rejected": -0.9838546514511108,
      "logps/chosen": -1.6177668571472168,
      "logps/rejected": -2.0553812980651855,
      "loss": 1.7207,
      "nll_loss": 1.6734217405319214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16177670657634735,
      "rewards/margins": 0.04376142844557762,
      "rewards/rejected": -0.20553813874721527,
      "step": 394
    },
    {
      "epoch": 0.6225374310480694,
      "grad_norm": 0.20605877041816711,
      "learning_rate": 3.4851856624520394e-06,
      "log_odds_chosen": 0.5313916802406311,
      "log_odds_ratio": -0.46628880500793457,
      "logits/chosen": 0.00429939292371273,
      "logits/rejected": -1.0193074941635132,
      "logps/chosen": -1.6535357236862183,
      "logps/rejected": -2.1009347438812256,
      "loss": 1.7514,
      "nll_loss": 1.7047302722930908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16535356640815735,
      "rewards/margins": 0.044739894568920135,
      "rewards/rejected": -0.21009346842765808,
      "step": 395
    },
    {
      "epoch": 0.624113475177305,
      "grad_norm": 0.2116907835006714,
      "learning_rate": 3.4814984120791664e-06,
      "log_odds_chosen": 0.4503478407859802,
      "log_odds_ratio": -0.4949477016925812,
      "logits/chosen": 0.009901851415634155,
      "logits/rejected": -0.8826244473457336,
      "logps/chosen": -1.6289476156234741,
      "logps/rejected": -2.004061698913574,
      "loss": 1.7392,
      "nll_loss": 1.6896657943725586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16289478540420532,
      "rewards/margins": 0.03751137852668762,
      "rewards/rejected": -0.20040614902973175,
      "step": 396
    },
    {
      "epoch": 0.6256895193065406,
      "grad_norm": 0.46380433440208435,
      "learning_rate": 3.477799969693407e-06,
      "log_odds_chosen": 0.4436909854412079,
      "log_odds_ratio": -0.5040198564529419,
      "logits/chosen": 0.04017059504985809,
      "logits/rejected": -0.9966317415237427,
      "logps/chosen": -1.6149952411651611,
      "logps/rejected": -1.980851173400879,
      "loss": 1.7189,
      "nll_loss": 1.6685235500335693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1614995300769806,
      "rewards/margins": 0.03658558428287506,
      "rewards/rejected": -0.19808511435985565,
      "step": 397
    },
    {
      "epoch": 0.6272655634357762,
      "grad_norm": 0.21046514809131622,
      "learning_rate": 3.474090363234728e-06,
      "log_odds_chosen": 0.7306087017059326,
      "log_odds_ratio": -0.4084674119949341,
      "logits/chosen": -0.1205843836069107,
      "logits/rejected": -1.0663551092147827,
      "logps/chosen": -1.558687448501587,
      "logps/rejected": -2.1748738288879395,
      "loss": 1.6557,
      "nll_loss": 1.6148791313171387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1558687388896942,
      "rewards/margins": 0.061618633568286896,
      "rewards/rejected": -0.2174873799085617,
      "step": 398
    },
    {
      "epoch": 0.6288416075650118,
      "grad_norm": 0.2060404270887375,
      "learning_rate": 3.4703696207274325e-06,
      "log_odds_chosen": 0.46504050493240356,
      "log_odds_ratio": -0.4943721890449524,
      "logits/chosen": -0.008130216039717197,
      "logits/rejected": -0.9074642658233643,
      "logps/chosen": -1.6367112398147583,
      "logps/rejected": -2.0245320796966553,
      "loss": 1.7255,
      "nll_loss": 1.6760835647583008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1636711210012436,
      "rewards/margins": 0.038782089948654175,
      "rewards/rejected": -0.20245321094989777,
      "step": 399
    },
    {
      "epoch": 0.6304176516942475,
      "grad_norm": 0.1868315488100052,
      "learning_rate": 3.4666377702799545e-06,
      "log_odds_chosen": 0.49212151765823364,
      "log_odds_ratio": -0.482626736164093,
      "logits/chosen": -0.12099798768758774,
      "logits/rejected": -1.2031362056732178,
      "logps/chosen": -1.5395071506500244,
      "logps/rejected": -1.9444780349731445,
      "loss": 1.6547,
      "nll_loss": 1.6063977479934692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15395070612430573,
      "rewards/margins": 0.040497105568647385,
      "rewards/rejected": -0.1944478154182434,
      "step": 400
    },
    {
      "epoch": 0.6319936958234831,
      "grad_norm": 0.20555970072746277,
      "learning_rate": 3.4628948400846417e-06,
      "log_odds_chosen": 0.6244911551475525,
      "log_odds_ratio": -0.44323962926864624,
      "logits/chosen": -0.041763000190258026,
      "logits/rejected": -1.1066770553588867,
      "logps/chosen": -1.6332682371139526,
      "logps/rejected": -2.165579319000244,
      "loss": 1.7357,
      "nll_loss": 1.6914199590682983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16332682967185974,
      "rewards/margins": 0.05323110893368721,
      "rewards/rejected": -0.21655793488025665,
      "step": 401
    },
    {
      "epoch": 0.6335697399527187,
      "grad_norm": 0.21198388934135437,
      "learning_rate": 3.4591408584175426e-06,
      "log_odds_chosen": 0.5584430694580078,
      "log_odds_ratio": -0.4683310091495514,
      "logits/chosen": -0.05661184713244438,
      "logits/rejected": -1.0141117572784424,
      "logps/chosen": -1.6664652824401855,
      "logps/rejected": -2.146893262863159,
      "loss": 1.7703,
      "nll_loss": 1.7234219312667847,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16664651036262512,
      "rewards/margins": 0.04804282262921333,
      "rewards/rejected": -0.21468935906887054,
      "step": 402
    },
    {
      "epoch": 0.6351457840819543,
      "grad_norm": 0.20266425609588623,
      "learning_rate": 3.4553758536381974e-06,
      "log_odds_chosen": 0.4527875483036041,
      "log_odds_ratio": -0.5067857503890991,
      "logits/chosen": -0.035650648176670074,
      "logits/rejected": -1.0273325443267822,
      "logps/chosen": -1.6237739324569702,
      "logps/rejected": -2.0050816535949707,
      "loss": 1.7259,
      "nll_loss": 1.675266146659851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16237738728523254,
      "rewards/margins": 0.03813077509403229,
      "rewards/rejected": -0.20050817728042603,
      "step": 403
    },
    {
      "epoch": 0.6367218282111899,
      "grad_norm": 0.21295951306819916,
      "learning_rate": 3.451599854189418e-06,
      "log_odds_chosen": 0.4978407025337219,
      "log_odds_ratio": -0.4797380268573761,
      "logits/chosen": -0.005185263231396675,
      "logits/rejected": -0.7095089554786682,
      "logps/chosen": -1.624720573425293,
      "logps/rejected": -2.042205572128296,
      "loss": 1.7394,
      "nll_loss": 1.6913968324661255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16247206926345825,
      "rewards/margins": 0.04174850881099701,
      "rewards/rejected": -0.20422056317329407,
      "step": 404
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 0.20290222764015198,
      "learning_rate": 3.4478128885970765e-06,
      "log_odds_chosen": 0.6090503334999084,
      "log_odds_ratio": -0.4394116997718811,
      "logits/chosen": -0.048805318772792816,
      "logits/rejected": -0.9924325942993164,
      "logps/chosen": -1.6847188472747803,
      "logps/rejected": -2.205502510070801,
      "loss": 1.7831,
      "nll_loss": 1.7391594648361206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16847188770771027,
      "rewards/margins": 0.05207835137844086,
      "rewards/rejected": -0.22055023908615112,
      "step": 405
    },
    {
      "epoch": 0.6398739164696612,
      "grad_norm": 0.19735999405384064,
      "learning_rate": 3.44401498546989e-06,
      "log_odds_chosen": 0.40471282601356506,
      "log_odds_ratio": -0.5204766392707825,
      "logits/chosen": -0.00239025941118598,
      "logits/rejected": -1.1112685203552246,
      "logps/chosen": -1.63077974319458,
      "logps/rejected": -1.9698301553726196,
      "loss": 1.7163,
      "nll_loss": 1.664223551750183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16307799518108368,
      "rewards/margins": 0.033905029296875,
      "rewards/rejected": -0.19698300957679749,
      "step": 406
    },
    {
      "epoch": 0.6414499605988968,
      "grad_norm": 0.20032718777656555,
      "learning_rate": 3.4402061734992005e-06,
      "log_odds_chosen": 0.4921063780784607,
      "log_odds_ratio": -0.48325401544570923,
      "logits/chosen": -0.03169507533311844,
      "logits/rejected": -1.0495617389678955,
      "logps/chosen": -1.5727291107177734,
      "logps/rejected": -1.982052206993103,
      "loss": 1.6824,
      "nll_loss": 1.634063482284546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15727292001247406,
      "rewards/margins": 0.040932293981313705,
      "rewards/rejected": -0.19820521771907806,
      "step": 407
    },
    {
      "epoch": 0.6430260047281324,
      "grad_norm": 0.21005932986736298,
      "learning_rate": 3.4363864814587656e-06,
      "log_odds_chosen": 0.45053696632385254,
      "log_odds_ratio": -0.49728235602378845,
      "logits/chosen": -0.14125539362430573,
      "logits/rejected": -0.6607908010482788,
      "logps/chosen": -1.55253005027771,
      "logps/rejected": -1.9263464212417603,
      "loss": 1.6646,
      "nll_loss": 1.6149158477783203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15525302290916443,
      "rewards/margins": 0.037381626665592194,
      "rewards/rejected": -0.19263465702533722,
      "step": 408
    },
    {
      "epoch": 0.644602048857368,
      "grad_norm": 0.21455970406532288,
      "learning_rate": 3.4325559382045343e-06,
      "log_odds_chosen": 0.33162471652030945,
      "log_odds_ratio": -0.5455266833305359,
      "logits/chosen": -0.10501392185688019,
      "logits/rejected": -1.1451892852783203,
      "logps/chosen": -1.567715048789978,
      "logps/rejected": -1.837162971496582,
      "loss": 1.6723,
      "nll_loss": 1.6177119016647339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1567714810371399,
      "rewards/margins": 0.026944806799292564,
      "rewards/rejected": -0.1837162971496582,
      "step": 409
    },
    {
      "epoch": 0.6461780929866037,
      "grad_norm": 0.20775310695171356,
      "learning_rate": 3.4287145726744295e-06,
      "log_odds_chosen": 0.5500257015228271,
      "log_odds_ratio": -0.46216368675231934,
      "logits/chosen": -0.13800697028636932,
      "logits/rejected": -0.9645453691482544,
      "logps/chosen": -1.550150990486145,
      "logps/rejected": -2.005794048309326,
      "loss": 1.6661,
      "nll_loss": 1.6199290752410889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15501509606838226,
      "rewards/margins": 0.04556430131196976,
      "rewards/rejected": -0.20057940483093262,
      "step": 410
    },
    {
      "epoch": 0.6477541371158393,
      "grad_norm": 0.2134082019329071,
      "learning_rate": 3.4248624138881335e-06,
      "log_odds_chosen": 0.40455663204193115,
      "log_odds_ratio": -0.5388070344924927,
      "logits/chosen": -0.026050057262182236,
      "logits/rejected": -0.811333417892456,
      "logps/chosen": -1.6882728338241577,
      "logps/rejected": -2.0449092388153076,
      "loss": 1.7892,
      "nll_loss": 1.7353262901306152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16882728040218353,
      "rewards/margins": 0.03566364943981171,
      "rewards/rejected": -0.20449092984199524,
      "step": 411
    },
    {
      "epoch": 0.6493301812450749,
      "grad_norm": 0.2133842557668686,
      "learning_rate": 3.4209994909468672e-06,
      "log_odds_chosen": 0.6076403260231018,
      "log_odds_ratio": -0.4480054974555969,
      "logits/chosen": -0.15131589770317078,
      "logits/rejected": -0.7187516093254089,
      "logps/chosen": -1.5922455787658691,
      "logps/rejected": -2.1052651405334473,
      "loss": 1.6941,
      "nll_loss": 1.649259090423584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15922458469867706,
      "rewards/margins": 0.05130196362733841,
      "rewards/rejected": -0.21052654087543488,
      "step": 412
    },
    {
      "epoch": 0.6509062253743105,
      "grad_norm": 0.20478829741477966,
      "learning_rate": 3.4171258330331667e-06,
      "log_odds_chosen": 0.39909109473228455,
      "log_odds_ratio": -0.5157694816589355,
      "logits/chosen": 0.006179895251989365,
      "logits/rejected": -0.8033927083015442,
      "logps/chosen": -1.6710724830627441,
      "logps/rejected": -2.0045435428619385,
      "loss": 1.7696,
      "nll_loss": 1.718004584312439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1671072542667389,
      "rewards/margins": 0.03334709256887436,
      "rewards/rejected": -0.20045433938503265,
      "step": 413
    },
    {
      "epoch": 0.6524822695035462,
      "grad_norm": 0.20214222371578217,
      "learning_rate": 3.4132414694106684e-06,
      "log_odds_chosen": 0.6251203417778015,
      "log_odds_ratio": -0.43651172518730164,
      "logits/chosen": -0.0376095250248909,
      "logits/rejected": -1.10747492313385,
      "logps/chosen": -1.5772483348846436,
      "logps/rejected": -2.1024982929229736,
      "loss": 1.6741,
      "nll_loss": 1.6304349899291992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15772482752799988,
      "rewards/margins": 0.05252499133348465,
      "rewards/rejected": -0.21024984121322632,
      "step": 414
    },
    {
      "epoch": 0.6540583136327817,
      "grad_norm": 0.2120778113603592,
      "learning_rate": 3.409346429423884e-06,
      "log_odds_chosen": 0.39810532331466675,
      "log_odds_ratio": -0.5165068507194519,
      "logits/chosen": 0.11685831099748611,
      "logits/rejected": -0.9671615362167358,
      "logps/chosen": -1.6512268781661987,
      "logps/rejected": -1.9838975667953491,
      "loss": 1.7451,
      "nll_loss": 1.693439245223999,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16512268781661987,
      "rewards/margins": 0.0332670658826828,
      "rewards/rejected": -0.19838975369930267,
      "step": 415
    },
    {
      "epoch": 0.6556343577620173,
      "grad_norm": 0.2049696147441864,
      "learning_rate": 3.40544074249798e-06,
      "log_odds_chosen": 0.6329183578491211,
      "log_odds_ratio": -0.43744218349456787,
      "logits/chosen": -0.06616753339767456,
      "logits/rejected": -1.1013216972351074,
      "logps/chosen": -1.6389892101287842,
      "logps/rejected": -2.1797854900360107,
      "loss": 1.7372,
      "nll_loss": 1.6934669017791748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16389892995357513,
      "rewards/margins": 0.05407961830496788,
      "rewards/rejected": -0.2179785519838333,
      "step": 416
    },
    {
      "epoch": 0.6572104018912529,
      "grad_norm": 0.22548918426036835,
      "learning_rate": 3.401524438138556e-06,
      "log_odds_chosen": 0.47032052278518677,
      "log_odds_ratio": -0.49687254428863525,
      "logits/chosen": 6.176391616463661e-05,
      "logits/rejected": -0.8082195520401001,
      "logps/chosen": -1.652639627456665,
      "logps/rejected": -2.0523738861083984,
      "loss": 1.7294,
      "nll_loss": 1.6797451972961426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16526396572589874,
      "rewards/margins": 0.039973437786102295,
      "rewards/rejected": -0.20523738861083984,
      "step": 417
    },
    {
      "epoch": 0.6587864460204885,
      "grad_norm": 0.20852544903755188,
      "learning_rate": 3.39759754593142e-06,
      "log_odds_chosen": 0.49835333228111267,
      "log_odds_ratio": -0.48722827434539795,
      "logits/chosen": -0.06864430010318756,
      "logits/rejected": -0.9766976833343506,
      "logps/chosen": -1.7005680799484253,
      "logps/rejected": -2.128114700317383,
      "loss": 1.7887,
      "nll_loss": 1.7399288415908813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1700568050146103,
      "rewards/margins": 0.04275466501712799,
      "rewards/rejected": -0.21281147003173828,
      "step": 418
    },
    {
      "epoch": 0.6603624901497241,
      "grad_norm": 0.21272076666355133,
      "learning_rate": 3.3936600955423683e-06,
      "log_odds_chosen": 0.6597129702568054,
      "log_odds_ratio": -0.42969611287117004,
      "logits/chosen": 0.03045894019305706,
      "logits/rejected": -0.764129638671875,
      "logps/chosen": -1.6032241582870483,
      "logps/rejected": -2.1603918075561523,
      "loss": 1.7045,
      "nll_loss": 1.6615636348724365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1603224277496338,
      "rewards/margins": 0.05571678280830383,
      "rewards/rejected": -0.21603919565677643,
      "step": 419
    },
    {
      "epoch": 0.6619385342789598,
      "grad_norm": 0.20755982398986816,
      "learning_rate": 3.3897121167169573e-06,
      "log_odds_chosen": 0.34852004051208496,
      "log_odds_ratio": -0.5454325079917908,
      "logits/chosen": -0.11937910318374634,
      "logits/rejected": -0.982914388179779,
      "logps/chosen": -1.5703413486480713,
      "logps/rejected": -1.858556866645813,
      "loss": 1.6827,
      "nll_loss": 1.62813138961792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15703415870666504,
      "rewards/margins": 0.028821531683206558,
      "rewards/rejected": -0.1858556866645813,
      "step": 420
    },
    {
      "epoch": 0.6635145784081954,
      "grad_norm": 0.20710468292236328,
      "learning_rate": 3.38575363928028e-06,
      "log_odds_chosen": 0.5697583556175232,
      "log_odds_ratio": -0.45452526211738586,
      "logits/chosen": -0.02085725963115692,
      "logits/rejected": -0.8392971754074097,
      "logps/chosen": -1.5740097761154175,
      "logps/rejected": -2.0498874187469482,
      "loss": 1.6588,
      "nll_loss": 1.6133930683135986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1574009507894516,
      "rewards/margins": 0.04758777469396591,
      "rewards/rejected": -0.2049887329339981,
      "step": 421
    },
    {
      "epoch": 0.665090622537431,
      "grad_norm": 0.2102571576833725,
      "learning_rate": 3.3817846931367452e-06,
      "log_odds_chosen": 0.41366758942604065,
      "log_odds_ratio": -0.5172683000564575,
      "logits/chosen": -0.10860089957714081,
      "logits/rejected": -0.8454552888870239,
      "logps/chosen": -1.565171718597412,
      "logps/rejected": -1.9058090448379517,
      "loss": 1.6679,
      "nll_loss": 1.6161460876464844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1565171778202057,
      "rewards/margins": 0.03406372666358948,
      "rewards/rejected": -0.19058090448379517,
      "step": 422
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 0.191580131649971,
      "learning_rate": 3.377805308269844e-06,
      "log_odds_chosen": 0.6439597606658936,
      "log_odds_ratio": -0.43700361251831055,
      "logits/chosen": -0.01071137934923172,
      "logits/rejected": -1.181455373764038,
      "logps/chosen": -1.608168601989746,
      "logps/rejected": -2.153388738632202,
      "loss": 1.6986,
      "nll_loss": 1.654897928237915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16081684827804565,
      "rewards/margins": 0.054522015154361725,
      "rewards/rejected": -0.21533887088298798,
      "step": 423
    },
    {
      "epoch": 0.6682427107959023,
      "grad_norm": 0.19327178597450256,
      "learning_rate": 3.3738155147419275e-06,
      "log_odds_chosen": 0.575452446937561,
      "log_odds_ratio": -0.45816850662231445,
      "logits/chosen": -0.13200272619724274,
      "logits/rejected": -0.9068926572799683,
      "logps/chosen": -1.6064414978027344,
      "logps/rejected": -2.092689275741577,
      "loss": 1.6817,
      "nll_loss": 1.6358880996704102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16064415872097015,
      "rewards/margins": 0.048624761402606964,
      "rewards/rejected": -0.20926889777183533,
      "step": 424
    },
    {
      "epoch": 0.6698187549251379,
      "grad_norm": 0.20792369544506073,
      "learning_rate": 3.3698153426939824e-06,
      "log_odds_chosen": 0.674706220626831,
      "log_odds_ratio": -0.4227129817008972,
      "logits/chosen": -0.11872132122516632,
      "logits/rejected": -0.758519172668457,
      "logps/chosen": -1.5436768531799316,
      "logps/rejected": -2.105792284011841,
      "loss": 1.6326,
      "nll_loss": 1.5903273820877075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15436768531799316,
      "rewards/margins": 0.05621154233813286,
      "rewards/rejected": -0.21057923138141632,
      "step": 425
    },
    {
      "epoch": 0.6713947990543735,
      "grad_norm": 0.2181132733821869,
      "learning_rate": 3.3658048223453954e-06,
      "log_odds_chosen": 0.6492812633514404,
      "log_odds_ratio": -0.43314385414123535,
      "logits/chosen": -0.07235290110111237,
      "logits/rejected": -0.9915630221366882,
      "logps/chosen": -1.6831142902374268,
      "logps/rejected": -2.2399418354034424,
      "loss": 1.7433,
      "nll_loss": 1.6999669075012207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16831141710281372,
      "rewards/margins": 0.05568274110555649,
      "rewards/rejected": -0.223994180560112,
      "step": 426
    },
    {
      "epoch": 0.6729708431836091,
      "grad_norm": 0.20445337891578674,
      "learning_rate": 3.3617839839937337e-06,
      "log_odds_chosen": 0.7020919919013977,
      "log_odds_ratio": -0.41846412420272827,
      "logits/chosen": -0.014502273872494698,
      "logits/rejected": -1.0373268127441406,
      "logps/chosen": -1.6425418853759766,
      "logps/rejected": -2.2469215393066406,
      "loss": 1.7212,
      "nll_loss": 1.6793993711471558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16425418853759766,
      "rewards/margins": 0.06043797731399536,
      "rewards/rejected": -0.22469216585159302,
      "step": 427
    },
    {
      "epoch": 0.6745468873128447,
      "grad_norm": 0.2129041850566864,
      "learning_rate": 3.3577528580145107e-06,
      "log_odds_chosen": 0.2998935580253601,
      "log_odds_ratio": -0.5577186346054077,
      "logits/chosen": -0.047993727028369904,
      "logits/rejected": -0.9543691277503967,
      "logps/chosen": -1.6607805490493774,
      "logps/rejected": -1.912225604057312,
      "loss": 1.7552,
      "nll_loss": 1.699418306350708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16607806086540222,
      "rewards/margins": 0.025144489482045174,
      "rewards/rejected": -0.19122257828712463,
      "step": 428
    },
    {
      "epoch": 0.6761229314420804,
      "grad_norm": 0.20701919496059418,
      "learning_rate": 3.353711474860956e-06,
      "log_odds_chosen": 0.513329029083252,
      "log_odds_ratio": -0.4874694049358368,
      "logits/chosen": -0.08273675292730331,
      "logits/rejected": -0.9507322907447815,
      "logps/chosen": -1.6140192747116089,
      "logps/rejected": -2.052968978881836,
      "loss": 1.6898,
      "nll_loss": 1.6410080194473267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16140194237232208,
      "rewards/margins": 0.043894968926906586,
      "rewards/rejected": -0.20529691874980927,
      "step": 429
    },
    {
      "epoch": 0.677698975571316,
      "grad_norm": 0.19609764218330383,
      "learning_rate": 3.3496598650637916e-06,
      "log_odds_chosen": 0.526725172996521,
      "log_odds_ratio": -0.47201910614967346,
      "logits/chosen": -0.09990064054727554,
      "logits/rejected": -0.8700954914093018,
      "logps/chosen": -1.5076314210891724,
      "logps/rejected": -1.9389755725860596,
      "loss": 1.6043,
      "nll_loss": 1.55709707736969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.150763139128685,
      "rewards/margins": 0.0431344136595726,
      "rewards/rejected": -0.1938975602388382,
      "step": 430
    },
    {
      "epoch": 0.6792750197005516,
      "grad_norm": 0.1991858035326004,
      "learning_rate": 3.3455980592309923e-06,
      "log_odds_chosen": 0.5936864614486694,
      "log_odds_ratio": -0.4512265920639038,
      "logits/chosen": -0.1363341212272644,
      "logits/rejected": -0.8848008513450623,
      "logps/chosen": -1.489920735359192,
      "logps/rejected": -1.9833248853683472,
      "loss": 1.5942,
      "nll_loss": 1.5491033792495728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14899206161499023,
      "rewards/margins": 0.04934043064713478,
      "rewards/rejected": -0.1983325034379959,
      "step": 431
    },
    {
      "epoch": 0.6808510638297872,
      "grad_norm": 0.20983241498470306,
      "learning_rate": 3.341526088047562e-06,
      "log_odds_chosen": 0.5074108839035034,
      "log_odds_ratio": -0.48232734203338623,
      "logits/chosen": -0.06561825424432755,
      "logits/rejected": -0.9465508460998535,
      "logps/chosen": -1.6827867031097412,
      "logps/rejected": -2.1165406703948975,
      "loss": 1.7561,
      "nll_loss": 1.7078619003295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16827866435050964,
      "rewards/margins": 0.043375398963689804,
      "rewards/rejected": -0.21165406703948975,
      "step": 432
    },
    {
      "epoch": 0.6824271079590228,
      "grad_norm": 0.22068631649017334,
      "learning_rate": 3.3374439822752972e-06,
      "log_odds_chosen": 0.3745832145214081,
      "log_odds_ratio": -0.5286702513694763,
      "logits/chosen": -0.008446307852864265,
      "logits/rejected": -0.816412627696991,
      "logps/chosen": -1.6505870819091797,
      "logps/rejected": -1.96774423122406,
      "loss": 1.7536,
      "nll_loss": 1.7007298469543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1650587022304535,
      "rewards/margins": 0.03171573579311371,
      "rewards/rejected": -0.196774423122406,
      "step": 433
    },
    {
      "epoch": 0.6840031520882585,
      "grad_norm": 0.21903684735298157,
      "learning_rate": 3.333351772752559e-06,
      "log_odds_chosen": 0.5724027752876282,
      "log_odds_ratio": -0.4620168209075928,
      "logits/chosen": -0.1362486630678177,
      "logits/rejected": -0.9352101683616638,
      "logps/chosen": -1.7139655351638794,
      "logps/rejected": -2.2006916999816895,
      "loss": 1.7887,
      "nll_loss": 1.7425075769424438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17139655351638794,
      "rewards/margins": 0.04867260903120041,
      "rewards/rejected": -0.22006915509700775,
      "step": 434
    },
    {
      "epoch": 0.6855791962174941,
      "grad_norm": 0.2128954529762268,
      "learning_rate": 3.3292494903940338e-06,
      "log_odds_chosen": 0.6801972985267639,
      "log_odds_ratio": -0.4192766547203064,
      "logits/chosen": -0.032803162932395935,
      "logits/rejected": -0.91535884141922,
      "logps/chosen": -1.6139404773712158,
      "logps/rejected": -2.1860525608062744,
      "loss": 1.7009,
      "nll_loss": 1.6589348316192627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16139404475688934,
      "rewards/margins": 0.057211197912693024,
      "rewards/rejected": -0.21860525012016296,
      "step": 435
    },
    {
      "epoch": 0.6871552403467297,
      "grad_norm": 0.2002972662448883,
      "learning_rate": 3.3251371661905063e-06,
      "log_odds_chosen": 0.5376681685447693,
      "log_odds_ratio": -0.4723852574825287,
      "logits/chosen": -0.08951784670352936,
      "logits/rejected": -0.8478531837463379,
      "logps/chosen": -1.4331471920013428,
      "logps/rejected": -1.8777246475219727,
      "loss": 1.5344,
      "nll_loss": 1.487158179283142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14331473410129547,
      "rewards/margins": 0.04445774853229523,
      "rewards/rejected": -0.1877724677324295,
      "step": 436
    },
    {
      "epoch": 0.6887312844759653,
      "grad_norm": 0.20821936428546906,
      "learning_rate": 3.321014831208622e-06,
      "log_odds_chosen": 0.5342722535133362,
      "log_odds_ratio": -0.46678629517555237,
      "logits/chosen": -0.08730736374855042,
      "logits/rejected": -0.8291983604431152,
      "logps/chosen": -1.500610589981079,
      "logps/rejected": -1.9357343912124634,
      "loss": 1.6101,
      "nll_loss": 1.5633900165557861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15006107091903687,
      "rewards/margins": 0.04351237416267395,
      "rewards/rejected": -0.19357344508171082,
      "step": 437
    },
    {
      "epoch": 0.6903073286052009,
      "grad_norm": 0.20071037113666534,
      "learning_rate": 3.316882516590652e-06,
      "log_odds_chosen": 0.5401603579521179,
      "log_odds_ratio": -0.4615623354911804,
      "logits/chosen": -0.07496978342533112,
      "logits/rejected": -0.99763023853302,
      "logps/chosen": -1.5472073554992676,
      "logps/rejected": -1.99533212184906,
      "loss": 1.6436,
      "nll_loss": 1.5974241495132446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1547207534313202,
      "rewards/margins": 0.044812463223934174,
      "rewards/rejected": -0.19953319430351257,
      "step": 438
    },
    {
      "epoch": 0.6918833727344366,
      "grad_norm": 0.22594793140888214,
      "learning_rate": 3.31274025355426e-06,
      "log_odds_chosen": 0.5037475228309631,
      "log_odds_ratio": -0.4795573353767395,
      "logits/chosen": -0.10893750190734863,
      "logits/rejected": -0.843508780002594,
      "logps/chosen": -1.6426455974578857,
      "logps/rejected": -2.06754207611084,
      "loss": 1.7082,
      "nll_loss": 1.6602920293807983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16426457464694977,
      "rewards/margins": 0.042489632964134216,
      "rewards/rejected": -0.20675420761108398,
      "step": 439
    },
    {
      "epoch": 0.6934594168636722,
      "grad_norm": 0.21982692182064056,
      "learning_rate": 3.308588073392265e-06,
      "log_odds_chosen": 0.5123088359832764,
      "log_odds_ratio": -0.48421573638916016,
      "logits/chosen": -0.10025706887245178,
      "logits/rejected": -0.8698948621749878,
      "logps/chosen": -1.6772215366363525,
      "logps/rejected": -2.1163253784179688,
      "loss": 1.7454,
      "nll_loss": 1.6969974040985107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16772215068340302,
      "rewards/margins": 0.043910376727581024,
      "rewards/rejected": -0.21163253486156464,
      "step": 440
    },
    {
      "epoch": 0.6950354609929078,
      "grad_norm": 0.20160053670406342,
      "learning_rate": 3.3044260074724035e-06,
      "log_odds_chosen": 0.5939540863037109,
      "log_odds_ratio": -0.4523179233074188,
      "logits/chosen": -0.11239210516214371,
      "logits/rejected": -1.0533446073532104,
      "logps/chosen": -1.577032208442688,
      "logps/rejected": -2.0725245475769043,
      "loss": 1.667,
      "nll_loss": 1.6217741966247559,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1577032506465912,
      "rewards/margins": 0.04954922944307327,
      "rewards/rejected": -0.20725245773792267,
      "step": 441
    },
    {
      "epoch": 0.6966115051221434,
      "grad_norm": 0.20258580148220062,
      "learning_rate": 3.300254087237097e-06,
      "log_odds_chosen": 0.49938884377479553,
      "log_odds_ratio": -0.4795615077018738,
      "logits/chosen": -0.07772234082221985,
      "logits/rejected": -0.969111442565918,
      "logps/chosen": -1.4605151414871216,
      "logps/rejected": -1.867677927017212,
      "loss": 1.5803,
      "nll_loss": 1.5323054790496826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460515260696411,
      "rewards/margins": 0.04071628302335739,
      "rewards/rejected": -0.1867678016424179,
      "step": 442
    },
    {
      "epoch": 0.698187549251379,
      "grad_norm": 0.20618461072444916,
      "learning_rate": 3.2960723442032105e-06,
      "log_odds_chosen": 0.6804747581481934,
      "log_odds_ratio": -0.4170026481151581,
      "logits/chosen": -0.14151881635189056,
      "logits/rejected": -1.196169376373291,
      "logps/chosen": -1.669822335243225,
      "logps/rejected": -2.254673957824707,
      "loss": 1.7346,
      "nll_loss": 1.6929285526275635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1669822335243225,
      "rewards/margins": 0.05848517641425133,
      "rewards/rejected": -0.22546738386154175,
      "step": 443
    },
    {
      "epoch": 0.6997635933806147,
      "grad_norm": 0.198680579662323,
      "learning_rate": 3.291880809961814e-06,
      "log_odds_chosen": 0.6177921295166016,
      "log_odds_ratio": -0.44483864307403564,
      "logits/chosen": -0.06093186140060425,
      "logits/rejected": -0.8600667715072632,
      "logps/chosen": -1.6328524351119995,
      "logps/rejected": -2.154387950897217,
      "loss": 1.7133,
      "nll_loss": 1.6687994003295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16328522562980652,
      "rewards/margins": 0.052153587341308594,
      "rewards/rejected": -0.2154388129711151,
      "step": 444
    },
    {
      "epoch": 0.7013396375098503,
      "grad_norm": 0.21375496685504913,
      "learning_rate": 3.2876795161779473e-06,
      "log_odds_chosen": 0.6847823858261108,
      "log_odds_ratio": -0.4106296896934509,
      "logits/chosen": -0.07752035558223724,
      "logits/rejected": -1.2114906311035156,
      "logps/chosen": -1.6328837871551514,
      "logps/rejected": -2.21699857711792,
      "loss": 1.6964,
      "nll_loss": 1.655341625213623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16328836977481842,
      "rewards/margins": 0.058411478996276855,
      "rewards/rejected": -0.22169984877109528,
      "step": 445
    },
    {
      "epoch": 0.7029156816390859,
      "grad_norm": 0.21601800620555878,
      "learning_rate": 3.2834684945903776e-06,
      "log_odds_chosen": 0.5214787125587463,
      "log_odds_ratio": -0.47087210416793823,
      "logits/chosen": -0.14094819128513336,
      "logits/rejected": -0.9257452487945557,
      "logps/chosen": -1.5604510307312012,
      "logps/rejected": -1.9898643493652344,
      "loss": 1.6529,
      "nll_loss": 1.6058588027954102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1560451090335846,
      "rewards/margins": 0.042941320687532425,
      "rewards/rejected": -0.19898641109466553,
      "step": 446
    },
    {
      "epoch": 0.7044917257683215,
      "grad_norm": 0.21135088801383972,
      "learning_rate": 3.2792477770113624e-06,
      "log_odds_chosen": 0.4751344323158264,
      "log_odds_ratio": -0.489793986082077,
      "logits/chosen": -0.21940332651138306,
      "logits/rejected": -1.019394874572754,
      "logps/chosen": -1.6616731882095337,
      "logps/rejected": -2.0628890991210938,
      "loss": 1.7293,
      "nll_loss": 1.680342435836792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16616733372211456,
      "rewards/margins": 0.04012158513069153,
      "rewards/rejected": -0.2062889188528061,
      "step": 447
    },
    {
      "epoch": 0.7060677698975572,
      "grad_norm": 0.2001531720161438,
      "learning_rate": 3.275017395326407e-06,
      "log_odds_chosen": 0.41810929775238037,
      "log_odds_ratio": -0.513521671295166,
      "logits/chosen": -0.03719010949134827,
      "logits/rejected": -0.803317666053772,
      "logps/chosen": -1.5478110313415527,
      "logps/rejected": -1.887550950050354,
      "loss": 1.644,
      "nll_loss": 1.5926766395568848,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15478110313415527,
      "rewards/margins": 0.033973999321460724,
      "rewards/rejected": -0.1887550950050354,
      "step": 448
    },
    {
      "epoch": 0.7076438140267928,
      "grad_norm": 0.2054382860660553,
      "learning_rate": 3.2707773814940244e-06,
      "log_odds_chosen": 0.5953386425971985,
      "log_odds_ratio": -0.44647330045700073,
      "logits/chosen": -0.07013065367937088,
      "logits/rejected": -1.0036925077438354,
      "logps/chosen": -1.5742981433868408,
      "logps/rejected": -2.065192222595215,
      "loss": 1.6672,
      "nll_loss": 1.6225340366363525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15742981433868408,
      "rewards/margins": 0.04908942058682442,
      "rewards/rejected": -0.2065192461013794,
      "step": 449
    },
    {
      "epoch": 0.7092198581560284,
      "grad_norm": 0.19840051233768463,
      "learning_rate": 3.2665277675454935e-06,
      "log_odds_chosen": 0.648217499256134,
      "log_odds_ratio": -0.4395694434642792,
      "logits/chosen": -0.13534650206565857,
      "logits/rejected": -1.220763087272644,
      "logps/chosen": -1.650367259979248,
      "logps/rejected": -2.209742546081543,
      "loss": 1.7142,
      "nll_loss": 1.6702194213867188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16503672301769257,
      "rewards/margins": 0.0559375137090683,
      "rewards/rejected": -0.22097423672676086,
      "step": 450
    },
    {
      "epoch": 0.710795902285264,
      "grad_norm": 0.1972638964653015,
      "learning_rate": 3.262268585584619e-06,
      "log_odds_chosen": 0.5737844705581665,
      "log_odds_ratio": -0.4500337839126587,
      "logits/chosen": -0.041236281394958496,
      "logits/rejected": -0.917519211769104,
      "logps/chosen": -1.5991514921188354,
      "logps/rejected": -2.0825254917144775,
      "loss": 1.6783,
      "nll_loss": 1.6332542896270752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15991514921188354,
      "rewards/margins": 0.048337388783693314,
      "rewards/rejected": -0.20825253427028656,
      "step": 451
    },
    {
      "epoch": 0.7123719464144996,
      "grad_norm": 0.21535180509090424,
      "learning_rate": 3.2579998677874853e-06,
      "log_odds_chosen": 0.6206746101379395,
      "log_odds_ratio": -0.44026538729667664,
      "logits/chosen": -0.12528139352798462,
      "logits/rejected": -0.9106650352478027,
      "logps/chosen": -1.6482163667678833,
      "logps/rejected": -2.1782073974609375,
      "loss": 1.7116,
      "nll_loss": 1.6675549745559692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16482165455818176,
      "rewards/margins": 0.05299907922744751,
      "rewards/rejected": -0.21782071888446808,
      "step": 452
    },
    {
      "epoch": 0.7139479905437353,
      "grad_norm": 0.1972205489873886,
      "learning_rate": 3.2537216464022155e-06,
      "log_odds_chosen": 0.7232244610786438,
      "log_odds_ratio": -0.4066394865512848,
      "logits/chosen": -0.11965236067771912,
      "logits/rejected": -0.8536319136619568,
      "logps/chosen": -1.64108145236969,
      "logps/rejected": -2.2579519748687744,
      "loss": 1.7132,
      "nll_loss": 1.6725622415542603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16410814225673676,
      "rewards/margins": 0.06168707460165024,
      "rewards/rejected": -0.2257952094078064,
      "step": 453
    },
    {
      "epoch": 0.7155240346729709,
      "grad_norm": 0.19788628816604614,
      "learning_rate": 3.2494339537487314e-06,
      "log_odds_chosen": 0.48940759897232056,
      "log_odds_ratio": -0.47984084486961365,
      "logits/chosen": -0.13651666045188904,
      "logits/rejected": -1.131873607635498,
      "logps/chosen": -1.6382532119750977,
      "logps/rejected": -2.0482289791107178,
      "loss": 1.7249,
      "nll_loss": 1.6769294738769531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16382533311843872,
      "rewards/margins": 0.04099758341908455,
      "rewards/rejected": -0.20482289791107178,
      "step": 454
    },
    {
      "epoch": 0.7171000788022065,
      "grad_norm": 0.1971239149570465,
      "learning_rate": 3.2451368222185006e-06,
      "log_odds_chosen": 0.3923978805541992,
      "log_odds_ratio": -0.5212135314941406,
      "logits/chosen": -0.12293338775634766,
      "logits/rejected": -0.8603593111038208,
      "logps/chosen": -1.5797085762023926,
      "logps/rejected": -1.901029348373413,
      "loss": 1.6756,
      "nll_loss": 1.6234517097473145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1579708606004715,
      "rewards/margins": 0.032132066786289215,
      "rewards/rejected": -0.1901029348373413,
      "step": 455
    },
    {
      "epoch": 0.7186761229314421,
      "grad_norm": 0.18168888986110687,
      "learning_rate": 3.2408302842743007e-06,
      "log_odds_chosen": 0.624774694442749,
      "log_odds_ratio": -0.4350818395614624,
      "logits/chosen": -0.08135272562503815,
      "logits/rejected": -0.9583423733711243,
      "logps/chosen": -1.5263901948928833,
      "logps/rejected": -2.04479718208313,
      "loss": 1.6239,
      "nll_loss": 1.580367922782898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526390165090561,
      "rewards/margins": 0.05184072256088257,
      "rewards/rejected": -0.20447973906993866,
      "step": 456
    },
    {
      "epoch": 0.7202521670606777,
      "grad_norm": 0.19839978218078613,
      "learning_rate": 3.2365143724499684e-06,
      "log_odds_chosen": 0.5523067712783813,
      "log_odds_ratio": -0.45656055212020874,
      "logits/chosen": -0.09882719814777374,
      "logits/rejected": -0.9976221323013306,
      "logps/chosen": -1.5835844278335571,
      "logps/rejected": -2.0447030067443848,
      "loss": 1.6806,
      "nll_loss": 1.6349064111709595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15835845470428467,
      "rewards/margins": 0.04611185938119888,
      "rewards/rejected": -0.20447030663490295,
      "step": 457
    },
    {
      "epoch": 0.7218282111899134,
      "grad_norm": 0.2024889588356018,
      "learning_rate": 3.2321891193501564e-06,
      "log_odds_chosen": 0.595514714717865,
      "log_odds_ratio": -0.4551790952682495,
      "logits/chosen": -0.06068085506558418,
      "logits/rejected": -0.5954574942588806,
      "logps/chosen": -1.5871148109436035,
      "logps/rejected": -2.089128017425537,
      "loss": 1.6641,
      "nll_loss": 1.6186047792434692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1587114930152893,
      "rewards/margins": 0.05020132660865784,
      "rewards/rejected": -0.20891280472278595,
      "step": 458
    },
    {
      "epoch": 0.723404255319149,
      "grad_norm": 0.19239795207977295,
      "learning_rate": 3.2278545576500858e-06,
      "log_odds_chosen": 0.7714004516601562,
      "log_odds_ratio": -0.38966119289398193,
      "logits/chosen": 0.014270449988543987,
      "logits/rejected": -0.7276147603988647,
      "logps/chosen": -1.4306761026382446,
      "logps/rejected": -2.06968355178833,
      "loss": 1.5279,
      "nll_loss": 1.4889153242111206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430675983428955,
      "rewards/margins": 0.06390075385570526,
      "rewards/rejected": -0.20696833729743958,
      "step": 459
    },
    {
      "epoch": 0.7249802994483846,
      "grad_norm": 0.20809946954250336,
      "learning_rate": 3.223510720095299e-06,
      "log_odds_chosen": 0.7304480671882629,
      "log_odds_ratio": -0.40124380588531494,
      "logits/chosen": -0.1674218326807022,
      "logits/rejected": -1.0309466123580933,
      "logps/chosen": -1.6155967712402344,
      "logps/rejected": -2.2373650074005127,
      "loss": 1.6819,
      "nll_loss": 1.6417958736419678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16155968606472015,
      "rewards/margins": 0.06217682361602783,
      "rewards/rejected": -0.2237364947795868,
      "step": 460
    },
    {
      "epoch": 0.7265563435776202,
      "grad_norm": 0.19244162738323212,
      "learning_rate": 3.2191576395014158e-06,
      "log_odds_chosen": 0.6560048460960388,
      "log_odds_ratio": -0.42095711827278137,
      "logits/chosen": -0.10353583842515945,
      "logits/rejected": -0.9026417136192322,
      "logps/chosen": -1.5507631301879883,
      "logps/rejected": -2.0990896224975586,
      "loss": 1.6312,
      "nll_loss": 1.5891027450561523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15507632493972778,
      "rewards/margins": 0.054832637310028076,
      "rewards/rejected": -0.20990897715091705,
      "step": 461
    },
    {
      "epoch": 0.7281323877068558,
      "grad_norm": 0.196467325091362,
      "learning_rate": 3.2147953487538794e-06,
      "log_odds_chosen": 0.6500884294509888,
      "log_odds_ratio": -0.42691317200660706,
      "logits/chosen": -0.04400169104337692,
      "logits/rejected": -0.973447322845459,
      "logps/chosen": -1.519189476966858,
      "logps/rejected": -2.0596096515655518,
      "loss": 1.5998,
      "nll_loss": 1.5571023225784302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1519189476966858,
      "rewards/margins": 0.054042041301727295,
      "rewards/rejected": -0.20596098899841309,
      "step": 462
    },
    {
      "epoch": 0.7297084318360915,
      "grad_norm": 0.20894914865493774,
      "learning_rate": 3.2104238808077133e-06,
      "log_odds_chosen": 0.5251021981239319,
      "log_odds_ratio": -0.474967896938324,
      "logits/chosen": -0.07077591866254807,
      "logits/rejected": -0.7537460327148438,
      "logps/chosen": -1.5554322004318237,
      "logps/rejected": -1.992119550704956,
      "loss": 1.6398,
      "nll_loss": 1.5922644138336182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1555432230234146,
      "rewards/margins": 0.04366873577237129,
      "rewards/rejected": -0.1992119550704956,
      "step": 463
    },
    {
      "epoch": 0.731284475965327,
      "grad_norm": 0.19019024074077606,
      "learning_rate": 3.2060432686872704e-06,
      "log_odds_chosen": 0.8081085681915283,
      "log_odds_ratio": -0.3813464343547821,
      "logits/chosen": -0.13858087360858917,
      "logits/rejected": -0.7866955995559692,
      "logps/chosen": -1.4605531692504883,
      "logps/rejected": -2.129701852798462,
      "loss": 1.5568,
      "nll_loss": 1.518630027770996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14605532586574554,
      "rewards/margins": 0.0669148787856102,
      "rewards/rejected": -0.21297019720077515,
      "step": 464
    },
    {
      "epoch": 0.7328605200945626,
      "grad_norm": 0.19666843116283417,
      "learning_rate": 3.201653545485982e-06,
      "log_odds_chosen": 0.6164845824241638,
      "log_odds_ratio": -0.44083070755004883,
      "logits/chosen": -0.0490589365363121,
      "logits/rejected": -0.8681581616401672,
      "logps/chosen": -1.6057840585708618,
      "logps/rejected": -2.1277639865875244,
      "loss": 1.6728,
      "nll_loss": 1.6286746263504028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1605783998966217,
      "rewards/margins": 0.05219800025224686,
      "rewards/rejected": -0.21277640759944916,
      "step": 465
    },
    {
      "epoch": 0.7344365642237982,
      "grad_norm": 0.2226024568080902,
      "learning_rate": 3.197254744366111e-06,
      "log_odds_chosen": 0.6797471642494202,
      "log_odds_ratio": -0.418096125125885,
      "logits/chosen": -0.03286769241094589,
      "logits/rejected": -0.8219181299209595,
      "logps/chosen": -1.515041708946228,
      "logps/rejected": -2.079200267791748,
      "loss": 1.6003,
      "nll_loss": 1.5584660768508911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15150417387485504,
      "rewards/margins": 0.0564158670604229,
      "rewards/rejected": -0.20792002975940704,
      "step": 466
    },
    {
      "epoch": 0.7360126083530338,
      "grad_norm": 0.2114436775445938,
      "learning_rate": 3.192846898558498e-06,
      "log_odds_chosen": 0.5047996044158936,
      "log_odds_ratio": -0.47521814703941345,
      "logits/chosen": -0.0795680359005928,
      "logits/rejected": -0.892188310623169,
      "logps/chosen": -1.6534972190856934,
      "logps/rejected": -2.0805842876434326,
      "loss": 1.7331,
      "nll_loss": 1.6855862140655518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16534970700740814,
      "rewards/margins": 0.042708709836006165,
      "rewards/rejected": -0.2080584317445755,
      "step": 467
    },
    {
      "epoch": 0.7375886524822695,
      "grad_norm": 0.20920921862125397,
      "learning_rate": 3.188430041362313e-06,
      "log_odds_chosen": 0.4469729959964752,
      "log_odds_ratio": -0.5012507438659668,
      "logits/chosen": -0.03374030441045761,
      "logits/rejected": -0.9043279886245728,
      "logps/chosen": -1.645135521888733,
      "logps/rejected": -2.022472381591797,
      "loss": 1.7118,
      "nll_loss": 1.6616994142532349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16451355814933777,
      "rewards/margins": 0.03773367404937744,
      "rewards/rejected": -0.2022472321987152,
      "step": 468
    },
    {
      "epoch": 0.7391646966115051,
      "grad_norm": 0.2058577537536621,
      "learning_rate": 3.184004206144803e-06,
      "log_odds_chosen": 0.6731261014938354,
      "log_odds_ratio": -0.420916885137558,
      "logits/chosen": -0.14038869738578796,
      "logits/rejected": -0.8868641257286072,
      "logps/chosen": -1.5474592447280884,
      "logps/rejected": -2.1062960624694824,
      "loss": 1.6267,
      "nll_loss": 1.5845807790756226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1547459214925766,
      "rewards/margins": 0.05588366463780403,
      "rewards/rejected": -0.21062961220741272,
      "step": 469
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 0.19909264147281647,
      "learning_rate": 3.1795694263410386e-06,
      "log_odds_chosen": 0.7366645932197571,
      "log_odds_ratio": -0.4054761230945587,
      "logits/chosen": -0.11322111636400223,
      "logits/rejected": -0.9165863990783691,
      "logps/chosen": -1.52811861038208,
      "logps/rejected": -2.152003049850464,
      "loss": 1.6016,
      "nll_loss": 1.5610570907592773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15281186997890472,
      "rewards/margins": 0.062388449907302856,
      "rewards/rejected": -0.21520031988620758,
      "step": 470
    },
    {
      "epoch": 0.7423167848699763,
      "grad_norm": 0.1952643245458603,
      "learning_rate": 3.1751257354536634e-06,
      "log_odds_chosen": 0.520018458366394,
      "log_odds_ratio": -0.48046696186065674,
      "logits/chosen": -0.07400179654359818,
      "logits/rejected": -1.0409823656082153,
      "logps/chosen": -1.5231245756149292,
      "logps/rejected": -1.958061695098877,
      "loss": 1.6132,
      "nll_loss": 1.5651495456695557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15231245756149292,
      "rewards/margins": 0.043493710458278656,
      "rewards/rejected": -0.19580616056919098,
      "step": 471
    },
    {
      "epoch": 0.7438928289992119,
      "grad_norm": 0.20432332158088684,
      "learning_rate": 3.1706731670526394e-06,
      "log_odds_chosen": 0.5711886882781982,
      "log_odds_ratio": -0.4503237009048462,
      "logits/chosen": -0.1537346988916397,
      "logits/rejected": -0.8866163492202759,
      "logps/chosen": -1.5807658433914185,
      "logps/rejected": -2.0559804439544678,
      "loss": 1.6442,
      "nll_loss": 1.5991592407226562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15807659924030304,
      "rewards/margins": 0.04752147197723389,
      "rewards/rejected": -0.20559807121753693,
      "step": 472
    },
    {
      "epoch": 0.7454688731284476,
      "grad_norm": 0.19950729608535767,
      "learning_rate": 3.166211754774994e-06,
      "log_odds_chosen": 0.6052391529083252,
      "log_odds_ratio": -0.4445112943649292,
      "logits/chosen": -0.1134694516658783,
      "logits/rejected": -1.0386989116668701,
      "logps/chosen": -1.629151701927185,
      "logps/rejected": -2.1410460472106934,
      "loss": 1.7275,
      "nll_loss": 1.683068871498108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1629151701927185,
      "rewards/margins": 0.05118943005800247,
      "rewards/rejected": -0.21410460770130157,
      "step": 473
    },
    {
      "epoch": 0.7470449172576832,
      "grad_norm": 0.20564451813697815,
      "learning_rate": 3.1617415323245665e-06,
      "log_odds_chosen": 0.5903938412666321,
      "log_odds_ratio": -0.45206212997436523,
      "logits/chosen": -0.14191381633281708,
      "logits/rejected": -0.9877166748046875,
      "logps/chosen": -1.5389567613601685,
      "logps/rejected": -2.0210845470428467,
      "loss": 1.6103,
      "nll_loss": 1.5650646686553955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15389567613601685,
      "rewards/margins": 0.04821278899908066,
      "rewards/rejected": -0.2021084576845169,
      "step": 474
    },
    {
      "epoch": 0.7486209613869188,
      "grad_norm": 0.19367830455303192,
      "learning_rate": 3.157262533471752e-06,
      "log_odds_chosen": 0.6683068871498108,
      "log_odds_ratio": -0.41824913024902344,
      "logits/chosen": -0.07336349785327911,
      "logits/rejected": -1.0358256101608276,
      "logps/chosen": -1.5820767879486084,
      "logps/rejected": -2.1471550464630127,
      "loss": 1.6612,
      "nll_loss": 1.6193286180496216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15820768475532532,
      "rewards/margins": 0.05650782957673073,
      "rewards/rejected": -0.21471551060676575,
      "step": 475
    },
    {
      "epoch": 0.7501970055161544,
      "grad_norm": 0.20863281190395355,
      "learning_rate": 3.1527747920532468e-06,
      "log_odds_chosen": 0.5243900418281555,
      "log_odds_ratio": -0.4661727547645569,
      "logits/chosen": -0.08033692091703415,
      "logits/rejected": -0.7630578279495239,
      "logps/chosen": -1.5552059412002563,
      "logps/rejected": -1.9899144172668457,
      "loss": 1.6392,
      "nll_loss": 1.5925623178482056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15552058815956116,
      "rewards/margins": 0.04347085952758789,
      "rewards/rejected": -0.19899143278598785,
      "step": 476
    },
    {
      "epoch": 0.75177304964539,
      "grad_norm": 0.19890214502811432,
      "learning_rate": 3.148278341971795e-06,
      "log_odds_chosen": 0.6338871717453003,
      "log_odds_ratio": -0.4345587193965912,
      "logits/chosen": -0.13997706770896912,
      "logits/rejected": -0.8114413619041443,
      "logps/chosen": -1.5798550844192505,
      "logps/rejected": -2.1142609119415283,
      "loss": 1.6548,
      "nll_loss": 1.6113535165786743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15798552334308624,
      "rewards/margins": 0.053440578281879425,
      "rewards/rejected": -0.21142609417438507,
      "step": 477
    },
    {
      "epoch": 0.7533490937746257,
      "grad_norm": 0.2100202441215515,
      "learning_rate": 3.143773217195929e-06,
      "log_odds_chosen": 0.694851279258728,
      "log_odds_ratio": -0.4138457775115967,
      "logits/chosen": -0.13817080855369568,
      "logits/rejected": -0.9909247159957886,
      "logps/chosen": -1.6483888626098633,
      "logps/rejected": -2.2417492866516113,
      "loss": 1.7141,
      "nll_loss": 1.6727272272109985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16483889520168304,
      "rewards/margins": 0.059336043894290924,
      "rewards/rejected": -0.22417494654655457,
      "step": 478
    },
    {
      "epoch": 0.7549251379038613,
      "grad_norm": 0.19809815287590027,
      "learning_rate": 3.139259451759714e-06,
      "log_odds_chosen": 0.5344829559326172,
      "log_odds_ratio": -0.4648602604866028,
      "logits/chosen": -0.15662164986133575,
      "logits/rejected": -0.9446511268615723,
      "logps/chosen": -1.6435620784759521,
      "logps/rejected": -2.094280242919922,
      "loss": 1.7243,
      "nll_loss": 1.6778401136398315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16435623168945312,
      "rewards/margins": 0.045071791857481,
      "rewards/rejected": -0.20942802727222443,
      "step": 479
    },
    {
      "epoch": 0.7565011820330969,
      "grad_norm": 0.1964648813009262,
      "learning_rate": 3.134737079762493e-06,
      "log_odds_chosen": 0.49474141001701355,
      "log_odds_ratio": -0.48102235794067383,
      "logits/chosen": -0.09573915600776672,
      "logits/rejected": -0.9279856085777283,
      "logps/chosen": -1.5789200067520142,
      "logps/rejected": -1.9883246421813965,
      "loss": 1.6553,
      "nll_loss": 1.6071677207946777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15789198875427246,
      "rewards/margins": 0.04094048589468002,
      "rewards/rejected": -0.1988324671983719,
      "step": 480
    },
    {
      "epoch": 0.7580772261623325,
      "grad_norm": 0.20796369016170502,
      "learning_rate": 3.130206135368626e-06,
      "log_odds_chosen": 0.5486950278282166,
      "log_odds_ratio": -0.46876269578933716,
      "logits/chosen": -0.14321769773960114,
      "logits/rejected": -0.7052893042564392,
      "logps/chosen": -1.4953614473342896,
      "logps/rejected": -1.9454658031463623,
      "loss": 1.5968,
      "nll_loss": 1.5499558448791504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495361477136612,
      "rewards/margins": 0.04501045495271683,
      "rewards/rejected": -0.19454661011695862,
      "step": 481
    },
    {
      "epoch": 0.7596532702915682,
      "grad_norm": 0.20492172241210938,
      "learning_rate": 3.1256666528072327e-06,
      "log_odds_chosen": 0.7209620475769043,
      "log_odds_ratio": -0.4015035629272461,
      "logits/chosen": -0.13152581453323364,
      "logits/rejected": -0.7092732787132263,
      "logps/chosen": -1.5489566326141357,
      "logps/rejected": -2.1531386375427246,
      "loss": 1.6228,
      "nll_loss": 1.5826079845428467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15489566326141357,
      "rewards/margins": 0.06041823327541351,
      "rewards/rejected": -0.2153138965368271,
      "step": 482
    },
    {
      "epoch": 0.7612293144208038,
      "grad_norm": 0.1936446577310562,
      "learning_rate": 3.121118666371937e-06,
      "log_odds_chosen": 0.5240681171417236,
      "log_odds_ratio": -0.47471654415130615,
      "logits/chosen": -0.07757510244846344,
      "logits/rejected": -1.1584354639053345,
      "logps/chosen": -1.6076598167419434,
      "logps/rejected": -2.050039291381836,
      "loss": 1.674,
      "nll_loss": 1.6264898777008057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16076599061489105,
      "rewards/margins": 0.044237952679395676,
      "rewards/rejected": -0.20500393211841583,
      "step": 483
    },
    {
      "epoch": 0.7628053585500394,
      "grad_norm": 0.2122434675693512,
      "learning_rate": 3.1165622104206034e-06,
      "log_odds_chosen": 0.7360461354255676,
      "log_odds_ratio": -0.40052106976509094,
      "logits/chosen": -0.27119144797325134,
      "logits/rejected": -0.8301371932029724,
      "logps/chosen": -1.541393518447876,
      "logps/rejected": -2.163299322128296,
      "loss": 1.618,
      "nll_loss": 1.5779178142547607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15413935482501984,
      "rewards/margins": 0.06219058483839035,
      "rewards/rejected": -0.2163299322128296,
      "step": 484
    },
    {
      "epoch": 0.764381402679275,
      "grad_norm": 0.19363641738891602,
      "learning_rate": 3.1119973193750816e-06,
      "log_odds_chosen": 0.620286226272583,
      "log_odds_ratio": -0.43595632910728455,
      "logits/chosen": -0.16278237104415894,
      "logits/rejected": -0.8815320730209351,
      "logps/chosen": -1.5484811067581177,
      "logps/rejected": -2.066248893737793,
      "loss": 1.6171,
      "nll_loss": 1.5735543966293335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.154848113656044,
      "rewards/margins": 0.05177678167819977,
      "rewards/rejected": -0.20662488043308258,
      "step": 485
    },
    {
      "epoch": 0.7659574468085106,
      "grad_norm": 0.20480599999427795,
      "learning_rate": 3.1074240277209408e-06,
      "log_odds_chosen": 0.5229519605636597,
      "log_odds_ratio": -0.4746035635471344,
      "logits/chosen": -0.13251370191574097,
      "logits/rejected": -0.8053449988365173,
      "logps/chosen": -1.5717124938964844,
      "logps/rejected": -2.003612756729126,
      "loss": 1.6288,
      "nll_loss": 1.581296443939209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15717123448848724,
      "rewards/margins": 0.043190039694309235,
      "rewards/rejected": -0.20036128163337708,
      "step": 486
    },
    {
      "epoch": 0.7675334909377463,
      "grad_norm": 0.22217483818531036,
      "learning_rate": 3.102842370007217e-06,
      "log_odds_chosen": 0.6027494668960571,
      "log_odds_ratio": -0.44312798976898193,
      "logits/chosen": -0.05970533564686775,
      "logits/rejected": -0.7939674258232117,
      "logps/chosen": -1.5866206884384155,
      "logps/rejected": -2.086024045944214,
      "loss": 1.6619,
      "nll_loss": 1.6176303625106812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1586620807647705,
      "rewards/margins": 0.0499403215944767,
      "rewards/rejected": -0.20860238373279572,
      "step": 487
    },
    {
      "epoch": 0.7691095350669819,
      "grad_norm": 0.2019231766462326,
      "learning_rate": 3.0982523808461454e-06,
      "log_odds_chosen": 0.5037473440170288,
      "log_odds_ratio": -0.47748374938964844,
      "logits/chosen": -0.07278791815042496,
      "logits/rejected": -0.8580007553100586,
      "logps/chosen": -1.6201282739639282,
      "logps/rejected": -2.038290023803711,
      "loss": 1.6866,
      "nll_loss": 1.6388630867004395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16201284527778625,
      "rewards/margins": 0.04181616008281708,
      "rewards/rejected": -0.20382899045944214,
      "step": 488
    },
    {
      "epoch": 0.7706855791962175,
      "grad_norm": 0.19960224628448486,
      "learning_rate": 3.0936540949129006e-06,
      "log_odds_chosen": 0.5877503752708435,
      "log_odds_ratio": -0.4546903073787689,
      "logits/chosen": -0.1274491399526596,
      "logits/rejected": -0.8250627517700195,
      "logps/chosen": -1.5944395065307617,
      "logps/rejected": -2.0901992321014404,
      "loss": 1.6698,
      "nll_loss": 1.6242817640304565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1594439595937729,
      "rewards/margins": 0.04957596957683563,
      "rewards/rejected": -0.20901991426944733,
      "step": 489
    },
    {
      "epoch": 0.7722616233254531,
      "grad_norm": 0.1848008632659912,
      "learning_rate": 3.0890475469453378e-06,
      "log_odds_chosen": 0.6991172432899475,
      "log_odds_ratio": -0.4125969707965851,
      "logits/chosen": -0.1196417286992073,
      "logits/rejected": -0.9863728284835815,
      "logps/chosen": -1.4715135097503662,
      "logps/rejected": -2.0471150875091553,
      "loss": 1.5488,
      "nll_loss": 1.507541537284851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14715135097503662,
      "rewards/margins": 0.05756012722849846,
      "rewards/rejected": -0.20471149682998657,
      "step": 490
    },
    {
      "epoch": 0.7738376674546887,
      "grad_norm": 0.19433538615703583,
      "learning_rate": 3.0844327717437263e-06,
      "log_odds_chosen": 0.6452741622924805,
      "log_odds_ratio": -0.4334539771080017,
      "logits/chosen": -0.1350182145833969,
      "logits/rejected": -0.9193748235702515,
      "logps/chosen": -1.4977014064788818,
      "logps/rejected": -2.036151885986328,
      "loss": 1.5787,
      "nll_loss": 1.5353686809539795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14977017045021057,
      "rewards/margins": 0.053845033049583435,
      "rewards/rejected": -0.203615203499794,
      "step": 491
    },
    {
      "epoch": 0.7754137115839244,
      "grad_norm": 0.19196495413780212,
      "learning_rate": 3.0798098041704892e-06,
      "log_odds_chosen": 0.44325220584869385,
      "log_odds_ratio": -0.5018943548202515,
      "logits/chosen": -0.08340089023113251,
      "logits/rejected": -0.9438948631286621,
      "logps/chosen": -1.4235002994537354,
      "logps/rejected": -1.7811520099639893,
      "loss": 1.5264,
      "nll_loss": 1.476258635520935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14235003292560577,
      "rewards/margins": 0.035765163600444794,
      "rewards/rejected": -0.17811518907546997,
      "step": 492
    },
    {
      "epoch": 0.77698975571316,
      "grad_norm": 0.1994369626045227,
      "learning_rate": 3.0751786791499368e-06,
      "log_odds_chosen": 0.5825287103652954,
      "log_odds_ratio": -0.45188358426094055,
      "logits/chosen": -0.10140110552310944,
      "logits/rejected": -0.8894228339195251,
      "logps/chosen": -1.6182955503463745,
      "logps/rejected": -2.1115620136260986,
      "loss": 1.6949,
      "nll_loss": 1.649696707725525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16182956099510193,
      "rewards/margins": 0.04932662099599838,
      "rewards/rejected": -0.2111561894416809,
      "step": 493
    },
    {
      "epoch": 0.7785657998423956,
      "grad_norm": 0.19131864607334137,
      "learning_rate": 3.070539431668008e-06,
      "log_odds_chosen": 0.5550589561462402,
      "log_odds_ratio": -0.4599633514881134,
      "logits/chosen": -0.09749419242143631,
      "logits/rejected": -0.8432252407073975,
      "logps/chosen": -1.6216144561767578,
      "logps/rejected": -2.092540979385376,
      "loss": 1.6915,
      "nll_loss": 1.645534634590149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1621614396572113,
      "rewards/margins": 0.04709266871213913,
      "rewards/rejected": -0.20925410091876984,
      "step": 494
    },
    {
      "epoch": 0.7801418439716312,
      "grad_norm": 0.20070280134677887,
      "learning_rate": 3.0658920967720018e-06,
      "log_odds_chosen": 0.8226516842842102,
      "log_odds_ratio": -0.3818792700767517,
      "logits/chosen": -0.25017157196998596,
      "logits/rejected": -0.8173326253890991,
      "logps/chosen": -1.558310866355896,
      "logps/rejected": -2.2607622146606445,
      "loss": 1.6328,
      "nll_loss": 1.5946537256240845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15583109855651855,
      "rewards/margins": 0.07024513185024261,
      "rewards/rejected": -0.22607621550559998,
      "step": 495
    },
    {
      "epoch": 0.7817178881008668,
      "grad_norm": 0.193608358502388,
      "learning_rate": 3.0612367095703116e-06,
      "log_odds_chosen": 0.6440973877906799,
      "log_odds_ratio": -0.42651355266571045,
      "logits/chosen": -0.10096383839845657,
      "logits/rejected": -1.0842796564102173,
      "logps/chosen": -1.5973050594329834,
      "logps/rejected": -2.1420528888702393,
      "loss": 1.6628,
      "nll_loss": 1.620169997215271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15973050892353058,
      "rewards/margins": 0.05447479337453842,
      "rewards/rejected": -0.2142052799463272,
      "step": 496
    },
    {
      "epoch": 0.7832939322301025,
      "grad_norm": 0.20013193786144257,
      "learning_rate": 3.056573305232167e-06,
      "log_odds_chosen": 0.7367360591888428,
      "log_odds_ratio": -0.39854782819747925,
      "logits/chosen": -0.12801437079906464,
      "logits/rejected": -0.9907239079475403,
      "logps/chosen": -1.5458834171295166,
      "logps/rejected": -2.1604068279266357,
      "loss": 1.6209,
      "nll_loss": 1.5810470581054688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15458835661411285,
      "rewards/margins": 0.061452336609363556,
      "rewards/rejected": -0.2160406857728958,
      "step": 497
    },
    {
      "epoch": 0.7848699763593381,
      "grad_norm": 0.19444431364536285,
      "learning_rate": 3.051901918987359e-06,
      "log_odds_chosen": 0.6834968328475952,
      "log_odds_ratio": -0.41310781240463257,
      "logits/chosen": -0.2773919105529785,
      "logits/rejected": -1.03933584690094,
      "logps/chosen": -1.4846152067184448,
      "logps/rejected": -2.0495989322662354,
      "loss": 1.5632,
      "nll_loss": 1.5218619108200073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14846152067184448,
      "rewards/margins": 0.05649835988879204,
      "rewards/rejected": -0.20495988428592682,
      "step": 498
    },
    {
      "epoch": 0.7864460204885737,
      "grad_norm": 0.22802606225013733,
      "learning_rate": 3.047222586125979e-06,
      "log_odds_chosen": 0.6992647647857666,
      "log_odds_ratio": -0.40760669112205505,
      "logits/chosen": -0.07471290230751038,
      "logits/rejected": -0.5809466242790222,
      "logps/chosen": -1.5056655406951904,
      "logps/rejected": -2.086801767349243,
      "loss": 1.5939,
      "nll_loss": 1.5531069040298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15056656301021576,
      "rewards/margins": 0.05811362713575363,
      "rewards/rejected": -0.2086801826953888,
      "step": 499
    },
    {
      "epoch": 0.7880220646178093,
      "grad_norm": 0.19122202694416046,
      "learning_rate": 3.042535341998152e-06,
      "log_odds_chosen": 0.4867512285709381,
      "log_odds_ratio": -0.485049307346344,
      "logits/chosen": 0.002201020484790206,
      "logits/rejected": -0.9413242936134338,
      "logps/chosen": -1.6487398147583008,
      "logps/rejected": -2.0607011318206787,
      "loss": 1.7045,
      "nll_loss": 1.6559950113296509,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16487398743629456,
      "rewards/margins": 0.04119614511728287,
      "rewards/rejected": -0.20607014000415802,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 1270,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}