{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.7880220646178093,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015760441292356187,
      "grad_norm": 0.8302866220474243,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.46107953786849976,
      "log_odds_ratio": -0.5080549120903015,
      "logits/chosen": -1.1763941049575806,
      "logits/rejected": -0.5066124200820923,
      "logps/chosen": -2.0480191707611084,
      "logps/rejected": -2.4605765342712402,
      "loss": 2.3016,
      "nll_loss": 2.2507846355438232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20480191707611084,
      "rewards/margins": 0.041255734860897064,
      "rewards/rejected": -0.2460576593875885,
      "step": 1
    },
    {
      "epoch": 0.0031520882584712374,
      "grad_norm": 0.7972496747970581,
      "learning_rate": 3.1496062992125985e-08,
      "log_odds_chosen": 0.44702666997909546,
      "log_odds_ratio": -0.5035712718963623,
      "logits/chosen": -1.1409798860549927,
      "logits/rejected": -0.378469854593277,
      "logps/chosen": -1.9499034881591797,
      "logps/rejected": -2.3424463272094727,
      "loss": 2.1951,
      "nll_loss": 2.1447677612304688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1949903815984726,
      "rewards/margins": 0.039254240691661835,
      "rewards/rejected": -0.23424461483955383,
      "step": 2
    },
    {
      "epoch": 0.004728132387706856,
      "grad_norm": 0.8205430507659912,
      "learning_rate": 6.299212598425197e-08,
      "log_odds_chosen": 0.643511176109314,
      "log_odds_ratio": -0.4351435601711273,
      "logits/chosen": -1.1954814195632935,
      "logits/rejected": -0.46929946541786194,
      "logps/chosen": -2.0658984184265137,
      "logps/rejected": -2.6481122970581055,
      "loss": 2.3057,
      "nll_loss": 2.2622337341308594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20658984780311584,
      "rewards/margins": 0.05822139233350754,
      "rewards/rejected": -0.264811247587204,
      "step": 3
    },
    {
      "epoch": 0.006304176516942475,
      "grad_norm": 0.7553535103797913,
      "learning_rate": 9.448818897637795e-08,
      "log_odds_chosen": 0.6655425429344177,
      "log_odds_ratio": -0.4300715923309326,
      "logits/chosen": -1.0765116214752197,
      "logits/rejected": -0.35491788387298584,
      "logps/chosen": -2.062553882598877,
      "logps/rejected": -2.668004274368286,
      "loss": 2.3004,
      "nll_loss": 2.2574236392974854,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20625539124011993,
      "rewards/margins": 0.06054502725601196,
      "rewards/rejected": -0.2668004333972931,
      "step": 4
    },
    {
      "epoch": 0.007880220646178092,
      "grad_norm": 0.8500381708145142,
      "learning_rate": 1.2598425196850394e-07,
      "log_odds_chosen": 0.8081256151199341,
      "log_odds_ratio": -0.3941689729690552,
      "logits/chosen": -1.2188713550567627,
      "logits/rejected": -0.5489533543586731,
      "logps/chosen": -1.9932475090026855,
      "logps/rejected": -2.7177062034606934,
      "loss": 2.2247,
      "nll_loss": 2.1853041648864746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19932474195957184,
      "rewards/margins": 0.07244586944580078,
      "rewards/rejected": -0.2717706263065338,
      "step": 5
    },
    {
      "epoch": 0.009456264775413711,
      "grad_norm": 0.7359945178031921,
      "learning_rate": 1.5748031496062992e-07,
      "log_odds_chosen": 0.4542209506034851,
      "log_odds_ratio": -0.49476009607315063,
      "logits/chosen": -1.026477336883545,
      "logits/rejected": -0.3336237370967865,
      "logps/chosen": -1.9348886013031006,
      "logps/rejected": -2.3354790210723877,
      "loss": 2.2114,
      "nll_loss": 2.16192626953125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19348883628845215,
      "rewards/margins": 0.04005904495716095,
      "rewards/rejected": -0.2335478812456131,
      "step": 6
    },
    {
      "epoch": 0.01103230890464933,
      "grad_norm": 0.7563782334327698,
      "learning_rate": 1.889763779527559e-07,
      "log_odds_chosen": 0.6660692095756531,
      "log_odds_ratio": -0.42836689949035645,
      "logits/chosen": -1.041650652885437,
      "logits/rejected": -0.4485802948474884,
      "logps/chosen": -2.0038270950317383,
      "logps/rejected": -2.6053221225738525,
      "loss": 2.2398,
      "nll_loss": 2.1969573497772217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20038272440433502,
      "rewards/margins": 0.06014948710799217,
      "rewards/rejected": -0.2605322301387787,
      "step": 7
    },
    {
      "epoch": 0.01260835303388495,
      "grad_norm": 1.0274051427841187,
      "learning_rate": 2.2047244094488187e-07,
      "log_odds_chosen": 0.9828638434410095,
      "log_odds_ratio": -0.32935816049575806,
      "logits/chosen": -1.1843122243881226,
      "logits/rejected": -0.48524755239486694,
      "logps/chosen": -2.126309871673584,
      "logps/rejected": -3.025899648666382,
      "loss": 2.3618,
      "nll_loss": 2.32889461517334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2126309871673584,
      "rewards/margins": 0.08995895087718964,
      "rewards/rejected": -0.30258995294570923,
      "step": 8
    },
    {
      "epoch": 0.014184397163120567,
      "grad_norm": 0.8085570931434631,
      "learning_rate": 2.519685039370079e-07,
      "log_odds_chosen": 0.5246677398681641,
      "log_odds_ratio": -0.47295087575912476,
      "logits/chosen": -1.1274431943893433,
      "logits/rejected": -0.4057602882385254,
      "logps/chosen": -2.0213370323181152,
      "logps/rejected": -2.4909844398498535,
      "loss": 2.2419,
      "nll_loss": 2.19463849067688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20213371515274048,
      "rewards/margins": 0.04696475714445114,
      "rewards/rejected": -0.24909847974777222,
      "step": 9
    },
    {
      "epoch": 0.015760441292356184,
      "grad_norm": 0.8120251893997192,
      "learning_rate": 2.8346456692913386e-07,
      "log_odds_chosen": 0.4664788842201233,
      "log_odds_ratio": -0.4924715459346771,
      "logits/chosen": -1.1194875240325928,
      "logits/rejected": -0.5692275166511536,
      "logps/chosen": -2.0412657260894775,
      "logps/rejected": -2.459512710571289,
      "loss": 2.2764,
      "nll_loss": 2.227172613143921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20412659645080566,
      "rewards/margins": 0.04182467237114906,
      "rewards/rejected": -0.24595126509666443,
      "step": 10
    },
    {
      "epoch": 0.017336485421591805,
      "grad_norm": 0.8484521508216858,
      "learning_rate": 3.1496062992125984e-07,
      "log_odds_chosen": 0.5531177520751953,
      "log_odds_ratio": -0.4599895477294922,
      "logits/chosen": -1.0869638919830322,
      "logits/rejected": -0.5173229575157166,
      "logps/chosen": -2.1112473011016846,
      "logps/rejected": -2.6104073524475098,
      "loss": 2.345,
      "nll_loss": 2.299004077911377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2111247330904007,
      "rewards/margins": 0.04991600662469864,
      "rewards/rejected": -0.26104071736335754,
      "step": 11
    },
    {
      "epoch": 0.018912529550827423,
      "grad_norm": 1.0205734968185425,
      "learning_rate": 3.464566929133858e-07,
      "log_odds_chosen": 0.6464576125144958,
      "log_odds_ratio": -0.4250268340110779,
      "logits/chosen": -1.194590449333191,
      "logits/rejected": -0.35468789935112,
      "logps/chosen": -2.164844274520874,
      "logps/rejected": -2.754187822341919,
      "loss": 2.4044,
      "nll_loss": 2.3619258403778076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2164844274520874,
      "rewards/margins": 0.05893436074256897,
      "rewards/rejected": -0.27541878819465637,
      "step": 12
    },
    {
      "epoch": 0.02048857368006304,
      "grad_norm": 0.9676651954650879,
      "learning_rate": 3.779527559055118e-07,
      "log_odds_chosen": 0.6292009949684143,
      "log_odds_ratio": -0.43755629658699036,
      "logits/chosen": -1.2696969509124756,
      "logits/rejected": -0.32901716232299805,
      "logps/chosen": -2.032942295074463,
      "logps/rejected": -2.598473072052002,
      "loss": 2.2742,
      "nll_loss": 2.2304177284240723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20329423248767853,
      "rewards/margins": 0.05655309930443764,
      "rewards/rejected": -0.25984734296798706,
      "step": 13
    },
    {
      "epoch": 0.02206461780929866,
      "grad_norm": 0.8347820043563843,
      "learning_rate": 4.0944881889763777e-07,
      "log_odds_chosen": 0.5126060247421265,
      "log_odds_ratio": -0.4873553514480591,
      "logits/chosen": -1.1185718774795532,
      "logits/rejected": -0.6450862884521484,
      "logps/chosen": -2.0508875846862793,
      "logps/rejected": -2.513838291168213,
      "loss": 2.2854,
      "nll_loss": 2.2367072105407715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2050887644290924,
      "rewards/margins": 0.04629506170749664,
      "rewards/rejected": -0.25138384103775024,
      "step": 14
    },
    {
      "epoch": 0.02364066193853428,
      "grad_norm": 0.9428237080574036,
      "learning_rate": 4.4094488188976375e-07,
      "log_odds_chosen": 0.45611634850502014,
      "log_odds_ratio": -0.49452677369117737,
      "logits/chosen": -1.2242058515548706,
      "logits/rejected": -0.40273138880729675,
      "logps/chosen": -2.0463950634002686,
      "logps/rejected": -2.456277847290039,
      "loss": 2.2971,
      "nll_loss": 2.2476325035095215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2046394795179367,
      "rewards/margins": 0.040988270193338394,
      "rewards/rejected": -0.245627760887146,
      "step": 15
    },
    {
      "epoch": 0.0252167060677699,
      "grad_norm": 0.8700807690620422,
      "learning_rate": 4.7244094488188973e-07,
      "log_odds_chosen": 0.48002687096595764,
      "log_odds_ratio": -0.4858929514884949,
      "logits/chosen": -1.1735236644744873,
      "logits/rejected": -0.44880083203315735,
      "logps/chosen": -2.0610547065734863,
      "logps/rejected": -2.49220871925354,
      "loss": 2.3018,
      "nll_loss": 2.253174304962158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20610548555850983,
      "rewards/margins": 0.04311539977788925,
      "rewards/rejected": -0.24922087788581848,
      "step": 16
    },
    {
      "epoch": 0.026792750197005517,
      "grad_norm": 0.7779992818832397,
      "learning_rate": 5.039370078740158e-07,
      "log_odds_chosen": 0.7011566162109375,
      "log_odds_ratio": -0.4202171266078949,
      "logits/chosen": -1.0814855098724365,
      "logits/rejected": -0.5505629777908325,
      "logps/chosen": -1.9978816509246826,
      "logps/rejected": -2.631817102432251,
      "loss": 2.2434,
      "nll_loss": 2.201402425765991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1997881531715393,
      "rewards/margins": 0.06339354813098907,
      "rewards/rejected": -0.2631816864013672,
      "step": 17
    },
    {
      "epoch": 0.028368794326241134,
      "grad_norm": 0.8743848204612732,
      "learning_rate": 5.354330708661418e-07,
      "log_odds_chosen": 0.46841195225715637,
      "log_odds_ratio": -0.49842947721481323,
      "logits/chosen": -1.1495157480239868,
      "logits/rejected": -0.2747257649898529,
      "logps/chosen": -2.0266854763031006,
      "logps/rejected": -2.4442427158355713,
      "loss": 2.2795,
      "nll_loss": 2.22969651222229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20266854763031006,
      "rewards/margins": 0.04175572097301483,
      "rewards/rejected": -0.2444242537021637,
      "step": 18
    },
    {
      "epoch": 0.029944838455476755,
      "grad_norm": 0.804558277130127,
      "learning_rate": 5.669291338582677e-07,
      "log_odds_chosen": 0.7196086645126343,
      "log_odds_ratio": -0.4083862602710724,
      "logits/chosen": -1.2688336372375488,
      "logits/rejected": -0.5452137589454651,
      "logps/chosen": -1.9217634201049805,
      "logps/rejected": -2.5629072189331055,
      "loss": 2.1817,
      "nll_loss": 2.140861988067627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19217635691165924,
      "rewards/margins": 0.06411437690258026,
      "rewards/rejected": -0.2562907338142395,
      "step": 19
    },
    {
      "epoch": 0.03152088258471237,
      "grad_norm": 0.8739065527915955,
      "learning_rate": 5.984251968503937e-07,
      "log_odds_chosen": 0.6575019359588623,
      "log_odds_ratio": -0.43753376603126526,
      "logits/chosen": -1.1189440488815308,
      "logits/rejected": -0.4846518933773041,
      "logps/chosen": -2.0866808891296387,
      "logps/rejected": -2.6833090782165527,
      "loss": 2.3356,
      "nll_loss": 2.291797637939453,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2086680680513382,
      "rewards/margins": 0.05966280773282051,
      "rewards/rejected": -0.2683309018611908,
      "step": 20
    },
    {
      "epoch": 0.03309692671394799,
      "grad_norm": 0.8919215798377991,
      "learning_rate": 6.299212598425197e-07,
      "log_odds_chosen": 0.4334430694580078,
      "log_odds_ratio": -0.505458652973175,
      "logits/chosen": -1.1694822311401367,
      "logits/rejected": -0.40514320135116577,
      "logps/chosen": -2.128809928894043,
      "logps/rejected": -2.52128529548645,
      "loss": 2.3712,
      "nll_loss": 2.3206286430358887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21288101375102997,
      "rewards/margins": 0.039247527718544006,
      "rewards/rejected": -0.252128541469574,
      "step": 21
    },
    {
      "epoch": 0.03467297084318361,
      "grad_norm": 0.7651381492614746,
      "learning_rate": 6.614173228346457e-07,
      "log_odds_chosen": 0.47628453373908997,
      "log_odds_ratio": -0.4947575330734253,
      "logits/chosen": -1.1822243928909302,
      "logits/rejected": -0.3983638286590576,
      "logps/chosen": -2.058316469192505,
      "logps/rejected": -2.486650228500366,
      "loss": 2.309,
      "nll_loss": 2.2594895362854004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2058316469192505,
      "rewards/margins": 0.042833391577005386,
      "rewards/rejected": -0.24866501986980438,
      "step": 22
    },
    {
      "epoch": 0.036249014972419225,
      "grad_norm": 0.8524187803268433,
      "learning_rate": 6.929133858267716e-07,
      "log_odds_chosen": 0.495402067899704,
      "log_odds_ratio": -0.4792757034301758,
      "logits/chosen": -1.201756238937378,
      "logits/rejected": -0.469428151845932,
      "logps/chosen": -2.114231824874878,
      "logps/rejected": -2.5619707107543945,
      "loss": 2.3369,
      "nll_loss": 2.2889456748962402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21142315864562988,
      "rewards/margins": 0.0447738915681839,
      "rewards/rejected": -0.25619709491729736,
      "step": 23
    },
    {
      "epoch": 0.037825059101654845,
      "grad_norm": 0.882310152053833,
      "learning_rate": 7.244094488188977e-07,
      "log_odds_chosen": 0.5385924577713013,
      "log_odds_ratio": -0.4700610339641571,
      "logits/chosen": -0.9846169948577881,
      "logits/rejected": -0.40137702226638794,
      "logps/chosen": -2.0582144260406494,
      "logps/rejected": -2.543081521987915,
      "loss": 2.3,
      "nll_loss": 2.2530057430267334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2058214396238327,
      "rewards/margins": 0.048486702144145966,
      "rewards/rejected": -0.25430813431739807,
      "step": 24
    },
    {
      "epoch": 0.039401103230890466,
      "grad_norm": 0.8820914030075073,
      "learning_rate": 7.559055118110236e-07,
      "log_odds_chosen": 0.4686305522918701,
      "log_odds_ratio": -0.496835321187973,
      "logits/chosen": -1.1956602334976196,
      "logits/rejected": -0.4693169891834259,
      "logps/chosen": -2.0715174674987793,
      "logps/rejected": -2.4980099201202393,
      "loss": 2.3124,
      "nll_loss": 2.262730598449707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20715174078941345,
      "rewards/margins": 0.04264923930168152,
      "rewards/rejected": -0.24980099499225616,
      "step": 25
    },
    {
      "epoch": 0.04097714736012608,
      "grad_norm": 0.8616840243339539,
      "learning_rate": 7.874015748031496e-07,
      "log_odds_chosen": 0.5564500093460083,
      "log_odds_ratio": -0.4641777276992798,
      "logits/chosen": -1.3434436321258545,
      "logits/rejected": -0.5958712100982666,
      "logps/chosen": -2.038738965988159,
      "logps/rejected": -2.541964292526245,
      "loss": 2.283,
      "nll_loss": 2.2365853786468506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2038739174604416,
      "rewards/margins": 0.050322525203228,
      "rewards/rejected": -0.254196435213089,
      "step": 26
    },
    {
      "epoch": 0.0425531914893617,
      "grad_norm": 0.8561832308769226,
      "learning_rate": 8.188976377952755e-07,
      "log_odds_chosen": 0.857874870300293,
      "log_odds_ratio": -0.37291765213012695,
      "logits/chosen": -1.1830471754074097,
      "logits/rejected": -0.4488833248615265,
      "logps/chosen": -2.08211612701416,
      "logps/rejected": -2.865851640701294,
      "loss": 2.3253,
      "nll_loss": 2.2880330085754395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20821160078048706,
      "rewards/margins": 0.07837357372045517,
      "rewards/rejected": -0.28658515214920044,
      "step": 27
    },
    {
      "epoch": 0.04412923561859732,
      "grad_norm": 0.8932236433029175,
      "learning_rate": 8.503937007874016e-07,
      "log_odds_chosen": 0.48930874466896057,
      "log_odds_ratio": -0.488573282957077,
      "logits/chosen": -1.2496106624603271,
      "logits/rejected": -0.4534827470779419,
      "logps/chosen": -2.0396242141723633,
      "logps/rejected": -2.4796881675720215,
      "loss": 2.2979,
      "nll_loss": 2.249030590057373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20396243035793304,
      "rewards/margins": 0.044006384909152985,
      "rewards/rejected": -0.24796880781650543,
      "step": 28
    },
    {
      "epoch": 0.045705279747832936,
      "grad_norm": 0.7671561241149902,
      "learning_rate": 8.818897637795275e-07,
      "log_odds_chosen": 0.45720863342285156,
      "log_odds_ratio": -0.4968447685241699,
      "logits/chosen": -0.9729629755020142,
      "logits/rejected": -0.4253592789173126,
      "logps/chosen": -2.1160941123962402,
      "logps/rejected": -2.5309629440307617,
      "loss": 2.3587,
      "nll_loss": 2.3090202808380127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21160940825939178,
      "rewards/margins": 0.04148688167333603,
      "rewards/rejected": -0.2530962824821472,
      "step": 29
    },
    {
      "epoch": 0.04728132387706856,
      "grad_norm": 0.9391067624092102,
      "learning_rate": 9.133858267716535e-07,
      "log_odds_chosen": 0.6915267109870911,
      "log_odds_ratio": -0.4149508774280548,
      "logits/chosen": -1.2198799848556519,
      "logits/rejected": -0.5714092254638672,
      "logps/chosen": -2.0917234420776367,
      "logps/rejected": -2.7195608615875244,
      "loss": 2.3653,
      "nll_loss": 2.3238537311553955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2091723531484604,
      "rewards/margins": 0.06278377026319504,
      "rewards/rejected": -0.27195608615875244,
      "step": 30
    },
    {
      "epoch": 0.04885736800630418,
      "grad_norm": 0.9174475073814392,
      "learning_rate": 9.448818897637795e-07,
      "log_odds_chosen": 0.7068430185317993,
      "log_odds_ratio": -0.4144725501537323,
      "logits/chosen": -1.1542574167251587,
      "logits/rejected": -0.2714862823486328,
      "logps/chosen": -2.1507794857025146,
      "logps/rejected": -2.7977755069732666,
      "loss": 2.3781,
      "nll_loss": 2.336669921875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2150779515504837,
      "rewards/margins": 0.06469962000846863,
      "rewards/rejected": -0.27977755665779114,
      "step": 31
    },
    {
      "epoch": 0.0504334121355398,
      "grad_norm": 0.7859240174293518,
      "learning_rate": 9.763779527559055e-07,
      "log_odds_chosen": 0.7807643413543701,
      "log_odds_ratio": -0.3978583812713623,
      "logits/chosen": -1.1065948009490967,
      "logits/rejected": -0.49257779121398926,
      "logps/chosen": -2.019965410232544,
      "logps/rejected": -2.7294366359710693,
      "loss": 2.2535,
      "nll_loss": 2.2136945724487305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20199653506278992,
      "rewards/margins": 0.07094711065292358,
      "rewards/rejected": -0.2729436457157135,
      "step": 32
    },
    {
      "epoch": 0.05200945626477541,
      "grad_norm": 0.7924520969390869,
      "learning_rate": 1.0078740157480315e-06,
      "log_odds_chosen": 0.6617920994758606,
      "log_odds_ratio": -0.42144879698753357,
      "logits/chosen": -1.0062627792358398,
      "logits/rejected": -0.4094982147216797,
      "logps/chosen": -2.043147563934326,
      "logps/rejected": -2.640801191329956,
      "loss": 2.2707,
      "nll_loss": 2.2286031246185303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20431476831436157,
      "rewards/margins": 0.059765346348285675,
      "rewards/rejected": -0.26408010721206665,
      "step": 33
    },
    {
      "epoch": 0.05358550039401103,
      "grad_norm": 0.8600611090660095,
      "learning_rate": 1.0393700787401573e-06,
      "log_odds_chosen": 0.3994464874267578,
      "log_odds_ratio": -0.5219913721084595,
      "logits/chosen": -1.075833797454834,
      "logits/rejected": -0.47573941946029663,
      "logps/chosen": -1.9926190376281738,
      "logps/rejected": -2.3437416553497314,
      "loss": 2.2453,
      "nll_loss": 2.1931371688842773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19926190376281738,
      "rewards/margins": 0.035112280398607254,
      "rewards/rejected": -0.23437418043613434,
      "step": 34
    },
    {
      "epoch": 0.055161544523246654,
      "grad_norm": 1.0218589305877686,
      "learning_rate": 1.0708661417322836e-06,
      "log_odds_chosen": 0.5247014760971069,
      "log_odds_ratio": -0.4662986695766449,
      "logits/chosen": -1.2422465085983276,
      "logits/rejected": -0.2719176113605499,
      "logps/chosen": -2.167281150817871,
      "logps/rejected": -2.6426916122436523,
      "loss": 2.4104,
      "nll_loss": 2.3638148307800293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2167281061410904,
      "rewards/margins": 0.047541067004203796,
      "rewards/rejected": -0.2642691731452942,
      "step": 35
    },
    {
      "epoch": 0.05673758865248227,
      "grad_norm": 0.8558648824691772,
      "learning_rate": 1.1023622047244094e-06,
      "log_odds_chosen": 0.643022358417511,
      "log_odds_ratio": -0.4252229630947113,
      "logits/chosen": -1.2594010829925537,
      "logits/rejected": -0.3225390613079071,
      "logps/chosen": -1.9943788051605225,
      "logps/rejected": -2.570138931274414,
      "loss": 2.2371,
      "nll_loss": 2.1945650577545166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19943787157535553,
      "rewards/margins": 0.0575760118663311,
      "rewards/rejected": -0.25701388716697693,
      "step": 36
    },
    {
      "epoch": 0.05831363278171789,
      "grad_norm": 0.7476872205734253,
      "learning_rate": 1.1338582677165354e-06,
      "log_odds_chosen": 0.4776890277862549,
      "log_odds_ratio": -0.4936005175113678,
      "logits/chosen": -1.096699595451355,
      "logits/rejected": -0.3185568153858185,
      "logps/chosen": -1.997284173965454,
      "logps/rejected": -2.4194047451019287,
      "loss": 2.2546,
      "nll_loss": 2.205209732055664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19972839951515198,
      "rewards/margins": 0.04221208393573761,
      "rewards/rejected": -0.24194049835205078,
      "step": 37
    },
    {
      "epoch": 0.05988967691095351,
      "grad_norm": 0.8627468347549438,
      "learning_rate": 1.1653543307086612e-06,
      "log_odds_chosen": 0.464214563369751,
      "log_odds_ratio": -0.4933066964149475,
      "logits/chosen": -1.277733325958252,
      "logits/rejected": -0.5121052861213684,
      "logps/chosen": -1.991890788078308,
      "logps/rejected": -2.401747226715088,
      "loss": 2.2537,
      "nll_loss": 2.20432186126709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19918911159038544,
      "rewards/margins": 0.040985628962516785,
      "rewards/rejected": -0.24017474055290222,
      "step": 38
    },
    {
      "epoch": 0.061465721040189124,
      "grad_norm": 0.9292140007019043,
      "learning_rate": 1.1968503937007875e-06,
      "log_odds_chosen": 0.30634605884552,
      "log_odds_ratio": -0.5660471320152283,
      "logits/chosen": -1.211570382118225,
      "logits/rejected": -0.37499088048934937,
      "logps/chosen": -2.146724224090576,
      "logps/rejected": -2.425945997238159,
      "loss": 2.4043,
      "nll_loss": 2.3476977348327637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21467243134975433,
      "rewards/margins": 0.027922188863158226,
      "rewards/rejected": -0.2425946146249771,
      "step": 39
    },
    {
      "epoch": 0.06304176516942474,
      "grad_norm": 0.7580855488777161,
      "learning_rate": 1.2283464566929133e-06,
      "log_odds_chosen": 0.4294065237045288,
      "log_odds_ratio": -0.5072289705276489,
      "logits/chosen": -1.155901312828064,
      "logits/rejected": -0.45106494426727295,
      "logps/chosen": -1.9809616804122925,
      "logps/rejected": -2.3610317707061768,
      "loss": 2.2388,
      "nll_loss": 2.188109874725342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1980961561203003,
      "rewards/margins": 0.03800702840089798,
      "rewards/rejected": -0.23610319197177887,
      "step": 40
    },
    {
      "epoch": 0.06461780929866036,
      "grad_norm": 0.7090384364128113,
      "learning_rate": 1.2598425196850393e-06,
      "log_odds_chosen": 0.32460513710975647,
      "log_odds_ratio": -0.5508446097373962,
      "logits/chosen": -1.0451905727386475,
      "logits/rejected": -0.36920034885406494,
      "logps/chosen": -2.0002119541168213,
      "logps/rejected": -2.289102792739868,
      "loss": 2.2423,
      "nll_loss": 2.187190294265747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2000211924314499,
      "rewards/margins": 0.02888909913599491,
      "rewards/rejected": -0.22891029715538025,
      "step": 41
    },
    {
      "epoch": 0.06619385342789598,
      "grad_norm": 0.7185447812080383,
      "learning_rate": 1.2913385826771652e-06,
      "log_odds_chosen": 0.38940221071243286,
      "log_odds_ratio": -0.5234697461128235,
      "logits/chosen": -1.081929326057434,
      "logits/rejected": -0.42588090896606445,
      "logps/chosen": -2.010281562805176,
      "logps/rejected": -2.356483221054077,
      "loss": 2.2614,
      "nll_loss": 2.209080457687378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20102813839912415,
      "rewards/margins": 0.03462015092372894,
      "rewards/rejected": -0.23564831912517548,
      "step": 42
    },
    {
      "epoch": 0.0677698975571316,
      "grad_norm": 0.784934937953949,
      "learning_rate": 1.3228346456692914e-06,
      "log_odds_chosen": 0.23201489448547363,
      "log_odds_ratio": -0.592602550983429,
      "logits/chosen": -1.0133355855941772,
      "logits/rejected": -0.35346853733062744,
      "logps/chosen": -2.0479795932769775,
      "logps/rejected": -2.253965377807617,
      "loss": 2.2972,
      "nll_loss": 2.237945318222046,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20479796826839447,
      "rewards/margins": 0.02059856988489628,
      "rewards/rejected": -0.2253965437412262,
      "step": 43
    },
    {
      "epoch": 0.06934594168636722,
      "grad_norm": 0.8619470000267029,
      "learning_rate": 1.3543307086614172e-06,
      "log_odds_chosen": 0.5962916612625122,
      "log_odds_ratio": -0.44894152879714966,
      "logits/chosen": -1.2180397510528564,
      "logits/rejected": -0.26432231068611145,
      "logps/chosen": -2.0331180095672607,
      "logps/rejected": -2.5704658031463623,
      "loss": 2.2728,
      "nll_loss": 2.2278692722320557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20331181585788727,
      "rewards/margins": 0.053734779357910156,
      "rewards/rejected": -0.25704658031463623,
      "step": 44
    },
    {
      "epoch": 0.07092198581560284,
      "grad_norm": 0.7422481179237366,
      "learning_rate": 1.3858267716535433e-06,
      "log_odds_chosen": 0.6413304805755615,
      "log_odds_ratio": -0.43954819440841675,
      "logits/chosen": -1.012795329093933,
      "logits/rejected": -0.33944788575172424,
      "logps/chosen": -1.9753161668777466,
      "logps/rejected": -2.550520420074463,
      "loss": 2.2443,
      "nll_loss": 2.2003304958343506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19753161072731018,
      "rewards/margins": 0.057520411908626556,
      "rewards/rejected": -0.25505203008651733,
      "step": 45
    },
    {
      "epoch": 0.07249802994483845,
      "grad_norm": 0.9379796981811523,
      "learning_rate": 1.417322834645669e-06,
      "log_odds_chosen": 0.6898038983345032,
      "log_odds_ratio": -0.4229612946510315,
      "logits/chosen": -1.2984787225723267,
      "logits/rejected": -0.6316989064216614,
      "logps/chosen": -2.0245230197906494,
      "logps/rejected": -2.6471643447875977,
      "loss": 2.2772,
      "nll_loss": 2.2349252700805664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20245231688022614,
      "rewards/margins": 0.06226412206888199,
      "rewards/rejected": -0.2647164463996887,
      "step": 46
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 0.7300897240638733,
      "learning_rate": 1.4488188976377953e-06,
      "log_odds_chosen": 0.34559765458106995,
      "log_odds_ratio": -0.5484777688980103,
      "logits/chosen": -1.1782827377319336,
      "logits/rejected": -0.4611784517765045,
      "logps/chosen": -2.0315334796905518,
      "logps/rejected": -2.342702627182007,
      "loss": 2.2798,
      "nll_loss": 2.2249908447265625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2031533569097519,
      "rewards/margins": 0.031116927042603493,
      "rewards/rejected": -0.23427024483680725,
      "step": 47
    },
    {
      "epoch": 0.07565011820330969,
      "grad_norm": 0.9040903449058533,
      "learning_rate": 1.4803149606299211e-06,
      "log_odds_chosen": 0.48583200573921204,
      "log_odds_ratio": -0.4895709156990051,
      "logits/chosen": -1.0726096630096436,
      "logits/rejected": -0.4579828083515167,
      "logps/chosen": -2.1052684783935547,
      "logps/rejected": -2.546491861343384,
      "loss": 2.3626,
      "nll_loss": 2.313596487045288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21052685379981995,
      "rewards/margins": 0.04412233084440231,
      "rewards/rejected": -0.25464919209480286,
      "step": 48
    },
    {
      "epoch": 0.07722616233254531,
      "grad_norm": 0.7999246716499329,
      "learning_rate": 1.5118110236220472e-06,
      "log_odds_chosen": 0.586567759513855,
      "log_odds_ratio": -0.4494791030883789,
      "logits/chosen": -1.1561152935028076,
      "logits/rejected": -0.3217093050479889,
      "logps/chosen": -1.985177993774414,
      "logps/rejected": -2.5069286823272705,
      "loss": 2.2018,
      "nll_loss": 2.1568918228149414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1985177993774414,
      "rewards/margins": 0.05217508226633072,
      "rewards/rejected": -0.25069287419319153,
      "step": 49
    },
    {
      "epoch": 0.07880220646178093,
      "grad_norm": 0.8259245753288269,
      "learning_rate": 1.543307086614173e-06,
      "log_odds_chosen": 0.36837247014045715,
      "log_odds_ratio": -0.5344362258911133,
      "logits/chosen": -1.125555157661438,
      "logits/rejected": -0.47867536544799805,
      "logps/chosen": -2.066678285598755,
      "logps/rejected": -2.398474931716919,
      "loss": 2.3174,
      "nll_loss": 2.26397967338562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20666784048080444,
      "rewards/margins": 0.0331796295940876,
      "rewards/rejected": -0.23984746634960175,
      "step": 50
    },
    {
      "epoch": 0.08037825059101655,
      "grad_norm": 0.781063973903656,
      "learning_rate": 1.5748031496062992e-06,
      "log_odds_chosen": 0.6516568660736084,
      "log_odds_ratio": -0.42411038279533386,
      "logits/chosen": -1.0272550582885742,
      "logits/rejected": -0.3421846330165863,
      "logps/chosen": -1.9430203437805176,
      "logps/rejected": -2.520514488220215,
      "loss": 2.1832,
      "nll_loss": 2.1408329010009766,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1943020224571228,
      "rewards/margins": 0.057749420404434204,
      "rewards/rejected": -0.252051442861557,
      "step": 51
    },
    {
      "epoch": 0.08195429472025216,
      "grad_norm": 0.7726197242736816,
      "learning_rate": 1.6062992125984253e-06,
      "log_odds_chosen": 0.6141126751899719,
      "log_odds_ratio": -0.441342830657959,
      "logits/chosen": -1.0188478231430054,
      "logits/rejected": -0.29862481355667114,
      "logps/chosen": -2.097641706466675,
      "logps/rejected": -2.6549832820892334,
      "loss": 2.3284,
      "nll_loss": 2.2842369079589844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20976416766643524,
      "rewards/margins": 0.055734165012836456,
      "rewards/rejected": -0.2654983401298523,
      "step": 52
    },
    {
      "epoch": 0.08353033884948778,
      "grad_norm": 0.8193818926811218,
      "learning_rate": 1.637795275590551e-06,
      "log_odds_chosen": 0.3866994082927704,
      "log_odds_ratio": -0.5209422707557678,
      "logits/chosen": -1.1183687448501587,
      "logits/rejected": -0.4406161308288574,
      "logps/chosen": -2.0367820262908936,
      "logps/rejected": -2.3801722526550293,
      "loss": 2.2824,
      "nll_loss": 2.2303147315979004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2036782205104828,
      "rewards/margins": 0.03433902934193611,
      "rewards/rejected": -0.2380172461271286,
      "step": 53
    },
    {
      "epoch": 0.0851063829787234,
      "grad_norm": 0.7303316593170166,
      "learning_rate": 1.6692913385826771e-06,
      "log_odds_chosen": 0.3992087244987488,
      "log_odds_ratio": -0.5159623622894287,
      "logits/chosen": -1.1815376281738281,
      "logits/rejected": -0.397248238325119,
      "logps/chosen": -1.9723021984100342,
      "logps/rejected": -2.3256380558013916,
      "loss": 2.2264,
      "nll_loss": 2.1747708320617676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19723021984100342,
      "rewards/margins": 0.035333603620529175,
      "rewards/rejected": -0.2325638383626938,
      "step": 54
    },
    {
      "epoch": 0.08668242710795902,
      "grad_norm": 0.8129874467849731,
      "learning_rate": 1.7007874015748031e-06,
      "log_odds_chosen": 0.47329652309417725,
      "log_odds_ratio": -0.4876212477684021,
      "logits/chosen": -1.096902847290039,
      "logits/rejected": -0.4683460593223572,
      "logps/chosen": -2.063098907470703,
      "logps/rejected": -2.4882240295410156,
      "loss": 2.3196,
      "nll_loss": 2.2708041667938232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20630988478660583,
      "rewards/margins": 0.04251253604888916,
      "rewards/rejected": -0.248822420835495,
      "step": 55
    },
    {
      "epoch": 0.08825847123719464,
      "grad_norm": 0.911749541759491,
      "learning_rate": 1.7322834645669292e-06,
      "log_odds_chosen": 0.4285861551761627,
      "log_odds_ratio": -0.514061450958252,
      "logits/chosen": -1.0361069440841675,
      "logits/rejected": -0.3017573356628418,
      "logps/chosen": -2.170567512512207,
      "logps/rejected": -2.559295654296875,
      "loss": 2.4113,
      "nll_loss": 2.359891176223755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2170567363500595,
      "rewards/margins": 0.03887280821800232,
      "rewards/rejected": -0.255929559469223,
      "step": 56
    },
    {
      "epoch": 0.08983451536643026,
      "grad_norm": 0.7290955185890198,
      "learning_rate": 1.763779527559055e-06,
      "log_odds_chosen": 0.44968611001968384,
      "log_odds_ratio": -0.4962458908557892,
      "logits/chosen": -1.018329381942749,
      "logits/rejected": -0.4052811563014984,
      "logps/chosen": -2.0387189388275146,
      "logps/rejected": -2.4409403800964355,
      "loss": 2.2741,
      "nll_loss": 2.2244579792022705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20387189090251923,
      "rewards/margins": 0.040222153067588806,
      "rewards/rejected": -0.24409404397010803,
      "step": 57
    },
    {
      "epoch": 0.09141055949566587,
      "grad_norm": 0.769049346446991,
      "learning_rate": 1.795275590551181e-06,
      "log_odds_chosen": 0.4315187931060791,
      "log_odds_ratio": -0.5120344161987305,
      "logits/chosen": -1.0596637725830078,
      "logits/rejected": -0.5503028631210327,
      "logps/chosen": -1.9930483102798462,
      "logps/rejected": -2.3822011947631836,
      "loss": 2.2429,
      "nll_loss": 2.1917452812194824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19930484890937805,
      "rewards/margins": 0.038915280252695084,
      "rewards/rejected": -0.23822011053562164,
      "step": 58
    },
    {
      "epoch": 0.09298660362490149,
      "grad_norm": 0.8504804968833923,
      "learning_rate": 1.826771653543307e-06,
      "log_odds_chosen": 0.7453770637512207,
      "log_odds_ratio": -0.39689797163009644,
      "logits/chosen": -0.9750216603279114,
      "logits/rejected": -0.22442440688610077,
      "logps/chosen": -2.101952075958252,
      "logps/rejected": -2.782055139541626,
      "loss": 2.3218,
      "nll_loss": 2.2820687294006348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21019521355628967,
      "rewards/margins": 0.06801028549671173,
      "rewards/rejected": -0.2782055139541626,
      "step": 59
    },
    {
      "epoch": 0.09456264775413711,
      "grad_norm": 0.861810028553009,
      "learning_rate": 1.858267716535433e-06,
      "log_odds_chosen": 0.5790901184082031,
      "log_odds_ratio": -0.45646488666534424,
      "logits/chosen": -1.1360684633255005,
      "logits/rejected": -0.53347247838974,
      "logps/chosen": -2.0499930381774902,
      "logps/rejected": -2.5749099254608154,
      "loss": 2.2932,
      "nll_loss": 2.2475974559783936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20499931275844574,
      "rewards/margins": 0.0524916872382164,
      "rewards/rejected": -0.25749099254608154,
      "step": 60
    },
    {
      "epoch": 0.09613869188337273,
      "grad_norm": 0.84038907289505,
      "learning_rate": 1.889763779527559e-06,
      "log_odds_chosen": 0.3383483290672302,
      "log_odds_ratio": -0.5470318794250488,
      "logits/chosen": -0.9991433620452881,
      "logits/rejected": -0.4732619822025299,
      "logps/chosen": -2.1595866680145264,
      "logps/rejected": -2.4666683673858643,
      "loss": 2.3946,
      "nll_loss": 2.3399457931518555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21595866978168488,
      "rewards/margins": 0.03070816770195961,
      "rewards/rejected": -0.24666684865951538,
      "step": 61
    },
    {
      "epoch": 0.09771473601260836,
      "grad_norm": 1.0108712911605835,
      "learning_rate": 1.9212598425196847e-06,
      "log_odds_chosen": 0.37255755066871643,
      "log_odds_ratio": -0.5352697968482971,
      "logits/chosen": -1.1140410900115967,
      "logits/rejected": -0.4322108030319214,
      "logps/chosen": -2.160331964492798,
      "logps/rejected": -2.4993157386779785,
      "loss": 2.4019,
      "nll_loss": 2.3483951091766357,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2160331904888153,
      "rewards/margins": 0.03389836102724075,
      "rewards/rejected": -0.24993157386779785,
      "step": 62
    },
    {
      "epoch": 0.09929078014184398,
      "grad_norm": 0.7643742561340332,
      "learning_rate": 1.952755905511811e-06,
      "log_odds_chosen": 0.6516906023025513,
      "log_odds_ratio": -0.42522722482681274,
      "logits/chosen": -1.1658883094787598,
      "logits/rejected": -0.45858240127563477,
      "logps/chosen": -1.878890037536621,
      "logps/rejected": -2.449535608291626,
      "loss": 2.138,
      "nll_loss": 2.0955119132995605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1878889948129654,
      "rewards/margins": 0.057064566761255264,
      "rewards/rejected": -0.24495357275009155,
      "step": 63
    },
    {
      "epoch": 0.1008668242710796,
      "grad_norm": 0.7522913813591003,
      "learning_rate": 1.9842519685039368e-06,
      "log_odds_chosen": 0.766040563583374,
      "log_odds_ratio": -0.397356241941452,
      "logits/chosen": -1.1226662397384644,
      "logits/rejected": -0.3395186960697174,
      "logps/chosen": -1.8949854373931885,
      "logps/rejected": -2.5772881507873535,
      "loss": 2.1507,
      "nll_loss": 2.110943555831909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18949854373931885,
      "rewards/margins": 0.06823026388883591,
      "rewards/rejected": -0.25772881507873535,
      "step": 64
    },
    {
      "epoch": 0.1024428684003152,
      "grad_norm": 0.7364575266838074,
      "learning_rate": 2.015748031496063e-06,
      "log_odds_chosen": 0.6714511513710022,
      "log_odds_ratio": -0.4194810390472412,
      "logits/chosen": -1.0760542154312134,
      "logits/rejected": -0.5785982012748718,
      "logps/chosen": -1.9915119409561157,
      "logps/rejected": -2.5946145057678223,
      "loss": 2.2392,
      "nll_loss": 2.1972577571868896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1991512030363083,
      "rewards/margins": 0.060310248285532,
      "rewards/rejected": -0.2594614326953888,
      "step": 65
    },
    {
      "epoch": 0.10401891252955082,
      "grad_norm": 0.7563380599021912,
      "learning_rate": 2.047244094488189e-06,
      "log_odds_chosen": 0.5152915716171265,
      "log_odds_ratio": -0.4739663004875183,
      "logits/chosen": -1.0727609395980835,
      "logits/rejected": -0.3997894525527954,
      "logps/chosen": -2.100337028503418,
      "logps/rejected": -2.5641911029815674,
      "loss": 2.353,
      "nll_loss": 2.305567741394043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21003369987010956,
      "rewards/margins": 0.046385399997234344,
      "rewards/rejected": -0.2564191222190857,
      "step": 66
    },
    {
      "epoch": 0.10559495665878645,
      "grad_norm": 0.8397193551063538,
      "learning_rate": 2.0787401574803147e-06,
      "log_odds_chosen": 0.6061793565750122,
      "log_odds_ratio": -0.4402039349079132,
      "logits/chosen": -1.0465192794799805,
      "logits/rejected": -0.266944944858551,
      "logps/chosen": -2.039686441421509,
      "logps/rejected": -2.583190441131592,
      "loss": 2.2643,
      "nll_loss": 2.220245122909546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20396864414215088,
      "rewards/margins": 0.054350391030311584,
      "rewards/rejected": -0.25831905007362366,
      "step": 67
    },
    {
      "epoch": 0.10717100078802207,
      "grad_norm": 0.8782281875610352,
      "learning_rate": 2.1102362204724405e-06,
      "log_odds_chosen": 0.4924987852573395,
      "log_odds_ratio": -0.4848763942718506,
      "logits/chosen": -1.1445033550262451,
      "logits/rejected": -0.40652817487716675,
      "logps/chosen": -2.0268895626068115,
      "logps/rejected": -2.46834397315979,
      "loss": 2.3111,
      "nll_loss": 2.2625656127929688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20268897712230682,
      "rewards/margins": 0.04414542764425278,
      "rewards/rejected": -0.246834397315979,
      "step": 68
    },
    {
      "epoch": 0.10874704491725769,
      "grad_norm": 0.8901786208152771,
      "learning_rate": 2.141732283464567e-06,
      "log_odds_chosen": 0.28019726276397705,
      "log_odds_ratio": -0.572468638420105,
      "logits/chosen": -0.9071764945983887,
      "logits/rejected": -0.36978206038475037,
      "logps/chosen": -2.1059303283691406,
      "logps/rejected": -2.357771396636963,
      "loss": 2.3697,
      "nll_loss": 2.3124704360961914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21059304475784302,
      "rewards/margins": 0.025184106081724167,
      "rewards/rejected": -0.2357771247625351,
      "step": 69
    },
    {
      "epoch": 0.11032308904649331,
      "grad_norm": 0.7347759008407593,
      "learning_rate": 2.173228346456693e-06,
      "log_odds_chosen": 0.5119097232818604,
      "log_odds_ratio": -0.47549861669540405,
      "logits/chosen": -0.9662876129150391,
      "logits/rejected": -0.3414289951324463,
      "logps/chosen": -2.018106460571289,
      "logps/rejected": -2.4745588302612305,
      "loss": 2.2778,
      "nll_loss": 2.230213165283203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20181065797805786,
      "rewards/margins": 0.04564525932073593,
      "rewards/rejected": -0.247455894947052,
      "step": 70
    },
    {
      "epoch": 0.11189913317572892,
      "grad_norm": 0.8959264159202576,
      "learning_rate": 2.204724409448819e-06,
      "log_odds_chosen": 0.5562199354171753,
      "log_odds_ratio": -0.46300366520881653,
      "logits/chosen": -1.2209943532943726,
      "logits/rejected": -0.5177184343338013,
      "logps/chosen": -1.9901155233383179,
      "logps/rejected": -2.489285707473755,
      "loss": 2.2383,
      "nll_loss": 2.1919736862182617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19901156425476074,
      "rewards/margins": 0.04991701990365982,
      "rewards/rejected": -0.24892857670783997,
      "step": 71
    },
    {
      "epoch": 0.11347517730496454,
      "grad_norm": 0.8339924216270447,
      "learning_rate": 2.2362204724409446e-06,
      "log_odds_chosen": 0.37862321734428406,
      "log_odds_ratio": -0.5375041961669922,
      "logits/chosen": -1.1112946271896362,
      "logits/rejected": -0.3363880217075348,
      "logps/chosen": -2.0221199989318848,
      "logps/rejected": -2.3621301651000977,
      "loss": 2.2684,
      "nll_loss": 2.2146549224853516,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20221197605133057,
      "rewards/margins": 0.03400101885199547,
      "rewards/rejected": -0.23621301352977753,
      "step": 72
    },
    {
      "epoch": 0.11505122143420016,
      "grad_norm": 0.7132512927055359,
      "learning_rate": 2.267716535433071e-06,
      "log_odds_chosen": 0.3846838176250458,
      "log_odds_ratio": -0.5223243236541748,
      "logits/chosen": -0.9467731714248657,
      "logits/rejected": -0.4224381446838379,
      "logps/chosen": -2.0288405418395996,
      "logps/rejected": -2.371960163116455,
      "loss": 2.2859,
      "nll_loss": 2.233703136444092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20288404822349548,
      "rewards/margins": 0.03431195765733719,
      "rewards/rejected": -0.23719602823257446,
      "step": 73
    },
    {
      "epoch": 0.11662726556343578,
      "grad_norm": 0.8109890222549438,
      "learning_rate": 2.2992125984251967e-06,
      "log_odds_chosen": 0.49797457456588745,
      "log_odds_ratio": -0.4787079095840454,
      "logits/chosen": -1.117811918258667,
      "logits/rejected": -0.44548889994621277,
      "logps/chosen": -1.9113142490386963,
      "logps/rejected": -2.3477611541748047,
      "loss": 2.1852,
      "nll_loss": 2.1373038291931152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19113142788410187,
      "rewards/margins": 0.04364468902349472,
      "rewards/rejected": -0.23477612435817719,
      "step": 74
    },
    {
      "epoch": 0.1182033096926714,
      "grad_norm": 0.7836765646934509,
      "learning_rate": 2.3307086614173225e-06,
      "log_odds_chosen": 0.5877864956855774,
      "log_odds_ratio": -0.4508923292160034,
      "logits/chosen": -1.0647002458572388,
      "logits/rejected": -0.5113080143928528,
      "logps/chosen": -1.9603990316390991,
      "logps/rejected": -2.4868850708007812,
      "loss": 2.207,
      "nll_loss": 2.1619343757629395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19603991508483887,
      "rewards/margins": 0.052648596465587616,
      "rewards/rejected": -0.2486885040998459,
      "step": 75
    },
    {
      "epoch": 0.11977935382190702,
      "grad_norm": 0.7919722199440002,
      "learning_rate": 2.3622047244094483e-06,
      "log_odds_chosen": 0.6009241342544556,
      "log_odds_ratio": -0.4470832049846649,
      "logits/chosen": -0.9855466485023499,
      "logits/rejected": -0.2876882553100586,
      "logps/chosen": -2.008293390274048,
      "logps/rejected": -2.5480880737304688,
      "loss": 2.2372,
      "nll_loss": 2.192521333694458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20082935690879822,
      "rewards/margins": 0.053979478776454926,
      "rewards/rejected": -0.25480881333351135,
      "step": 76
    },
    {
      "epoch": 0.12135539795114263,
      "grad_norm": 0.8516606688499451,
      "learning_rate": 2.393700787401575e-06,
      "log_odds_chosen": 0.5337265729904175,
      "log_odds_ratio": -0.47431042790412903,
      "logits/chosen": -1.0908458232879639,
      "logits/rejected": -0.4919825494289398,
      "logps/chosen": -2.0387442111968994,
      "logps/rejected": -2.518265724182129,
      "loss": 2.2933,
      "nll_loss": 2.245901346206665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20387442409992218,
      "rewards/margins": 0.04795214533805847,
      "rewards/rejected": -0.25182655453681946,
      "step": 77
    },
    {
      "epoch": 0.12293144208037825,
      "grad_norm": 0.7652468085289001,
      "learning_rate": 2.425196850393701e-06,
      "log_odds_chosen": 0.4612125754356384,
      "log_odds_ratio": -0.49879589676856995,
      "logits/chosen": -0.9081002473831177,
      "logits/rejected": -0.6156851053237915,
      "logps/chosen": -2.115405559539795,
      "logps/rejected": -2.532501697540283,
      "loss": 2.3363,
      "nll_loss": 2.28646183013916,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2115405648946762,
      "rewards/margins": 0.04170961678028107,
      "rewards/rejected": -0.2532501816749573,
      "step": 78
    },
    {
      "epoch": 0.12450748620961387,
      "grad_norm": 0.7574028372764587,
      "learning_rate": 2.4566929133858266e-06,
      "log_odds_chosen": 0.5914417505264282,
      "log_odds_ratio": -0.45016127824783325,
      "logits/chosen": -0.937514066696167,
      "logits/rejected": -0.5195568799972534,
      "logps/chosen": -2.0114662647247314,
      "logps/rejected": -2.5420799255371094,
      "loss": 2.2491,
      "nll_loss": 2.2040576934814453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20114661753177643,
      "rewards/margins": 0.05306137725710869,
      "rewards/rejected": -0.2542080283164978,
      "step": 79
    },
    {
      "epoch": 0.12608353033884948,
      "grad_norm": 0.7048937678337097,
      "learning_rate": 2.488188976377953e-06,
      "log_odds_chosen": 0.5971450209617615,
      "log_odds_ratio": -0.443267822265625,
      "logits/chosen": -1.0094927549362183,
      "logits/rejected": -0.4586561918258667,
      "logps/chosen": -1.9250359535217285,
      "logps/rejected": -2.4535298347473145,
      "loss": 2.1666,
      "nll_loss": 2.122244358062744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19250361621379852,
      "rewards/margins": 0.05284937843680382,
      "rewards/rejected": -0.24535296857357025,
      "step": 80
    },
    {
      "epoch": 0.1276595744680851,
      "grad_norm": 0.66707843542099,
      "learning_rate": 2.5196850393700787e-06,
      "log_odds_chosen": 0.43576544523239136,
      "log_odds_ratio": -0.5106025338172913,
      "logits/chosen": -0.8745508193969727,
      "logits/rejected": -0.5163943767547607,
      "logps/chosen": -2.004204273223877,
      "logps/rejected": -2.396484851837158,
      "loss": 2.2337,
      "nll_loss": 2.182626724243164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20042043924331665,
      "rewards/margins": 0.039228055626153946,
      "rewards/rejected": -0.2396484911441803,
      "step": 81
    },
    {
      "epoch": 0.12923561859732072,
      "grad_norm": 0.6626860499382019,
      "learning_rate": 2.5511811023622045e-06,
      "log_odds_chosen": 0.6387014985084534,
      "log_odds_ratio": -0.4447665214538574,
      "logits/chosen": -0.8805458545684814,
      "logits/rejected": -0.41358864307403564,
      "logps/chosen": -1.9391623735427856,
      "logps/rejected": -2.517059803009033,
      "loss": 2.1816,
      "nll_loss": 2.137143611907959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19391624629497528,
      "rewards/margins": 0.05778975784778595,
      "rewards/rejected": -0.25170600414276123,
      "step": 82
    },
    {
      "epoch": 0.13081166272655634,
      "grad_norm": 0.7182599306106567,
      "learning_rate": 2.5826771653543303e-06,
      "log_odds_chosen": 0.43168193101882935,
      "log_odds_ratio": -0.5112774968147278,
      "logits/chosen": -0.8949125409126282,
      "logits/rejected": -0.4265552759170532,
      "logps/chosen": -1.9767383337020874,
      "logps/rejected": -2.3620519638061523,
      "loss": 2.2121,
      "nll_loss": 2.160935401916504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19767382740974426,
      "rewards/margins": 0.038531385362148285,
      "rewards/rejected": -0.23620523512363434,
      "step": 83
    },
    {
      "epoch": 0.13238770685579196,
      "grad_norm": 0.7394052743911743,
      "learning_rate": 2.6141732283464566e-06,
      "log_odds_chosen": 0.6156221628189087,
      "log_odds_ratio": -0.449825644493103,
      "logits/chosen": -0.9797751903533936,
      "logits/rejected": -0.4194489121437073,
      "logps/chosen": -1.9231019020080566,
      "logps/rejected": -2.4722418785095215,
      "loss": 2.1964,
      "nll_loss": 2.1514475345611572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19231019914150238,
      "rewards/margins": 0.054913993924856186,
      "rewards/rejected": -0.24722418189048767,
      "step": 84
    },
    {
      "epoch": 0.13396375098502758,
      "grad_norm": 0.6888207197189331,
      "learning_rate": 2.645669291338583e-06,
      "log_odds_chosen": 0.5399656295776367,
      "log_odds_ratio": -0.4604187309741974,
      "logits/chosen": -0.813186526298523,
      "logits/rejected": -0.4043978750705719,
      "logps/chosen": -1.9997724294662476,
      "logps/rejected": -2.4816389083862305,
      "loss": 2.2291,
      "nll_loss": 2.1830945014953613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19997724890708923,
      "rewards/margins": 0.04818664863705635,
      "rewards/rejected": -0.24816389381885529,
      "step": 85
    },
    {
      "epoch": 0.1355397951142632,
      "grad_norm": 0.7315590381622314,
      "learning_rate": 2.6771653543307086e-06,
      "log_odds_chosen": 0.5595324039459229,
      "log_odds_ratio": -0.45697805285453796,
      "logits/chosen": -1.0269588232040405,
      "logits/rejected": -0.3898090124130249,
      "logps/chosen": -1.973376989364624,
      "logps/rejected": -2.4706506729125977,
      "loss": 2.216,
      "nll_loss": 2.1702566146850586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19733770191669464,
      "rewards/margins": 0.04972738027572632,
      "rewards/rejected": -0.24706508219242096,
      "step": 86
    },
    {
      "epoch": 0.13711583924349882,
      "grad_norm": 0.672721803188324,
      "learning_rate": 2.7086614173228344e-06,
      "log_odds_chosen": 0.38126063346862793,
      "log_odds_ratio": -0.5237147212028503,
      "logits/chosen": -0.8164312243461609,
      "logits/rejected": -0.5732689499855042,
      "logps/chosen": -1.9951059818267822,
      "logps/rejected": -2.3318116664886475,
      "loss": 2.2279,
      "nll_loss": 2.175508737564087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1995105892419815,
      "rewards/margins": 0.033670589327812195,
      "rewards/rejected": -0.2331811785697937,
      "step": 87
    },
    {
      "epoch": 0.13869188337273444,
      "grad_norm": 0.7415196895599365,
      "learning_rate": 2.7401574803149607e-06,
      "log_odds_chosen": 0.6299477815628052,
      "log_odds_ratio": -0.44362398982048035,
      "logits/chosen": -1.0215085744857788,
      "logits/rejected": -0.34737205505371094,
      "logps/chosen": -1.8929115533828735,
      "logps/rejected": -2.453705310821533,
      "loss": 2.1671,
      "nll_loss": 2.1227200031280518,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18929119408130646,
      "rewards/margins": 0.056079376488924026,
      "rewards/rejected": -0.2453705370426178,
      "step": 88
    },
    {
      "epoch": 0.14026792750197006,
      "grad_norm": 0.7291280627250671,
      "learning_rate": 2.7716535433070865e-06,
      "log_odds_chosen": 0.5796247124671936,
      "log_odds_ratio": -0.4462001621723175,
      "logits/chosen": -0.8788402676582336,
      "logits/rejected": -0.38318932056427,
      "logps/chosen": -1.9982614517211914,
      "logps/rejected": -2.5162978172302246,
      "loss": 2.2492,
      "nll_loss": 2.2046027183532715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19982615113258362,
      "rewards/margins": 0.05180366337299347,
      "rewards/rejected": -0.2516297996044159,
      "step": 89
    },
    {
      "epoch": 0.14184397163120568,
      "grad_norm": 0.7039697170257568,
      "learning_rate": 2.8031496062992123e-06,
      "log_odds_chosen": 0.4399077296257019,
      "log_odds_ratio": -0.5028520822525024,
      "logits/chosen": -0.80506831407547,
      "logits/rejected": -0.3771530091762543,
      "logps/chosen": -2.065699577331543,
      "logps/rejected": -2.4619250297546387,
      "loss": 2.2826,
      "nll_loss": 2.232358932495117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20656995475292206,
      "rewards/margins": 0.03962254151701927,
      "rewards/rejected": -0.24619249999523163,
      "step": 90
    },
    {
      "epoch": 0.1434200157604413,
      "grad_norm": 0.6275699138641357,
      "learning_rate": 2.834645669291338e-06,
      "log_odds_chosen": 0.49364161491394043,
      "log_odds_ratio": -0.48472219705581665,
      "logits/chosen": -0.8276112675666809,
      "logits/rejected": -0.5154532790184021,
      "logps/chosen": -1.857814908027649,
      "logps/rejected": -2.2916698455810547,
      "loss": 2.0939,
      "nll_loss": 2.0454044342041016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18578149378299713,
      "rewards/margins": 0.04338550567626953,
      "rewards/rejected": -0.22916698455810547,
      "step": 91
    },
    {
      "epoch": 0.1449960598896769,
      "grad_norm": 0.6308448314666748,
      "learning_rate": 2.8661417322834644e-06,
      "log_odds_chosen": 0.5374231934547424,
      "log_odds_ratio": -0.4650093913078308,
      "logits/chosen": -0.8458312153816223,
      "logits/rejected": -0.35253289341926575,
      "logps/chosen": -1.8836688995361328,
      "logps/rejected": -2.357243776321411,
      "loss": 2.136,
      "nll_loss": 2.0894808769226074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18836688995361328,
      "rewards/margins": 0.04735749959945679,
      "rewards/rejected": -0.23572437465190887,
      "step": 92
    },
    {
      "epoch": 0.14657210401891252,
      "grad_norm": 0.7316494584083557,
      "learning_rate": 2.8976377952755906e-06,
      "log_odds_chosen": 0.5097925662994385,
      "log_odds_ratio": -0.49444442987442017,
      "logits/chosen": -0.8357096910476685,
      "logits/rejected": -0.41637447476387024,
      "logps/chosen": -1.9938026666641235,
      "logps/rejected": -2.4533045291900635,
      "loss": 2.2321,
      "nll_loss": 2.1826608180999756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1993802785873413,
      "rewards/margins": 0.045950211584568024,
      "rewards/rejected": -0.24533049762248993,
      "step": 93
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 0.639308750629425,
      "learning_rate": 2.9291338582677165e-06,
      "log_odds_chosen": 0.49024736881256104,
      "log_odds_ratio": -0.4883335828781128,
      "logits/chosen": -0.8006777763366699,
      "logits/rejected": -0.49406468868255615,
      "logps/chosen": -1.9806185960769653,
      "logps/rejected": -2.414760112762451,
      "loss": 2.2086,
      "nll_loss": 2.1598000526428223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19806188344955444,
      "rewards/margins": 0.043414145708084106,
      "rewards/rejected": -0.24147599935531616,
      "step": 94
    },
    {
      "epoch": 0.14972419227738376,
      "grad_norm": 0.7330454587936401,
      "learning_rate": 2.9606299212598423e-06,
      "log_odds_chosen": 0.6175916790962219,
      "log_odds_ratio": -0.44003960490226746,
      "logits/chosen": -0.8804186582565308,
      "logits/rejected": -0.3671495020389557,
      "logps/chosen": -1.9839096069335938,
      "logps/rejected": -2.5373644828796387,
      "loss": 2.225,
      "nll_loss": 2.1809871196746826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19839096069335938,
      "rewards/margins": 0.05534551292657852,
      "rewards/rejected": -0.2537364363670349,
      "step": 95
    },
    {
      "epoch": 0.15130023640661938,
      "grad_norm": 0.6999992728233337,
      "learning_rate": 2.9921259842519685e-06,
      "log_odds_chosen": 0.5814043283462524,
      "log_odds_ratio": -0.44918328523635864,
      "logits/chosen": -0.8479225635528564,
      "logits/rejected": -0.4675160050392151,
      "logps/chosen": -1.991041660308838,
      "logps/rejected": -2.510003089904785,
      "loss": 2.2023,
      "nll_loss": 2.1574201583862305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1991041898727417,
      "rewards/margins": 0.051896125078201294,
      "rewards/rejected": -0.2510002851486206,
      "step": 96
    },
    {
      "epoch": 0.152876280535855,
      "grad_norm": 0.6133814454078674,
      "learning_rate": 3.0236220472440943e-06,
      "log_odds_chosen": 0.4028412103652954,
      "log_odds_ratio": -0.5129313468933105,
      "logits/chosen": -0.8440722227096558,
      "logits/rejected": -0.4072326421737671,
      "logps/chosen": -1.9427242279052734,
      "logps/rejected": -2.2954676151275635,
      "loss": 2.1638,
      "nll_loss": 2.112459897994995,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19427242875099182,
      "rewards/margins": 0.03527434170246124,
      "rewards/rejected": -0.22954675555229187,
      "step": 97
    },
    {
      "epoch": 0.15445232466509062,
      "grad_norm": 0.651321291923523,
      "learning_rate": 3.05511811023622e-06,
      "log_odds_chosen": 0.42689526081085205,
      "log_odds_ratio": -0.5087873935699463,
      "logits/chosen": -0.7033334374427795,
      "logits/rejected": -0.442612886428833,
      "logps/chosen": -2.056950092315674,
      "logps/rejected": -2.4395601749420166,
      "loss": 2.2954,
      "nll_loss": 2.2444937229156494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2056950181722641,
      "rewards/margins": 0.038261011242866516,
      "rewards/rejected": -0.24395602941513062,
      "step": 98
    },
    {
      "epoch": 0.15602836879432624,
      "grad_norm": 0.7059087157249451,
      "learning_rate": 3.086614173228346e-06,
      "log_odds_chosen": 0.29518887400627136,
      "log_odds_ratio": -0.5591641068458557,
      "logits/chosen": -0.9677470922470093,
      "logits/rejected": -0.5977285504341125,
      "logps/chosen": -1.9788846969604492,
      "logps/rejected": -2.2380669116973877,
      "loss": 2.2282,
      "nll_loss": 2.1722562313079834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19788846373558044,
      "rewards/margins": 0.025918252766132355,
      "rewards/rejected": -0.2238067388534546,
      "step": 99
    },
    {
      "epoch": 0.15760441292356187,
      "grad_norm": 0.6437969207763672,
      "learning_rate": 3.1181102362204722e-06,
      "log_odds_chosen": 0.41778114438056946,
      "log_odds_ratio": -0.5134991407394409,
      "logits/chosen": -0.8367648124694824,
      "logits/rejected": -0.4996865689754486,
      "logps/chosen": -1.8884031772613525,
      "logps/rejected": -2.258099317550659,
      "loss": 2.1384,
      "nll_loss": 2.0870912075042725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18884031474590302,
      "rewards/margins": 0.036969613283872604,
      "rewards/rejected": -0.2258099466562271,
      "step": 100
    },
    {
      "epoch": 0.15918045705279749,
      "grad_norm": 0.5721230506896973,
      "learning_rate": 3.1496062992125985e-06,
      "log_odds_chosen": 0.5020022392272949,
      "log_odds_ratio": -0.4791727066040039,
      "logits/chosen": -0.7251911163330078,
      "logits/rejected": -0.40484821796417236,
      "logps/chosen": -1.8599562644958496,
      "logps/rejected": -2.3015549182891846,
      "loss": 2.1179,
      "nll_loss": 2.069948673248291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18599562346935272,
      "rewards/margins": 0.044159889221191406,
      "rewards/rejected": -0.23015552759170532,
      "step": 101
    },
    {
      "epoch": 0.1607565011820331,
      "grad_norm": 0.5419020652770996,
      "learning_rate": 3.1811023622047243e-06,
      "log_odds_chosen": 0.4814803898334503,
      "log_odds_ratio": -0.49633079767227173,
      "logits/chosen": -0.7592243552207947,
      "logits/rejected": -0.513361930847168,
      "logps/chosen": -1.8242498636245728,
      "logps/rejected": -2.2473723888397217,
      "loss": 2.0742,
      "nll_loss": 2.024540662765503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18242499232292175,
      "rewards/margins": 0.04231225699186325,
      "rewards/rejected": -0.2247372418642044,
      "step": 102
    },
    {
      "epoch": 0.16233254531126873,
      "grad_norm": 0.53934645652771,
      "learning_rate": 3.2125984251968505e-06,
      "log_odds_chosen": 0.5330245494842529,
      "log_odds_ratio": -0.4746904671192169,
      "logits/chosen": -0.6480450630187988,
      "logits/rejected": -0.27741724252700806,
      "logps/chosen": -1.8202755451202393,
      "logps/rejected": -2.283982992172241,
      "loss": 2.0747,
      "nll_loss": 2.0272209644317627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18202754855155945,
      "rewards/margins": 0.046370729804039,
      "rewards/rejected": -0.22839829325675964,
      "step": 103
    },
    {
      "epoch": 0.16390858944050432,
      "grad_norm": 0.6298537254333496,
      "learning_rate": 3.2440944881889763e-06,
      "log_odds_chosen": 0.5380266904830933,
      "log_odds_ratio": -0.4674755930900574,
      "logits/chosen": -0.6194661855697632,
      "logits/rejected": -0.28310033679008484,
      "logps/chosen": -1.9528425931930542,
      "logps/rejected": -2.4316210746765137,
      "loss": 2.1984,
      "nll_loss": 2.1516404151916504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19528424739837646,
      "rewards/margins": 0.047877877950668335,
      "rewards/rejected": -0.2431621104478836,
      "step": 104
    },
    {
      "epoch": 0.16548463356973994,
      "grad_norm": 0.5288484692573547,
      "learning_rate": 3.275590551181102e-06,
      "log_odds_chosen": 0.5468251705169678,
      "log_odds_ratio": -0.4628355801105499,
      "logits/chosen": -0.7265529036521912,
      "logits/rejected": -0.3702164590358734,
      "logps/chosen": -1.8432681560516357,
      "logps/rejected": -2.3218398094177246,
      "loss": 2.1017,
      "nll_loss": 2.0554370880126953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18432679772377014,
      "rewards/margins": 0.04785720631480217,
      "rewards/rejected": -0.2321840077638626,
      "step": 105
    },
    {
      "epoch": 0.16706067769897556,
      "grad_norm": 0.5358848571777344,
      "learning_rate": 3.307086614173228e-06,
      "log_odds_chosen": 0.4864036440849304,
      "log_odds_ratio": -0.4927322268486023,
      "logits/chosen": -0.6903339624404907,
      "logits/rejected": -0.27819010615348816,
      "logps/chosen": -1.8541500568389893,
      "logps/rejected": -2.2708048820495605,
      "loss": 2.0971,
      "nll_loss": 2.0478692054748535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18541499972343445,
      "rewards/margins": 0.04166547954082489,
      "rewards/rejected": -0.22708047926425934,
      "step": 106
    },
    {
      "epoch": 0.16863672182821118,
      "grad_norm": 0.5739277601242065,
      "learning_rate": 3.3385826771653542e-06,
      "log_odds_chosen": 0.48060721158981323,
      "log_odds_ratio": -0.4924314022064209,
      "logits/chosen": -0.6585574746131897,
      "logits/rejected": -0.4523526728153229,
      "logps/chosen": -1.9171538352966309,
      "logps/rejected": -2.3419744968414307,
      "loss": 2.1566,
      "nll_loss": 2.1073999404907227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19171538949012756,
      "rewards/margins": 0.04248207062482834,
      "rewards/rejected": -0.2341974675655365,
      "step": 107
    },
    {
      "epoch": 0.1702127659574468,
      "grad_norm": 0.511557936668396,
      "learning_rate": 3.37007874015748e-06,
      "log_odds_chosen": 0.365371972322464,
      "log_odds_ratio": -0.5382389426231384,
      "logits/chosen": -0.605251669883728,
      "logits/rejected": -0.4923926591873169,
      "logps/chosen": -1.8904098272323608,
      "logps/rejected": -2.2083511352539062,
      "loss": 2.132,
      "nll_loss": 2.0781614780426025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18904098868370056,
      "rewards/margins": 0.03179413452744484,
      "rewards/rejected": -0.2208351194858551,
      "step": 108
    },
    {
      "epoch": 0.17178881008668243,
      "grad_norm": 0.5645051002502441,
      "learning_rate": 3.4015748031496063e-06,
      "log_odds_chosen": 0.5187886953353882,
      "log_odds_ratio": -0.47012859582901,
      "logits/chosen": -0.5434896945953369,
      "logits/rejected": -0.47223836183547974,
      "logps/chosen": -1.965927004814148,
      "logps/rejected": -2.424842596054077,
      "loss": 2.1994,
      "nll_loss": 2.152377128601074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19659270346164703,
      "rewards/margins": 0.04589153826236725,
      "rewards/rejected": -0.24248425662517548,
      "step": 109
    },
    {
      "epoch": 0.17336485421591805,
      "grad_norm": 0.5361037254333496,
      "learning_rate": 3.433070866141732e-06,
      "log_odds_chosen": 0.5615408420562744,
      "log_odds_ratio": -0.4578676223754883,
      "logits/chosen": -0.6213514804840088,
      "logits/rejected": -0.4045155346393585,
      "logps/chosen": -1.8413625955581665,
      "logps/rejected": -2.3273088932037354,
      "loss": 2.0554,
      "nll_loss": 2.009617805480957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1841362714767456,
      "rewards/margins": 0.04859462380409241,
      "rewards/rejected": -0.23273088037967682,
      "step": 110
    },
    {
      "epoch": 0.17494089834515367,
      "grad_norm": 0.591759443283081,
      "learning_rate": 3.4645669291338583e-06,
      "log_odds_chosen": 0.5804085731506348,
      "log_odds_ratio": -0.4556342363357544,
      "logits/chosen": -0.6123775839805603,
      "logits/rejected": -0.4591195583343506,
      "logps/chosen": -1.9501187801361084,
      "logps/rejected": -2.4626777172088623,
      "loss": 2.1912,
      "nll_loss": 2.145684242248535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19501186907291412,
      "rewards/margins": 0.051255881786346436,
      "rewards/rejected": -0.24626776576042175,
      "step": 111
    },
    {
      "epoch": 0.1765169424743893,
      "grad_norm": 0.5324497222900391,
      "learning_rate": 3.496062992125984e-06,
      "log_odds_chosen": 0.4026699662208557,
      "log_odds_ratio": -0.5162889957427979,
      "logits/chosen": -0.5372949838638306,
      "logits/rejected": -0.2981308400630951,
      "logps/chosen": -1.9884939193725586,
      "logps/rejected": -2.3435468673706055,
      "loss": 2.2198,
      "nll_loss": 2.16818904876709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1988493949174881,
      "rewards/margins": 0.03550529479980469,
      "rewards/rejected": -0.23435468971729279,
      "step": 112
    },
    {
      "epoch": 0.1780929866036249,
      "grad_norm": 0.553754448890686,
      "learning_rate": 3.52755905511811e-06,
      "log_odds_chosen": 0.40890055894851685,
      "log_odds_ratio": -0.5179369449615479,
      "logits/chosen": -0.7111020088195801,
      "logits/rejected": -0.3697856664657593,
      "logps/chosen": -1.8428016901016235,
      "logps/rejected": -2.200305700302124,
      "loss": 2.0924,
      "nll_loss": 2.040616512298584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1842801570892334,
      "rewards/margins": 0.03575042262673378,
      "rewards/rejected": -0.22003059089183807,
      "step": 113
    },
    {
      "epoch": 0.17966903073286053,
      "grad_norm": 0.5356653332710266,
      "learning_rate": 3.559055118110236e-06,
      "log_odds_chosen": 0.3291313946247101,
      "log_odds_ratio": -0.5585582852363586,
      "logits/chosen": -0.4512363076210022,
      "logits/rejected": -0.43510857224464417,
      "logps/chosen": -1.9846851825714111,
      "logps/rejected": -2.2790331840515137,
      "loss": 2.2259,
      "nll_loss": 2.1699955463409424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19846852123737335,
      "rewards/margins": 0.02943480759859085,
      "rewards/rejected": -0.2279033213853836,
      "step": 114
    },
    {
      "epoch": 0.18124507486209615,
      "grad_norm": 0.5100815296173096,
      "learning_rate": 3.590551181102362e-06,
      "log_odds_chosen": 0.49927371740341187,
      "log_odds_ratio": -0.48482194542884827,
      "logits/chosen": -0.5574390888214111,
      "logits/rejected": -0.5628021955490112,
      "logps/chosen": -1.8782962560653687,
      "logps/rejected": -2.317220687866211,
      "loss": 2.11,
      "nll_loss": 2.0615200996398926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1878296285867691,
      "rewards/margins": 0.04389244690537453,
      "rewards/rejected": -0.23172207176685333,
      "step": 115
    },
    {
      "epoch": 0.18282111899133174,
      "grad_norm": 0.5005074739456177,
      "learning_rate": 3.622047244094488e-06,
      "log_odds_chosen": 0.39984914660453796,
      "log_odds_ratio": -0.5320521593093872,
      "logits/chosen": -0.5708422660827637,
      "logits/rejected": -0.3728347420692444,
      "logps/chosen": -1.935167670249939,
      "logps/rejected": -2.2881405353546143,
      "loss": 2.1566,
      "nll_loss": 2.103365898132324,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19351676106452942,
      "rewards/margins": 0.035297293215990067,
      "rewards/rejected": -0.2288140505552292,
      "step": 116
    },
    {
      "epoch": 0.18439716312056736,
      "grad_norm": 0.5071951746940613,
      "learning_rate": 3.653543307086614e-06,
      "log_odds_chosen": 0.5789820551872253,
      "log_odds_ratio": -0.46050727367401123,
      "logits/chosen": -0.55865877866745,
      "logits/rejected": -0.624116837978363,
      "logps/chosen": -1.8576948642730713,
      "logps/rejected": -2.3663711547851562,
      "loss": 2.1024,
      "nll_loss": 2.0563366413116455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18576949834823608,
      "rewards/margins": 0.05086761713027954,
      "rewards/rejected": -0.23663711547851562,
      "step": 117
    },
    {
      "epoch": 0.18597320724980299,
      "grad_norm": 0.5110164880752563,
      "learning_rate": 3.68503937007874e-06,
      "log_odds_chosen": 0.3938359022140503,
      "log_odds_ratio": -0.5259670615196228,
      "logits/chosen": -0.4433292746543884,
      "logits/rejected": -0.4191721975803375,
      "logps/chosen": -1.8846337795257568,
      "logps/rejected": -2.2300126552581787,
      "loss": 2.1262,
      "nll_loss": 2.0735833644866943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18846337497234344,
      "rewards/margins": 0.03453788906335831,
      "rewards/rejected": -0.22300127148628235,
      "step": 118
    },
    {
      "epoch": 0.1875492513790386,
      "grad_norm": 0.5024784803390503,
      "learning_rate": 3.716535433070866e-06,
      "log_odds_chosen": 0.33231812715530396,
      "log_odds_ratio": -0.5509342551231384,
      "logits/chosen": -0.4613322913646698,
      "logits/rejected": -0.3955138325691223,
      "logps/chosen": -1.8655431270599365,
      "logps/rejected": -2.155794858932495,
      "loss": 2.1247,
      "nll_loss": 2.0695743560791016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18655429780483246,
      "rewards/margins": 0.029025178402662277,
      "rewards/rejected": -0.21557949483394623,
      "step": 119
    },
    {
      "epoch": 0.18912529550827423,
      "grad_norm": 0.47591114044189453,
      "learning_rate": 3.748031496062992e-06,
      "log_odds_chosen": 0.6754725575447083,
      "log_odds_ratio": -0.42014336585998535,
      "logits/chosen": -0.5019458532333374,
      "logits/rejected": -0.39064449071884155,
      "logps/chosen": -1.7608760595321655,
      "logps/rejected": -2.3479392528533936,
      "loss": 1.9898,
      "nll_loss": 1.9478257894515991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1760876327753067,
      "rewards/margins": 0.05870632082223892,
      "rewards/rejected": -0.23479396104812622,
      "step": 120
    },
    {
      "epoch": 0.19070133963750985,
      "grad_norm": 0.4543094336986542,
      "learning_rate": 3.779527559055118e-06,
      "log_odds_chosen": 0.6511515378952026,
      "log_odds_ratio": -0.42243996262550354,
      "logits/chosen": -0.5351437926292419,
      "logits/rejected": -0.5634927153587341,
      "logps/chosen": -1.8719269037246704,
      "logps/rejected": -2.4463436603546143,
      "loss": 2.0912,
      "nll_loss": 2.0489981174468994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18719267845153809,
      "rewards/margins": 0.057441674172878265,
      "rewards/rejected": -0.24463436007499695,
      "step": 121
    },
    {
      "epoch": 0.19227738376674547,
      "grad_norm": 0.4782586395740509,
      "learning_rate": 3.8110236220472436e-06,
      "log_odds_chosen": 0.5679782032966614,
      "log_odds_ratio": -0.46660980582237244,
      "logits/chosen": -0.582790195941925,
      "logits/rejected": -0.4242536127567291,
      "logps/chosen": -1.790369987487793,
      "logps/rejected": -2.2882192134857178,
      "loss": 2.0132,
      "nll_loss": 1.9665888547897339,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17903698980808258,
      "rewards/margins": 0.04978492856025696,
      "rewards/rejected": -0.22882193326950073,
      "step": 122
    },
    {
      "epoch": 0.1938534278959811,
      "grad_norm": 0.5209844708442688,
      "learning_rate": 3.8425196850393695e-06,
      "log_odds_chosen": 0.43676841259002686,
      "log_odds_ratio": -0.5001434087753296,
      "logits/chosen": -0.41858866810798645,
      "logits/rejected": -0.46535009145736694,
      "logps/chosen": -1.8954147100448608,
      "logps/rejected": -2.27744197845459,
      "loss": 2.1337,
      "nll_loss": 2.0836901664733887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18954147398471832,
      "rewards/margins": 0.038202736526727676,
      "rewards/rejected": -0.2277442067861557,
      "step": 123
    },
    {
      "epoch": 0.1954294720252167,
      "grad_norm": 0.43751877546310425,
      "learning_rate": 3.874015748031496e-06,
      "log_odds_chosen": 0.5021814703941345,
      "log_odds_ratio": -0.48468679189682007,
      "logits/chosen": -0.5214758515357971,
      "logits/rejected": -0.42654621601104736,
      "logps/chosen": -1.7678717374801636,
      "logps/rejected": -2.193263292312622,
      "loss": 1.9906,
      "nll_loss": 1.9421806335449219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17678718268871307,
      "rewards/margins": 0.04253914952278137,
      "rewards/rejected": -0.21932633221149445,
      "step": 124
    },
    {
      "epoch": 0.19700551615445233,
      "grad_norm": 0.46247461438179016,
      "learning_rate": 3.905511811023622e-06,
      "log_odds_chosen": 0.4802154302597046,
      "log_odds_ratio": -0.48964813351631165,
      "logits/chosen": -0.49467337131500244,
      "logits/rejected": -0.5816658735275269,
      "logps/chosen": -1.8051879405975342,
      "logps/rejected": -2.221101760864258,
      "loss": 2.0493,
      "nll_loss": 2.0002968311309814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18051880598068237,
      "rewards/margins": 0.04159136861562729,
      "rewards/rejected": -0.22211016714572906,
      "step": 125
    },
    {
      "epoch": 0.19858156028368795,
      "grad_norm": 0.5007897615432739,
      "learning_rate": 3.937007874015748e-06,
      "log_odds_chosen": 0.6812261939048767,
      "log_odds_ratio": -0.4165576696395874,
      "logits/chosen": -0.5011131167411804,
      "logits/rejected": -0.3981563448905945,
      "logps/chosen": -1.824156403541565,
      "logps/rejected": -2.42551589012146,
      "loss": 2.0473,
      "nll_loss": 2.0055999755859375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1824156492948532,
      "rewards/margins": 0.06013592332601547,
      "rewards/rejected": -0.24255158007144928,
      "step": 126
    },
    {
      "epoch": 0.20015760441292357,
      "grad_norm": 0.5308762192726135,
      "learning_rate": 3.9685039370078736e-06,
      "log_odds_chosen": 0.5868005752563477,
      "log_odds_ratio": -0.4556260108947754,
      "logits/chosen": -0.47778087854385376,
      "logits/rejected": -0.39855217933654785,
      "logps/chosen": -1.8529423475265503,
      "logps/rejected": -2.371786594390869,
      "loss": 2.1134,
      "nll_loss": 2.0678582191467285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1852942407131195,
      "rewards/margins": 0.05188443511724472,
      "rewards/rejected": -0.23717869818210602,
      "step": 127
    },
    {
      "epoch": 0.2017336485421592,
      "grad_norm": 0.4748990535736084,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.36945709586143494,
      "log_odds_ratio": -0.529854953289032,
      "logits/chosen": -0.31703251600265503,
      "logits/rejected": -0.261596143245697,
      "logps/chosen": -1.9309293031692505,
      "logps/rejected": -2.2549469470977783,
      "loss": 2.1479,
      "nll_loss": 2.094945192337036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1930929273366928,
      "rewards/margins": 0.032401762902736664,
      "rewards/rejected": -0.22549471259117126,
      "step": 128
    },
    {
      "epoch": 0.2033096926713948,
      "grad_norm": 0.46808141469955444,
      "learning_rate": 3.999992445477635e-06,
      "log_odds_chosen": 0.4924158751964569,
      "log_odds_ratio": -0.4864352345466614,
      "logits/chosen": -0.3923236131668091,
      "logits/rejected": -0.5614160299301147,
      "logps/chosen": -1.926079273223877,
      "logps/rejected": -2.361663579940796,
      "loss": 2.1291,
      "nll_loss": 2.080496311187744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19260793924331665,
      "rewards/margins": 0.04355840012431145,
      "rewards/rejected": -0.2361663281917572,
      "step": 129
    },
    {
      "epoch": 0.2048857368006304,
      "grad_norm": 0.4758654534816742,
      "learning_rate": 3.999969781967615e-06,
      "log_odds_chosen": 0.3168904185295105,
      "log_odds_ratio": -0.5500271916389465,
      "logits/chosen": -0.3438476622104645,
      "logits/rejected": -0.36578765511512756,
      "logps/chosen": -2.0286383628845215,
      "logps/rejected": -2.31013560295105,
      "loss": 2.2287,
      "nll_loss": 2.1736607551574707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20286381244659424,
      "rewards/margins": 0.028149710968136787,
      "rewards/rejected": -0.23101355135440826,
      "step": 130
    },
    {
      "epoch": 0.20646178092986603,
      "grad_norm": 0.4352067708969116,
      "learning_rate": 3.99993200964115e-06,
      "log_odds_chosen": 0.41043001413345337,
      "log_odds_ratio": -0.5113096237182617,
      "logits/chosen": -0.4144492745399475,
      "logits/rejected": -0.5521720051765442,
      "logps/chosen": -1.8470234870910645,
      "logps/rejected": -2.2023518085479736,
      "loss": 2.0658,
      "nll_loss": 2.014646053314209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18470235168933868,
      "rewards/margins": 0.03553282842040062,
      "rewards/rejected": -0.2202351987361908,
      "step": 131
    },
    {
      "epoch": 0.20803782505910165,
      "grad_norm": 0.5278255939483643,
      "learning_rate": 3.99987912878359e-06,
      "log_odds_chosen": 0.21506069600582123,
      "log_odds_ratio": -0.5979167222976685,
      "logits/chosen": -0.43242937326431274,
      "logits/rejected": -0.22841165959835052,
      "logps/chosen": -1.8215473890304565,
      "logps/rejected": -2.0002083778381348,
      "loss": 2.0893,
      "nll_loss": 2.029538631439209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18215471506118774,
      "rewards/margins": 0.017866121605038643,
      "rewards/rejected": -0.20002083480358124,
      "step": 132
    },
    {
      "epoch": 0.20961386918833727,
      "grad_norm": 0.44630110263824463,
      "learning_rate": 3.999811139794429e-06,
      "log_odds_chosen": 0.4566144049167633,
      "log_odds_ratio": -0.4949381947517395,
      "logits/chosen": -0.38445478677749634,
      "logits/rejected": -0.503831148147583,
      "logps/chosen": -1.8512368202209473,
      "logps/rejected": -2.2488574981689453,
      "loss": 2.0611,
      "nll_loss": 2.0115725994110107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18512369692325592,
      "rewards/margins": 0.039762042462825775,
      "rewards/rejected": -0.2248857468366623,
      "step": 133
    },
    {
      "epoch": 0.2111899133175729,
      "grad_norm": 0.47832149267196655,
      "learning_rate": 3.999728043187288e-06,
      "log_odds_chosen": 0.4043869972229004,
      "log_odds_ratio": -0.5232419967651367,
      "logits/chosen": -0.3849250078201294,
      "logits/rejected": -0.4331829845905304,
      "logps/chosen": -1.916317105293274,
      "logps/rejected": -2.273916721343994,
      "loss": 2.1495,
      "nll_loss": 2.0971601009368896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1916317194700241,
      "rewards/margins": 0.03575997054576874,
      "rewards/rejected": -0.22739167511463165,
      "step": 134
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 0.4719514548778534,
      "learning_rate": 3.999629839589922e-06,
      "log_odds_chosen": 0.2638218402862549,
      "log_odds_ratio": -0.5730746388435364,
      "logits/chosen": -0.2890802025794983,
      "logits/rejected": -0.6256790161132812,
      "logps/chosen": -1.9855725765228271,
      "logps/rejected": -2.2172610759735107,
      "loss": 2.2169,
      "nll_loss": 2.1596412658691406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19855724275112152,
      "rewards/margins": 0.02316887117922306,
      "rewards/rejected": -0.22172610461711884,
      "step": 135
    },
    {
      "epoch": 0.21434200157604413,
      "grad_norm": 0.4406312108039856,
      "learning_rate": 3.999516529744215e-06,
      "log_odds_chosen": 0.38678330183029175,
      "log_odds_ratio": -0.5220622420310974,
      "logits/chosen": -0.22331157326698303,
      "logits/rejected": -0.5899175405502319,
      "logps/chosen": -1.8535821437835693,
      "logps/rejected": -2.1891560554504395,
      "loss": 2.0776,
      "nll_loss": 2.0253748893737793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1853582262992859,
      "rewards/margins": 0.033557381480932236,
      "rewards/rejected": -0.21891558170318604,
      "step": 136
    },
    {
      "epoch": 0.21591804570527975,
      "grad_norm": 0.4380626380443573,
      "learning_rate": 3.999388114506166e-06,
      "log_odds_chosen": 0.4394814074039459,
      "log_odds_ratio": -0.5038915872573853,
      "logits/chosen": -0.3584860861301422,
      "logits/rejected": -0.6305355429649353,
      "logps/chosen": -1.76337730884552,
      "logps/rejected": -2.14279842376709,
      "loss": 1.9779,
      "nll_loss": 1.9275436401367188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17633774876594543,
      "rewards/margins": 0.03794211149215698,
      "rewards/rejected": -0.21427986025810242,
      "step": 137
    },
    {
      "epoch": 0.21749408983451538,
      "grad_norm": 0.4562850594520569,
      "learning_rate": 3.999244594845892e-06,
      "log_odds_chosen": 0.3186016082763672,
      "log_odds_ratio": -0.5537752509117126,
      "logits/chosen": -0.3234477937221527,
      "logits/rejected": -0.5436648726463318,
      "logps/chosen": -1.8082385063171387,
      "logps/rejected": -2.0849738121032715,
      "loss": 2.0387,
      "nll_loss": 1.9833385944366455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18082384765148163,
      "rewards/margins": 0.027673546224832535,
      "rewards/rejected": -0.20849739015102386,
      "step": 138
    },
    {
      "epoch": 0.219070133963751,
      "grad_norm": 0.4723531901836395,
      "learning_rate": 3.999085971847616e-06,
      "log_odds_chosen": 0.2793772220611572,
      "log_odds_ratio": -0.5707423090934753,
      "logits/chosen": -0.3115157186985016,
      "logits/rejected": -0.6855476498603821,
      "logps/chosen": -1.8707714080810547,
      "logps/rejected": -2.1145730018615723,
      "loss": 2.0908,
      "nll_loss": 2.033709764480591,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18707714974880219,
      "rewards/margins": 0.02438015677034855,
      "rewards/rejected": -0.211457297205925,
      "step": 139
    },
    {
      "epoch": 0.22064617809298662,
      "grad_norm": 0.47324880957603455,
      "learning_rate": 3.998912246709658e-06,
      "log_odds_chosen": 0.43111467361450195,
      "log_odds_ratio": -0.5151071548461914,
      "logits/chosen": -0.3707660734653473,
      "logits/rejected": -0.6392085552215576,
      "logps/chosen": -1.7731448411941528,
      "logps/rejected": -2.146456480026245,
      "loss": 1.9936,
      "nll_loss": 1.942044973373413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17731450498104095,
      "rewards/margins": 0.03733116388320923,
      "rewards/rejected": -0.214645653963089,
      "step": 140
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 0.49398210644721985,
      "learning_rate": 3.9987234207444295e-06,
      "log_odds_chosen": 0.4180254638195038,
      "log_odds_ratio": -0.5135904550552368,
      "logits/chosen": -0.3750038146972656,
      "logits/rejected": -0.5034846067428589,
      "logps/chosen": -1.868428111076355,
      "logps/rejected": -2.2344772815704346,
      "loss": 2.092,
      "nll_loss": 2.0405919551849365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18684281408786774,
      "rewards/margins": 0.03660491853952408,
      "rewards/rejected": -0.2234477400779724,
      "step": 141
    },
    {
      "epoch": 0.22379826635145783,
      "grad_norm": 0.4737972617149353,
      "learning_rate": 3.998519495378419e-06,
      "log_odds_chosen": 0.397806853055954,
      "log_odds_ratio": -0.5191542506217957,
      "logits/chosen": -0.2983032763004303,
      "logits/rejected": -0.4037090539932251,
      "logps/chosen": -1.9413141012191772,
      "logps/rejected": -2.2922468185424805,
      "loss": 2.1382,
      "nll_loss": 2.086291551589966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19413141906261444,
      "rewards/margins": 0.0350932851433754,
      "rewards/rejected": -0.22922469675540924,
      "step": 142
    },
    {
      "epoch": 0.22537431048069345,
      "grad_norm": 0.42424681782722473,
      "learning_rate": 3.998300472152187e-06,
      "log_odds_chosen": 0.37584933638572693,
      "log_odds_ratio": -0.5324170589447021,
      "logits/chosen": -0.3758591413497925,
      "logits/rejected": -0.7293614745140076,
      "logps/chosen": -1.730176568031311,
      "logps/rejected": -2.052025079727173,
      "loss": 1.9556,
      "nll_loss": 1.9023665189743042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17301765084266663,
      "rewards/margins": 0.03218483552336693,
      "rewards/rejected": -0.20520250499248505,
      "step": 143
    },
    {
      "epoch": 0.22695035460992907,
      "grad_norm": 0.46012207865715027,
      "learning_rate": 3.998066352720347e-06,
      "log_odds_chosen": 0.48952746391296387,
      "log_odds_ratio": -0.4826250374317169,
      "logits/chosen": -0.1988898515701294,
      "logits/rejected": -0.5319833755493164,
      "logps/chosen": -1.8468257188796997,
      "logps/rejected": -2.2748098373413086,
      "loss": 2.0578,
      "nll_loss": 2.009584665298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18468256294727325,
      "rewards/margins": 0.04279841482639313,
      "rewards/rejected": -0.22748097777366638,
      "step": 144
    },
    {
      "epoch": 0.2285263987391647,
      "grad_norm": 0.4577552080154419,
      "learning_rate": 3.997817138851562e-06,
      "log_odds_chosen": 0.3373531997203827,
      "log_odds_ratio": -0.5514544248580933,
      "logits/chosen": -0.2713150680065155,
      "logits/rejected": -0.46479517221450806,
      "logps/chosen": -1.9415301084518433,
      "logps/rejected": -2.2407379150390625,
      "loss": 2.1553,
      "nll_loss": 2.100133180618286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1941530406475067,
      "rewards/margins": 0.029920781031250954,
      "rewards/rejected": -0.22407379746437073,
      "step": 145
    },
    {
      "epoch": 0.23010244286840031,
      "grad_norm": 0.4259653389453888,
      "learning_rate": 3.997552832428522e-06,
      "log_odds_chosen": 0.43442001938819885,
      "log_odds_ratio": -0.5027788281440735,
      "logits/chosen": -0.3454759120941162,
      "logits/rejected": -0.5418952703475952,
      "logps/chosen": -1.732759714126587,
      "logps/rejected": -2.1025242805480957,
      "loss": 1.974,
      "nll_loss": 1.9237622022628784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17327596247196198,
      "rewards/margins": 0.03697647154331207,
      "rewards/rejected": -0.21025243401527405,
      "step": 146
    },
    {
      "epoch": 0.23167848699763594,
      "grad_norm": 0.4360640347003937,
      "learning_rate": 3.9972734354479366e-06,
      "log_odds_chosen": 0.39015060663223267,
      "log_odds_ratio": -0.5267472863197327,
      "logits/chosen": -0.2306804656982422,
      "logits/rejected": -0.7212989330291748,
      "logps/chosen": -1.725273847579956,
      "logps/rejected": -2.0561888217926025,
      "loss": 1.9515,
      "nll_loss": 1.8987786769866943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17252738773822784,
      "rewards/margins": 0.03309149667620659,
      "rewards/rejected": -0.20561888813972473,
      "step": 147
    },
    {
      "epoch": 0.23325453112687156,
      "grad_norm": 0.4830004870891571,
      "learning_rate": 3.996978950020517e-06,
      "log_odds_chosen": 0.2517290413379669,
      "log_odds_ratio": -0.589171826839447,
      "logits/chosen": -0.220802903175354,
      "logits/rejected": -0.5540719032287598,
      "logps/chosen": -1.916334629058838,
      "logps/rejected": -2.1323041915893555,
      "loss": 2.1418,
      "nll_loss": 2.082899570465088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1916334629058838,
      "rewards/margins": 0.021596934646368027,
      "rewards/rejected": -0.21323040127754211,
      "step": 148
    },
    {
      "epoch": 0.23483057525610718,
      "grad_norm": 0.4216887950897217,
      "learning_rate": 3.996669378370959e-06,
      "log_odds_chosen": 0.5976248979568481,
      "log_odds_ratio": -0.44520941376686096,
      "logits/chosen": -0.2759765088558197,
      "logits/rejected": -0.5601774454116821,
      "logps/chosen": -1.6960978507995605,
      "logps/rejected": -2.2107391357421875,
      "loss": 1.9157,
      "nll_loss": 1.8712081909179688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16960978507995605,
      "rewards/margins": 0.05146413296461105,
      "rewards/rejected": -0.2210739254951477,
      "step": 149
    },
    {
      "epoch": 0.2364066193853428,
      "grad_norm": 0.4519270658493042,
      "learning_rate": 3.996344722837929e-06,
      "log_odds_chosen": 0.5262175798416138,
      "log_odds_ratio": -0.46979856491088867,
      "logits/chosen": -0.2899366617202759,
      "logits/rejected": -0.48541364073753357,
      "logps/chosen": -1.868219256401062,
      "logps/rejected": -2.3306198120117188,
      "loss": 2.0673,
      "nll_loss": 2.0203018188476562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18682192265987396,
      "rewards/margins": 0.04624004662036896,
      "rewards/rejected": -0.23306196928024292,
      "step": 150
    },
    {
      "epoch": 0.23798266351457842,
      "grad_norm": 0.39388707280158997,
      "learning_rate": 3.996004985874043e-06,
      "log_odds_chosen": 0.2795698940753937,
      "log_odds_ratio": -0.5656891465187073,
      "logits/chosen": -0.16603723168373108,
      "logits/rejected": -0.464844673871994,
      "logps/chosen": -1.7436405420303345,
      "logps/rejected": -1.9796698093414307,
      "loss": 1.9733,
      "nll_loss": 1.9167754650115967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17436404526233673,
      "rewards/margins": 0.023602934554219246,
      "rewards/rejected": -0.19796699285507202,
      "step": 151
    },
    {
      "epoch": 0.23955870764381404,
      "grad_norm": 0.4452219605445862,
      "learning_rate": 3.995650170045855e-06,
      "log_odds_chosen": 0.3562104403972626,
      "log_odds_ratio": -0.5355157256126404,
      "logits/chosen": -0.2569115161895752,
      "logits/rejected": -0.5924301743507385,
      "logps/chosen": -1.8175992965698242,
      "logps/rejected": -2.1236917972564697,
      "loss": 2.0328,
      "nll_loss": 1.9792625904083252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18175993859767914,
      "rewards/margins": 0.030609263107180595,
      "rewards/rejected": -0.21236920356750488,
      "step": 152
    },
    {
      "epoch": 0.24113475177304963,
      "grad_norm": 0.43792277574539185,
      "learning_rate": 3.995280278033825e-06,
      "log_odds_chosen": 0.25159722566604614,
      "log_odds_ratio": -0.5846930146217346,
      "logits/chosen": -0.1688135266304016,
      "logits/rejected": -0.4975782036781311,
      "logps/chosen": -1.919213891029358,
      "logps/rejected": -2.1364455223083496,
      "loss": 2.1351,
      "nll_loss": 2.0765841007232666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1919213980436325,
      "rewards/margins": 0.021723175421357155,
      "rewards/rejected": -0.2136445790529251,
      "step": 153
    },
    {
      "epoch": 0.24271079590228525,
      "grad_norm": 0.4284445345401764,
      "learning_rate": 3.994895312632314e-06,
      "log_odds_chosen": 0.583359956741333,
      "log_odds_ratio": -0.4485571086406708,
      "logits/chosen": -0.14007489383220673,
      "logits/rejected": -0.9004555940628052,
      "logps/chosen": -1.787564754486084,
      "logps/rejected": -2.293922185897827,
      "loss": 1.9995,
      "nll_loss": 1.9546149969100952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1787564754486084,
      "rewards/margins": 0.05063574016094208,
      "rewards/rejected": -0.22939221560955048,
      "step": 154
    },
    {
      "epoch": 0.24428684003152087,
      "grad_norm": 0.4128466546535492,
      "learning_rate": 3.994495276749549e-06,
      "log_odds_chosen": 0.5421090722084045,
      "log_odds_ratio": -0.4647715091705322,
      "logits/chosen": -0.21934741735458374,
      "logits/rejected": -0.5551028251647949,
      "logps/chosen": -1.8062323331832886,
      "logps/rejected": -2.2790470123291016,
      "loss": 2.006,
      "nll_loss": 1.959566593170166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18062324821949005,
      "rewards/margins": 0.047281473875045776,
      "rewards/rejected": -0.22790470719337463,
      "step": 155
    },
    {
      "epoch": 0.2458628841607565,
      "grad_norm": 0.40027129650115967,
      "learning_rate": 3.994080173407612e-06,
      "log_odds_chosen": 0.3059383034706116,
      "log_odds_ratio": -0.5557867884635925,
      "logits/chosen": -0.1562999039888382,
      "logits/rejected": -0.658608615398407,
      "logps/chosen": -1.844901442527771,
      "logps/rejected": -2.1075332164764404,
      "loss": 2.0498,
      "nll_loss": 1.9942554235458374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1844901591539383,
      "rewards/margins": 0.02626318484544754,
      "rewards/rejected": -0.21075333654880524,
      "step": 156
    },
    {
      "epoch": 0.24743892828999212,
      "grad_norm": 0.42715558409690857,
      "learning_rate": 3.993650005742409e-06,
      "log_odds_chosen": 0.3714084029197693,
      "log_odds_ratio": -0.5295048952102661,
      "logits/chosen": -0.26001468300819397,
      "logits/rejected": -0.5350618958473206,
      "logps/chosen": -1.8118646144866943,
      "logps/rejected": -2.1319515705108643,
      "loss": 2.0239,
      "nll_loss": 1.9709175825119019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18118645250797272,
      "rewards/margins": 0.03200869262218475,
      "rewards/rejected": -0.21319516003131866,
      "step": 157
    },
    {
      "epoch": 0.24901497241922774,
      "grad_norm": 0.3839566707611084,
      "learning_rate": 3.993204777003652e-06,
      "log_odds_chosen": 0.393911749124527,
      "log_odds_ratio": -0.5220350623130798,
      "logits/chosen": -0.1995089203119278,
      "logits/rejected": -0.6347463130950928,
      "logps/chosen": -1.7502682209014893,
      "logps/rejected": -2.0890250205993652,
      "loss": 1.9534,
      "nll_loss": 1.9011733531951904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1750268191099167,
      "rewards/margins": 0.033875688910484314,
      "rewards/rejected": -0.208902508020401,
      "step": 158
    },
    {
      "epoch": 0.25059101654846333,
      "grad_norm": 0.4023450016975403,
      "learning_rate": 3.992744490554832e-06,
      "log_odds_chosen": 0.38417020440101624,
      "log_odds_ratio": -0.5247639417648315,
      "logits/chosen": -0.0957825779914856,
      "logits/rejected": -0.511121392250061,
      "logps/chosen": -1.786711573600769,
      "logps/rejected": -2.116995334625244,
      "loss": 2.0068,
      "nll_loss": 1.9543219804763794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17867115139961243,
      "rewards/margins": 0.033028386533260345,
      "rewards/rejected": -0.21169954538345337,
      "step": 159
    },
    {
      "epoch": 0.25216706067769895,
      "grad_norm": 0.37720558047294617,
      "learning_rate": 3.992269149873192e-06,
      "log_odds_chosen": 0.37647151947021484,
      "log_odds_ratio": -0.5282561779022217,
      "logits/chosen": -0.18793515861034393,
      "logits/rejected": -0.6444622874259949,
      "logps/chosen": -1.759155511856079,
      "logps/rejected": -2.0770745277404785,
      "loss": 1.9655,
      "nll_loss": 1.9126847982406616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17591555416584015,
      "rewards/margins": 0.03179190307855606,
      "rewards/rejected": -0.2077074497938156,
      "step": 160
    },
    {
      "epoch": 0.25374310480693457,
      "grad_norm": 0.3518170416355133,
      "learning_rate": 3.991778758549705e-06,
      "log_odds_chosen": 0.2357814759016037,
      "log_odds_ratio": -0.5892237424850464,
      "logits/chosen": -0.039820123463869095,
      "logits/rejected": -0.4496867060661316,
      "logps/chosen": -1.8794186115264893,
      "logps/rejected": -2.0856754779815674,
      "loss": 2.0673,
      "nll_loss": 2.008375644683838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18794189393520355,
      "rewards/margins": 0.020625660195946693,
      "rewards/rejected": -0.2085675299167633,
      "step": 161
    },
    {
      "epoch": 0.2553191489361702,
      "grad_norm": 0.38300883769989014,
      "learning_rate": 3.9912733202890415e-06,
      "log_odds_chosen": 0.381821870803833,
      "log_odds_ratio": -0.5251815319061279,
      "logits/chosen": -0.13694928586483002,
      "logits/rejected": -0.618095338344574,
      "logps/chosen": -1.8078151941299438,
      "logps/rejected": -2.135481834411621,
      "loss": 2.0087,
      "nll_loss": 1.9562112092971802,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1807815134525299,
      "rewards/margins": 0.0327666774392128,
      "rewards/rejected": -0.2135481834411621,
      "step": 162
    },
    {
      "epoch": 0.2568951930654058,
      "grad_norm": 0.36690783500671387,
      "learning_rate": 3.990752838909548e-06,
      "log_odds_chosen": 0.24148699641227722,
      "log_odds_ratio": -0.5847919583320618,
      "logits/chosen": -0.05623656138777733,
      "logits/rejected": -0.5564668774604797,
      "logps/chosen": -1.9099113941192627,
      "logps/rejected": -2.1186718940734863,
      "loss": 2.101,
      "nll_loss": 2.0425591468811035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1909911334514618,
      "rewards/margins": 0.020876042544841766,
      "rewards/rejected": -0.21186719834804535,
      "step": 163
    },
    {
      "epoch": 0.25847123719464143,
      "grad_norm": 0.38993462920188904,
      "learning_rate": 3.990217318343213e-06,
      "log_odds_chosen": 0.5827608108520508,
      "log_odds_ratio": -0.4458136260509491,
      "logits/chosen": -0.12328344583511353,
      "logits/rejected": -0.7965791821479797,
      "logps/chosen": -1.9730701446533203,
      "logps/rejected": -2.4925732612609863,
      "loss": 2.1531,
      "nll_loss": 2.108518123626709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1973070204257965,
      "rewards/margins": 0.05195032060146332,
      "rewards/rejected": -0.24925734102725983,
      "step": 164
    },
    {
      "epoch": 0.26004728132387706,
      "grad_norm": 0.37487825751304626,
      "learning_rate": 3.989666762635637e-06,
      "log_odds_chosen": 0.3498647212982178,
      "log_odds_ratio": -0.5365273356437683,
      "logits/chosen": -0.12066411972045898,
      "logits/rejected": -0.5407615900039673,
      "logps/chosen": -1.8215625286102295,
      "logps/rejected": -2.1220715045928955,
      "loss": 2.0347,
      "nll_loss": 1.9810354709625244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1821562498807907,
      "rewards/margins": 0.03005088120698929,
      "rewards/rejected": -0.2122071385383606,
      "step": 165
    },
    {
      "epoch": 0.2616233254531127,
      "grad_norm": 0.3764452040195465,
      "learning_rate": 3.9891011759460056e-06,
      "log_odds_chosen": 0.43623411655426025,
      "log_odds_ratio": -0.5043083429336548,
      "logits/chosen": -0.1440771371126175,
      "logits/rejected": -0.6117334365844727,
      "logps/chosen": -1.8048644065856934,
      "logps/rejected": -2.175879955291748,
      "loss": 2.0089,
      "nll_loss": 1.9585102796554565,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18048644065856934,
      "rewards/margins": 0.037101540714502335,
      "rewards/rejected": -0.21758797764778137,
      "step": 166
    },
    {
      "epoch": 0.2631993695823483,
      "grad_norm": 0.38358378410339355,
      "learning_rate": 3.988520562547057e-06,
      "log_odds_chosen": 0.4678844213485718,
      "log_odds_ratio": -0.4977923333644867,
      "logits/chosen": -0.09336389601230621,
      "logits/rejected": -0.6688787937164307,
      "logps/chosen": -1.7605222463607788,
      "logps/rejected": -2.160085678100586,
      "loss": 1.9465,
      "nll_loss": 1.8967318534851074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17605219781398773,
      "rewards/margins": 0.03995633125305176,
      "rewards/rejected": -0.21600855886936188,
      "step": 167
    },
    {
      "epoch": 0.2647754137115839,
      "grad_norm": 0.34890928864479065,
      "learning_rate": 3.987924926825047e-06,
      "log_odds_chosen": 0.49037039279937744,
      "log_odds_ratio": -0.4920588731765747,
      "logits/chosen": -0.11033634841442108,
      "logits/rejected": -0.5727885961532593,
      "logps/chosen": -1.8396610021591187,
      "logps/rejected": -2.2677671909332275,
      "loss": 2.0476,
      "nll_loss": 1.9984408617019653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18396610021591187,
      "rewards/margins": 0.04281061887741089,
      "rewards/rejected": -0.22677670419216156,
      "step": 168
    },
    {
      "epoch": 0.26635145784081954,
      "grad_norm": 0.3390553891658783,
      "learning_rate": 3.98731427327972e-06,
      "log_odds_chosen": 0.45071348547935486,
      "log_odds_ratio": -0.493795782327652,
      "logits/chosen": -0.1533864587545395,
      "logits/rejected": -0.8038709163665771,
      "logps/chosen": -1.7674872875213623,
      "logps/rejected": -2.15299916267395,
      "loss": 1.9535,
      "nll_loss": 1.9041638374328613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17674873769283295,
      "rewards/margins": 0.03855118155479431,
      "rewards/rejected": -0.21529991924762726,
      "step": 169
    },
    {
      "epoch": 0.26792750197005516,
      "grad_norm": 0.33687132596969604,
      "learning_rate": 3.986688606524273e-06,
      "log_odds_chosen": 0.3874381482601166,
      "log_odds_ratio": -0.5277330279350281,
      "logits/chosen": -0.053588882088661194,
      "logits/rejected": -0.9633825421333313,
      "logps/chosen": -1.799920678138733,
      "logps/rejected": -2.136354684829712,
      "loss": 1.9903,
      "nll_loss": 1.9375666379928589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17999204993247986,
      "rewards/margins": 0.033643417060375214,
      "rewards/rejected": -0.21363547444343567,
      "step": 170
    },
    {
      "epoch": 0.2695035460992908,
      "grad_norm": 0.3421062231063843,
      "learning_rate": 3.986047931285315e-06,
      "log_odds_chosen": 0.5469809174537659,
      "log_odds_ratio": -0.461223840713501,
      "logits/chosen": -0.13250774145126343,
      "logits/rejected": -0.8443288207054138,
      "logps/chosen": -1.714763879776001,
      "logps/rejected": -2.1810569763183594,
      "loss": 1.9014,
      "nll_loss": 1.8553175926208496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17147637903690338,
      "rewards/margins": 0.046629298478364944,
      "rewards/rejected": -0.21810568869113922,
      "step": 171
    },
    {
      "epoch": 0.2710795902285264,
      "grad_norm": 0.3105310797691345,
      "learning_rate": 3.985392252402847e-06,
      "log_odds_chosen": 0.2694023847579956,
      "log_odds_ratio": -0.5704121589660645,
      "logits/chosen": -0.11545141041278839,
      "logits/rejected": -0.635871171951294,
      "logps/chosen": -1.700569748878479,
      "logps/rejected": -1.9258500337600708,
      "loss": 1.8986,
      "nll_loss": 1.8415968418121338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17005696892738342,
      "rewards/margins": 0.022528033703565598,
      "rewards/rejected": -0.1925850212574005,
      "step": 172
    },
    {
      "epoch": 0.272655634357762,
      "grad_norm": 0.3302405774593353,
      "learning_rate": 3.984721574830206e-06,
      "log_odds_chosen": 0.34335118532180786,
      "log_odds_ratio": -0.5412152409553528,
      "logits/chosen": -0.08807546645402908,
      "logits/rejected": -0.8193111419677734,
      "logps/chosen": -1.8096662759780884,
      "logps/rejected": -2.1047701835632324,
      "loss": 1.9973,
      "nll_loss": 1.9431415796279907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18096663057804108,
      "rewards/margins": 0.029510382562875748,
      "rewards/rejected": -0.21047702431678772,
      "step": 173
    },
    {
      "epoch": 0.27423167848699764,
      "grad_norm": 0.30101102590560913,
      "learning_rate": 3.984035903634041e-06,
      "log_odds_chosen": 0.3278449773788452,
      "log_odds_ratio": -0.549487292766571,
      "logits/chosen": -0.033350471407175064,
      "logits/rejected": -0.8235560655593872,
      "logps/chosen": -1.7809809446334839,
      "logps/rejected": -2.0601303577423096,
      "loss": 1.9562,
      "nll_loss": 1.9012670516967773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17809809744358063,
      "rewards/margins": 0.02791496179997921,
      "rewards/rejected": -0.2060130536556244,
      "step": 174
    },
    {
      "epoch": 0.27580772261623326,
      "grad_norm": 0.313924640417099,
      "learning_rate": 3.983335243994273e-06,
      "log_odds_chosen": 0.44480204582214355,
      "log_odds_ratio": -0.5034283399581909,
      "logits/chosen": -0.015027942135930061,
      "logits/rejected": -0.7614739537239075,
      "logps/chosen": -1.772882342338562,
      "logps/rejected": -2.154425859451294,
      "loss": 1.96,
      "nll_loss": 1.9096373319625854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1772882342338562,
      "rewards/margins": 0.03815435618162155,
      "rewards/rejected": -0.21544259786605835,
      "step": 175
    },
    {
      "epoch": 0.2773837667454689,
      "grad_norm": 0.36805829405784607,
      "learning_rate": 3.982619601204049e-06,
      "log_odds_chosen": 0.37449923157691956,
      "log_odds_ratio": -0.5290423631668091,
      "logits/chosen": -0.09903114289045334,
      "logits/rejected": -0.7904164791107178,
      "logps/chosen": -1.8346233367919922,
      "logps/rejected": -2.158191680908203,
      "loss": 2.0219,
      "nll_loss": 1.9690214395523071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18346232175827026,
      "rewards/margins": 0.03235683590173721,
      "rewards/rejected": -0.21581916511058807,
      "step": 176
    },
    {
      "epoch": 0.2789598108747045,
      "grad_norm": 0.3392786383628845,
      "learning_rate": 3.9818889806697085e-06,
      "log_odds_chosen": 0.43580567836761475,
      "log_odds_ratio": -0.5084404349327087,
      "logits/chosen": 0.015692677348852158,
      "logits/rejected": -0.8096188306808472,
      "logps/chosen": -1.8684344291687012,
      "logps/rejected": -2.249403476715088,
      "loss": 2.0317,
      "nll_loss": 1.9808424711227417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18684343993663788,
      "rewards/margins": 0.038096919655799866,
      "rewards/rejected": -0.22494037449359894,
      "step": 177
    },
    {
      "epoch": 0.2805358550039401,
      "grad_norm": 0.31521204113960266,
      "learning_rate": 3.98114338791074e-06,
      "log_odds_chosen": 0.515329897403717,
      "log_odds_ratio": -0.4845845699310303,
      "logits/chosen": -0.09881779551506042,
      "logits/rejected": -0.6956612467765808,
      "logps/chosen": -1.74465811252594,
      "logps/rejected": -2.1908679008483887,
      "loss": 1.9357,
      "nll_loss": 1.8872735500335693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1744658201932907,
      "rewards/margins": 0.04462098330259323,
      "rewards/rejected": -0.21908681094646454,
      "step": 178
    },
    {
      "epoch": 0.28211189913317575,
      "grad_norm": 0.30939748883247375,
      "learning_rate": 3.980382828559742e-06,
      "log_odds_chosen": 0.42107006907463074,
      "log_odds_ratio": -0.510704755783081,
      "logits/chosen": -0.11055820435285568,
      "logits/rejected": -0.8583495616912842,
      "logps/chosen": -1.78211510181427,
      "logps/rejected": -2.143000841140747,
      "loss": 1.951,
      "nll_loss": 1.8998899459838867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1782114952802658,
      "rewards/margins": 0.03608858957886696,
      "rewards/rejected": -0.21430009603500366,
      "step": 179
    },
    {
      "epoch": 0.28368794326241137,
      "grad_norm": 0.3161435127258301,
      "learning_rate": 3.9796073083623774e-06,
      "log_odds_chosen": 0.41550877690315247,
      "log_odds_ratio": -0.5219519734382629,
      "logits/chosen": -0.08611653745174408,
      "logits/rejected": -0.8567744493484497,
      "logps/chosen": -1.8524975776672363,
      "logps/rejected": -2.2126336097717285,
      "loss": 2.0187,
      "nll_loss": 1.9664819240570068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18524976074695587,
      "rewards/margins": 0.036013588309288025,
      "rewards/rejected": -0.2212633341550827,
      "step": 180
    },
    {
      "epoch": 0.285263987391647,
      "grad_norm": 0.31543034315109253,
      "learning_rate": 3.978816833177329e-06,
      "log_odds_chosen": 0.3989214599132538,
      "log_odds_ratio": -0.519888162612915,
      "logits/chosen": -0.01459362730383873,
      "logits/rejected": -0.6901648044586182,
      "logps/chosen": -1.7767245769500732,
      "logps/rejected": -2.118866443634033,
      "loss": 1.9515,
      "nll_loss": 1.8994776010513306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17767244577407837,
      "rewards/margins": 0.03421417623758316,
      "rewards/rejected": -0.21188661456108093,
      "step": 181
    },
    {
      "epoch": 0.2868400315208826,
      "grad_norm": 0.32000571489334106,
      "learning_rate": 3.978011408976261e-06,
      "log_odds_chosen": 0.2851777970790863,
      "log_odds_ratio": -0.5661397576332092,
      "logits/chosen": -0.02493867464363575,
      "logits/rejected": -0.7876978516578674,
      "logps/chosen": -1.7986279726028442,
      "logps/rejected": -2.0402181148529053,
      "loss": 1.9869,
      "nll_loss": 1.9302453994750977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17986279726028442,
      "rewards/margins": 0.02415899559855461,
      "rewards/rejected": -0.20402181148529053,
      "step": 182
    },
    {
      "epoch": 0.28841607565011823,
      "grad_norm": 0.32417967915534973,
      "learning_rate": 3.9771910418437674e-06,
      "log_odds_chosen": 0.3460734486579895,
      "log_odds_ratio": -0.5362839102745056,
      "logits/chosen": -0.03848784416913986,
      "logits/rejected": -0.570169985294342,
      "logps/chosen": -1.8317880630493164,
      "logps/rejected": -2.131016731262207,
      "loss": 2.003,
      "nll_loss": 1.9494034051895142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1831788271665573,
      "rewards/margins": 0.029922863468527794,
      "rewards/rejected": -0.21310168504714966,
      "step": 183
    },
    {
      "epoch": 0.2899921197793538,
      "grad_norm": 0.3104395866394043,
      "learning_rate": 3.976355737977332e-06,
      "log_odds_chosen": 0.43051034212112427,
      "log_odds_ratio": -0.5086094737052917,
      "logits/chosen": -0.004376018885523081,
      "logits/rejected": -0.7855688333511353,
      "logps/chosen": -1.7710585594177246,
      "logps/rejected": -2.1398942470550537,
      "loss": 1.9511,
      "nll_loss": 1.9002678394317627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1771058589220047,
      "rewards/margins": 0.03688357025384903,
      "rewards/rejected": -0.21398943662643433,
      "step": 184
    },
    {
      "epoch": 0.2915681639085894,
      "grad_norm": 0.28238430619239807,
      "learning_rate": 3.975505503687274e-06,
      "log_odds_chosen": 0.32954704761505127,
      "log_odds_ratio": -0.5454957485198975,
      "logits/chosen": 0.03291358798742294,
      "logits/rejected": -0.48518458008766174,
      "logps/chosen": -1.764021635055542,
      "logps/rejected": -2.045203924179077,
      "loss": 1.9398,
      "nll_loss": 1.8852983713150024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17640215158462524,
      "rewards/margins": 0.028118234127759933,
      "rewards/rejected": -0.20452038943767548,
      "step": 185
    },
    {
      "epoch": 0.29314420803782504,
      "grad_norm": 0.27702274918556213,
      "learning_rate": 3.974640345396708e-06,
      "log_odds_chosen": 0.5177258253097534,
      "log_odds_ratio": -0.4748190641403198,
      "logits/chosen": -0.08025577664375305,
      "logits/rejected": -1.0515739917755127,
      "logps/chosen": -1.7293744087219238,
      "logps/rejected": -2.171941041946411,
      "loss": 1.9022,
      "nll_loss": 1.8546966314315796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17293745279312134,
      "rewards/margins": 0.044256679713726044,
      "rewards/rejected": -0.21719412505626678,
      "step": 186
    },
    {
      "epoch": 0.29472025216706066,
      "grad_norm": 0.2979573905467987,
      "learning_rate": 3.9737602696414925e-06,
      "log_odds_chosen": 0.2020474076271057,
      "log_odds_ratio": -0.6071421504020691,
      "logits/chosen": 0.029590757563710213,
      "logits/rejected": -0.8974511027336121,
      "logps/chosen": -1.8263177871704102,
      "logps/rejected": -1.9980144500732422,
      "loss": 1.9794,
      "nll_loss": 1.9187321662902832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18263177573680878,
      "rewards/margins": 0.017169667407870293,
      "rewards/rejected": -0.19980144500732422,
      "step": 187
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 0.2755199074745178,
      "learning_rate": 3.972865283070179e-06,
      "log_odds_chosen": 0.4493619203567505,
      "log_odds_ratio": -0.49928387999534607,
      "logits/chosen": -0.015258762054145336,
      "logits/rejected": -0.9950686097145081,
      "logps/chosen": -1.721350908279419,
      "logps/rejected": -2.1068735122680664,
      "loss": 1.9017,
      "nll_loss": 1.8517918586730957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1721350997686386,
      "rewards/margins": 0.03855225443840027,
      "rewards/rejected": -0.21068735420703888,
      "step": 188
    },
    {
      "epoch": 0.2978723404255319,
      "grad_norm": 0.2944152057170868,
      "learning_rate": 3.971955392443965e-06,
      "log_odds_chosen": 0.39672476053237915,
      "log_odds_ratio": -0.5201811790466309,
      "logits/chosen": -0.016328200697898865,
      "logits/rejected": -0.786607027053833,
      "logps/chosen": -1.7574641704559326,
      "logps/rejected": -2.0998551845550537,
      "loss": 1.9253,
      "nll_loss": 1.8732632398605347,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17574642598628998,
      "rewards/margins": 0.03423908352851868,
      "rewards/rejected": -0.20998550951480865,
      "step": 189
    },
    {
      "epoch": 0.2994483845547675,
      "grad_norm": 0.28858453035354614,
      "learning_rate": 3.971030604636637e-06,
      "log_odds_chosen": 0.27192696928977966,
      "log_odds_ratio": -0.570553183555603,
      "logits/chosen": 0.050630006939172745,
      "logits/rejected": -0.7770216464996338,
      "logps/chosen": -1.7724601030349731,
      "logps/rejected": -2.0022082328796387,
      "loss": 1.9388,
      "nll_loss": 1.8817503452301025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17724601924419403,
      "rewards/margins": 0.02297479659318924,
      "rewards/rejected": -0.20022080838680267,
      "step": 190
    },
    {
      "epoch": 0.30102442868400314,
      "grad_norm": 0.2995285987854004,
      "learning_rate": 3.970090926634526e-06,
      "log_odds_chosen": 0.3720911741256714,
      "log_odds_ratio": -0.5282139778137207,
      "logits/chosen": 0.11160154640674591,
      "logits/rejected": -0.9845487475395203,
      "logps/chosen": -1.8277368545532227,
      "logps/rejected": -2.1482725143432617,
      "loss": 2.0007,
      "nll_loss": 1.9478685855865479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18277369439601898,
      "rewards/margins": 0.032053571194410324,
      "rewards/rejected": -0.2148272544145584,
      "step": 191
    },
    {
      "epoch": 0.30260047281323876,
      "grad_norm": 0.3174213767051697,
      "learning_rate": 3.9691363655364526e-06,
      "log_odds_chosen": 0.4017809331417084,
      "log_odds_ratio": -0.5195269584655762,
      "logits/chosen": 0.0210281815379858,
      "logits/rejected": -0.7612247467041016,
      "logps/chosen": -1.8064415454864502,
      "logps/rejected": -2.152747392654419,
      "loss": 1.978,
      "nll_loss": 1.9260954856872559,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18064415454864502,
      "rewards/margins": 0.03463058918714523,
      "rewards/rejected": -0.21527475118637085,
      "step": 192
    },
    {
      "epoch": 0.3041765169424744,
      "grad_norm": 0.278577595949173,
      "learning_rate": 3.968166928553666e-06,
      "log_odds_chosen": 0.3343183696269989,
      "log_odds_ratio": -0.544468104839325,
      "logits/chosen": 0.06776704639196396,
      "logits/rejected": -0.9449382424354553,
      "logps/chosen": -1.7551405429840088,
      "logps/rejected": -2.0379421710968018,
      "loss": 1.9431,
      "nll_loss": 1.8886576890945435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17551404237747192,
      "rewards/margins": 0.02828015573322773,
      "rewards/rejected": -0.2037942111492157,
      "step": 193
    },
    {
      "epoch": 0.30575256107171,
      "grad_norm": 0.2866746485233307,
      "learning_rate": 3.967182623009804e-06,
      "log_odds_chosen": 0.46259909868240356,
      "log_odds_ratio": -0.4940981864929199,
      "logits/chosen": 0.020696064457297325,
      "logits/rejected": -0.7941763401031494,
      "logps/chosen": -1.7806702852249146,
      "logps/rejected": -2.181614637374878,
      "loss": 1.9371,
      "nll_loss": 1.8876622915267944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17806704342365265,
      "rewards/margins": 0.040094420313835144,
      "rewards/rejected": -0.218161478638649,
      "step": 194
    },
    {
      "epoch": 0.3073286052009456,
      "grad_norm": 0.30178844928741455,
      "learning_rate": 3.966183456340821e-06,
      "log_odds_chosen": 0.238200843334198,
      "log_odds_ratio": -0.5833666324615479,
      "logits/chosen": 0.06038721278309822,
      "logits/rejected": -0.818547785282135,
      "logps/chosen": -1.8105565309524536,
      "logps/rejected": -2.0120017528533936,
      "loss": 1.9804,
      "nll_loss": 1.922109842300415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18105565011501312,
      "rewards/margins": 0.020144525915384293,
      "rewards/rejected": -0.20120017230510712,
      "step": 195
    },
    {
      "epoch": 0.30890464933018125,
      "grad_norm": 0.30308058857917786,
      "learning_rate": 3.965169436094947e-06,
      "log_odds_chosen": 0.43042975664138794,
      "log_odds_ratio": -0.5097436308860779,
      "logits/chosen": 0.0861726701259613,
      "logits/rejected": -0.8697912693023682,
      "logps/chosen": -1.7020423412322998,
      "logps/rejected": -2.069258451461792,
      "loss": 1.8798,
      "nll_loss": 1.828870415687561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17020423710346222,
      "rewards/margins": 0.0367216058075428,
      "rewards/rejected": -0.20692585408687592,
      "step": 196
    },
    {
      "epoch": 0.31048069345941687,
      "grad_norm": 0.28854966163635254,
      "learning_rate": 3.964140569932618e-06,
      "log_odds_chosen": 0.3548341393470764,
      "log_odds_ratio": -0.549152135848999,
      "logits/chosen": 0.04638773947954178,
      "logits/rejected": -1.1066012382507324,
      "logps/chosen": -1.8225290775299072,
      "logps/rejected": -2.125335931777954,
      "loss": 1.9827,
      "nll_loss": 1.9277925491333008,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.182252898812294,
      "rewards/margins": 0.030280686914920807,
      "rewards/rejected": -0.2125336080789566,
      "step": 197
    },
    {
      "epoch": 0.3120567375886525,
      "grad_norm": 0.3032393157482147,
      "learning_rate": 3.9630968656264285e-06,
      "log_odds_chosen": 0.4571669101715088,
      "log_odds_ratio": -0.4972551167011261,
      "logits/chosen": 0.05622638389468193,
      "logits/rejected": -0.7387775778770447,
      "logps/chosen": -1.7499175071716309,
      "logps/rejected": -2.142073154449463,
      "loss": 1.9064,
      "nll_loss": 1.8567097187042236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17499174177646637,
      "rewards/margins": 0.039215561002492905,
      "rewards/rejected": -0.21420730650424957,
      "step": 198
    },
    {
      "epoch": 0.3136327817178881,
      "grad_norm": 0.300789475440979,
      "learning_rate": 3.962038331061065e-06,
      "log_odds_chosen": 0.20108665525913239,
      "log_odds_ratio": -0.6110847592353821,
      "logits/chosen": 0.13670037686824799,
      "logits/rejected": -0.8646966218948364,
      "logps/chosen": -1.8039584159851074,
      "logps/rejected": -1.9805593490600586,
      "loss": 1.9825,
      "nll_loss": 1.921362280845642,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18039584159851074,
      "rewards/margins": 0.017660098150372505,
      "rewards/rejected": -0.1980559378862381,
      "step": 199
    },
    {
      "epoch": 0.31520882584712373,
      "grad_norm": 0.28587841987609863,
      "learning_rate": 3.96096497423325e-06,
      "log_odds_chosen": 0.298963338136673,
      "log_odds_ratio": -0.5593155026435852,
      "logits/chosen": 0.16921967267990112,
      "logits/rejected": -0.5161201357841492,
      "logps/chosen": -1.6970977783203125,
      "logps/rejected": -1.9485868215560913,
      "loss": 1.8973,
      "nll_loss": 1.8413777351379395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16970978677272797,
      "rewards/margins": 0.025148894637823105,
      "rewards/rejected": -0.19485867023468018,
      "step": 200
    },
    {
      "epoch": 0.31678486997635935,
      "grad_norm": 0.2650342285633087,
      "learning_rate": 3.959876803251684e-06,
      "log_odds_chosen": 0.42556777596473694,
      "log_odds_ratio": -0.5099666118621826,
      "logits/chosen": 0.04669623449444771,
      "logits/rejected": -0.791134238243103,
      "logps/chosen": -1.718097448348999,
      "logps/rejected": -2.0799450874328613,
      "loss": 1.8699,
      "nll_loss": 1.818863034248352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17180974781513214,
      "rewards/margins": 0.03618478775024414,
      "rewards/rejected": -0.20799453556537628,
      "step": 201
    },
    {
      "epoch": 0.31836091410559497,
      "grad_norm": 0.2691303491592407,
      "learning_rate": 3.958773826336977e-06,
      "log_odds_chosen": 0.3119014501571655,
      "log_odds_ratio": -0.5508911609649658,
      "logits/chosen": 0.1294005662202835,
      "logits/rejected": -0.6723183989524841,
      "logps/chosen": -1.7649327516555786,
      "logps/rejected": -2.0296618938446045,
      "loss": 1.9357,
      "nll_loss": 1.8805636167526245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17649328708648682,
      "rewards/margins": 0.026472903788089752,
      "rewards/rejected": -0.20296618342399597,
      "step": 202
    },
    {
      "epoch": 0.3199369582348306,
      "grad_norm": 0.27943408489227295,
      "learning_rate": 3.957656051821592e-06,
      "log_odds_chosen": 0.265775203704834,
      "log_odds_ratio": -0.5743451714515686,
      "logits/chosen": 0.14156483113765717,
      "logits/rejected": -1.0007867813110352,
      "logps/chosen": -1.794228434562683,
      "logps/rejected": -2.022888660430908,
      "loss": 1.9404,
      "nll_loss": 1.8829922676086426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17942282557487488,
      "rewards/margins": 0.022866029292345047,
      "rewards/rejected": -0.20228886604309082,
      "step": 203
    },
    {
      "epoch": 0.3215130023640662,
      "grad_norm": 0.2855702042579651,
      "learning_rate": 3.956523488149783e-06,
      "log_odds_chosen": 0.3671448826789856,
      "log_odds_ratio": -0.5314764976501465,
      "logits/chosen": 0.1590101420879364,
      "logits/rejected": -0.7161625623703003,
      "logps/chosen": -1.8966209888458252,
      "logps/rejected": -2.216151237487793,
      "loss": 2.0521,
      "nll_loss": 1.9989579916000366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18966209888458252,
      "rewards/margins": 0.03195303678512573,
      "rewards/rejected": -0.22161512076854706,
      "step": 204
    },
    {
      "epoch": 0.32308904649330183,
      "grad_norm": 0.2792893350124359,
      "learning_rate": 3.9553761438775285e-06,
      "log_odds_chosen": 0.5917212963104248,
      "log_odds_ratio": -0.4462578594684601,
      "logits/chosen": 0.04287697374820709,
      "logits/rejected": -0.7457516193389893,
      "logps/chosen": -1.629404067993164,
      "logps/rejected": -2.1322901248931885,
      "loss": 1.7862,
      "nll_loss": 1.741592526435852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16294041275978088,
      "rewards/margins": 0.0502886101603508,
      "rewards/rejected": -0.2132290005683899,
      "step": 205
    },
    {
      "epoch": 0.32466509062253746,
      "grad_norm": 0.2581063210964203,
      "learning_rate": 3.954214027672465e-06,
      "log_odds_chosen": 0.553436815738678,
      "log_odds_ratio": -0.4622824788093567,
      "logits/chosen": 0.06957760453224182,
      "logits/rejected": -1.1285384893417358,
      "logps/chosen": -1.7171605825424194,
      "logps/rejected": -2.1953935623168945,
      "loss": 1.867,
      "nll_loss": 1.8207881450653076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17171606421470642,
      "rewards/margins": 0.04782331734895706,
      "rewards/rejected": -0.21953937411308289,
      "step": 206
    },
    {
      "epoch": 0.3262411347517731,
      "grad_norm": 0.28155970573425293,
      "learning_rate": 3.953037148313825e-06,
      "log_odds_chosen": 0.3361928164958954,
      "log_odds_ratio": -0.5446957945823669,
      "logits/chosen": 0.1372230350971222,
      "logits/rejected": -0.8692892789840698,
      "logps/chosen": -1.7089570760726929,
      "logps/rejected": -1.995591640472412,
      "loss": 1.8908,
      "nll_loss": 1.8363476991653442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17089569568634033,
      "rewards/margins": 0.028663454577326775,
      "rewards/rejected": -0.19955916702747345,
      "step": 207
    },
    {
      "epoch": 0.32781717888100864,
      "grad_norm": 0.2895718216896057,
      "learning_rate": 3.951845514692371e-06,
      "log_odds_chosen": 0.4320535957813263,
      "log_odds_ratio": -0.5092025399208069,
      "logits/chosen": 0.06572133302688599,
      "logits/rejected": -0.7182308435440063,
      "logps/chosen": -1.7720617055892944,
      "logps/rejected": -2.1444907188415527,
      "loss": 1.9423,
      "nll_loss": 1.8913987874984741,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1772061586380005,
      "rewards/margins": 0.037242915481328964,
      "rewards/rejected": -0.21444910764694214,
      "step": 208
    },
    {
      "epoch": 0.32939322301024426,
      "grad_norm": 0.2872414290904999,
      "learning_rate": 3.950639135810325e-06,
      "log_odds_chosen": 0.3550606369972229,
      "log_odds_ratio": -0.5360490083694458,
      "logits/chosen": 0.2004653960466385,
      "logits/rejected": -0.6822468042373657,
      "logps/chosen": -1.889462947845459,
      "logps/rejected": -2.1966936588287354,
      "loss": 2.0309,
      "nll_loss": 1.977307677268982,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18894629180431366,
      "rewards/margins": 0.030723070725798607,
      "rewards/rejected": -0.21966935694217682,
      "step": 209
    },
    {
      "epoch": 0.3309692671394799,
      "grad_norm": 0.27572956681251526,
      "learning_rate": 3.9494180207813044e-06,
      "log_odds_chosen": 0.2917708456516266,
      "log_odds_ratio": -0.5690155029296875,
      "logits/chosen": 0.18270650506019592,
      "logits/rejected": -0.9679818153381348,
      "logps/chosen": -1.759641408920288,
      "logps/rejected": -2.006713628768921,
      "loss": 1.9279,
      "nll_loss": 1.8709876537322998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17596416175365448,
      "rewards/margins": 0.024707205593585968,
      "rewards/rejected": -0.20067137479782104,
      "step": 210
    },
    {
      "epoch": 0.3325453112687155,
      "grad_norm": 0.27592262625694275,
      "learning_rate": 3.948182178830249e-06,
      "log_odds_chosen": 0.3494272828102112,
      "log_odds_ratio": -0.5527153015136719,
      "logits/chosen": 0.17400677502155304,
      "logits/rejected": -1.0591527223587036,
      "logps/chosen": -1.7874795198440552,
      "logps/rejected": -2.091515302658081,
      "loss": 1.9384,
      "nll_loss": 1.883126974105835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1787479668855667,
      "rewards/margins": 0.030403554439544678,
      "rewards/rejected": -0.2091515213251114,
      "step": 211
    },
    {
      "epoch": 0.3341213553979511,
      "grad_norm": 0.2815802991390228,
      "learning_rate": 3.9469316192933545e-06,
      "log_odds_chosen": 0.5855627059936523,
      "log_odds_ratio": -0.451069712638855,
      "logits/chosen": 0.05581940338015556,
      "logits/rejected": -0.6912811994552612,
      "logps/chosen": -1.643700361251831,
      "logps/rejected": -2.1421589851379395,
      "loss": 1.809,
      "nll_loss": 1.7639025449752808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16437003016471863,
      "rewards/margins": 0.0498458594083786,
      "rewards/rejected": -0.21421588957309723,
      "step": 212
    },
    {
      "epoch": 0.33569739952718675,
      "grad_norm": 0.3026936948299408,
      "learning_rate": 3.945666351618001e-06,
      "log_odds_chosen": 0.323087215423584,
      "log_odds_ratio": -0.5478854775428772,
      "logits/chosen": 0.14214320480823517,
      "logits/rejected": -0.8071248531341553,
      "logps/chosen": -1.700455665588379,
      "logps/rejected": -1.9733654260635376,
      "loss": 1.8643,
      "nll_loss": 1.8095039129257202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17004558444023132,
      "rewards/margins": 0.027290964499115944,
      "rewards/rejected": -0.19733653962612152,
      "step": 213
    },
    {
      "epoch": 0.33727344365642237,
      "grad_norm": 0.2719425857067108,
      "learning_rate": 3.9443863853626825e-06,
      "log_odds_chosen": 0.36333683133125305,
      "log_odds_ratio": -0.5374127626419067,
      "logits/chosen": 0.16779878735542297,
      "logits/rejected": -0.8444754481315613,
      "logps/chosen": -1.6595702171325684,
      "logps/rejected": -1.9579966068267822,
      "loss": 1.8462,
      "nll_loss": 1.7924606800079346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1659570336341858,
      "rewards/margins": 0.029842618852853775,
      "rewards/rejected": -0.19579966366291046,
      "step": 214
    },
    {
      "epoch": 0.338849487785658,
      "grad_norm": 0.26499754190444946,
      "learning_rate": 3.943091730196931e-06,
      "log_odds_chosen": 0.32084012031555176,
      "log_odds_ratio": -0.5536172986030579,
      "logits/chosen": 0.2257343828678131,
      "logits/rejected": -0.6080964803695679,
      "logps/chosen": -1.6629269123077393,
      "logps/rejected": -1.9345366954803467,
      "loss": 1.8348,
      "nll_loss": 1.7794655561447144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1662926971912384,
      "rewards/margins": 0.02716096118092537,
      "rewards/rejected": -0.19345368444919586,
      "step": 215
    },
    {
      "epoch": 0.3404255319148936,
      "grad_norm": 0.2940061092376709,
      "learning_rate": 3.941782395901249e-06,
      "log_odds_chosen": 0.3910900950431824,
      "log_odds_ratio": -0.522650420665741,
      "logits/chosen": 0.13361163437366486,
      "logits/rejected": -0.7325764298439026,
      "logps/chosen": -1.7365449666976929,
      "logps/rejected": -2.0691425800323486,
      "loss": 1.9089,
      "nll_loss": 1.8566128015518188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17365451157093048,
      "rewards/margins": 0.033259760588407516,
      "rewards/rejected": -0.2069142609834671,
      "step": 216
    },
    {
      "epoch": 0.34200157604412923,
      "grad_norm": 0.2855311334133148,
      "learning_rate": 3.940458392367032e-06,
      "log_odds_chosen": 0.359374463558197,
      "log_odds_ratio": -0.533739447593689,
      "logits/chosen": 0.17437973618507385,
      "logits/rejected": -1.037083387374878,
      "logps/chosen": -1.79496431350708,
      "logps/rejected": -2.1024041175842285,
      "loss": 1.9325,
      "nll_loss": 1.8790829181671143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17949643731117249,
      "rewards/margins": 0.030744003131985664,
      "rewards/rejected": -0.2102404236793518,
      "step": 217
    },
    {
      "epoch": 0.34357762017336485,
      "grad_norm": 0.27844125032424927,
      "learning_rate": 3.939119729596493e-06,
      "log_odds_chosen": 0.400441437959671,
      "log_odds_ratio": -0.5161818265914917,
      "logits/chosen": 0.2301069051027298,
      "logits/rejected": -0.8860409259796143,
      "logps/chosen": -1.7927645444869995,
      "logps/rejected": -2.137216567993164,
      "loss": 1.9434,
      "nll_loss": 1.89176344871521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1792764663696289,
      "rewards/margins": 0.034445181488990784,
      "rewards/rejected": -0.2137216329574585,
      "step": 218
    },
    {
      "epoch": 0.34515366430260047,
      "grad_norm": 0.26965805888175964,
      "learning_rate": 3.93776641770259e-06,
      "log_odds_chosen": 0.328607439994812,
      "log_odds_ratio": -0.5486437082290649,
      "logits/chosen": 0.20194567739963531,
      "logits/rejected": -0.8480868935585022,
      "logps/chosen": -1.7666839361190796,
      "logps/rejected": -2.047614097595215,
      "loss": 1.9141,
      "nll_loss": 1.859285593032837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17666839063167572,
      "rewards/margins": 0.028093010187149048,
      "rewards/rejected": -0.20476141571998596,
      "step": 219
    },
    {
      "epoch": 0.3467297084318361,
      "grad_norm": 0.26573604345321655,
      "learning_rate": 3.93639846690895e-06,
      "log_odds_chosen": 0.47803783416748047,
      "log_odds_ratio": -0.4869340658187866,
      "logits/chosen": 0.22956933081150055,
      "logits/rejected": -1.1470171213150024,
      "logps/chosen": -1.7806720733642578,
      "logps/rejected": -2.194432258605957,
      "loss": 1.9229,
      "nll_loss": 1.8741583824157715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17806722223758698,
      "rewards/margins": 0.04137600213289261,
      "rewards/rejected": -0.21944323182106018,
      "step": 220
    },
    {
      "epoch": 0.3483057525610717,
      "grad_norm": 0.27221164107322693,
      "learning_rate": 3.9350158875497855e-06,
      "log_odds_chosen": 0.5355910062789917,
      "log_odds_ratio": -0.4647471606731415,
      "logits/chosen": 0.2502666711807251,
      "logits/rejected": -0.9178094863891602,
      "logps/chosen": -1.693802833557129,
      "logps/rejected": -2.1492807865142822,
      "loss": 1.851,
      "nll_loss": 1.8045127391815186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1693802773952484,
      "rewards/margins": 0.04554780200123787,
      "rewards/rejected": -0.21492807567119598,
      "step": 221
    },
    {
      "epoch": 0.34988179669030733,
      "grad_norm": 0.2870781719684601,
      "learning_rate": 3.933618690069824e-06,
      "log_odds_chosen": 0.514340341091156,
      "log_odds_ratio": -0.4729236364364624,
      "logits/chosen": 0.1029556393623352,
      "logits/rejected": -0.9085869789123535,
      "logps/chosen": -1.63947331905365,
      "logps/rejected": -2.0694899559020996,
      "loss": 1.8078,
      "nll_loss": 1.760536789894104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16394734382629395,
      "rewards/margins": 0.04300164431333542,
      "rewards/rejected": -0.20694898068904877,
      "step": 222
    },
    {
      "epoch": 0.35145784081954295,
      "grad_norm": 0.26160404086112976,
      "learning_rate": 3.932206885024226e-06,
      "log_odds_chosen": 0.2591567635536194,
      "log_odds_ratio": -0.5751827359199524,
      "logits/chosen": 0.14362215995788574,
      "logits/rejected": -0.8560265898704529,
      "logps/chosen": -1.7445533275604248,
      "logps/rejected": -1.963904857635498,
      "loss": 1.8865,
      "nll_loss": 1.82896888256073,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17445534467697144,
      "rewards/margins": 0.021935151889920235,
      "rewards/rejected": -0.19639047980308533,
      "step": 223
    },
    {
      "epoch": 0.3530338849487786,
      "grad_norm": 0.24491266906261444,
      "learning_rate": 3.930780483078502e-06,
      "log_odds_chosen": 0.39663180708885193,
      "log_odds_ratio": -0.5192946195602417,
      "logits/chosen": 0.13351722061634064,
      "logits/rejected": -0.8884856104850769,
      "logps/chosen": -1.6729098558425903,
      "logps/rejected": -2.0054969787597656,
      "loss": 1.8405,
      "nll_loss": 1.7885487079620361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16729097068309784,
      "rewards/margins": 0.03325873613357544,
      "rewards/rejected": -0.20054970681667328,
      "step": 224
    },
    {
      "epoch": 0.3546099290780142,
      "grad_norm": 0.26580068469047546,
      "learning_rate": 3.92933949500844e-06,
      "log_odds_chosen": 0.4522182047367096,
      "log_odds_ratio": -0.5091683268547058,
      "logits/chosen": 0.20834729075431824,
      "logits/rejected": -0.8979159593582153,
      "logps/chosen": -1.6526035070419312,
      "logps/rejected": -2.027092933654785,
      "loss": 1.8166,
      "nll_loss": 1.765660285949707,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16526035964488983,
      "rewards/margins": 0.0374489426612854,
      "rewards/rejected": -0.20270927250385284,
      "step": 225
    },
    {
      "epoch": 0.3561859732072498,
      "grad_norm": 0.2466343641281128,
      "learning_rate": 3.9278839317000155e-06,
      "log_odds_chosen": 0.4413149952888489,
      "log_odds_ratio": -0.502187192440033,
      "logits/chosen": 0.22558066248893738,
      "logits/rejected": -0.8436870574951172,
      "logps/chosen": -1.676627516746521,
      "logps/rejected": -2.049926519393921,
      "loss": 1.8305,
      "nll_loss": 1.7802934646606445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16766275465488434,
      "rewards/margins": 0.037329915910959244,
      "rewards/rejected": -0.20499266684055328,
      "step": 226
    },
    {
      "epoch": 0.35776201733648544,
      "grad_norm": 0.2658051550388336,
      "learning_rate": 3.926413804149314e-06,
      "log_odds_chosen": 0.179185152053833,
      "log_odds_ratio": -0.6169071793556213,
      "logits/chosen": 0.24110640585422516,
      "logits/rejected": -0.8408032059669495,
      "logps/chosen": -1.758495569229126,
      "logps/rejected": -1.9097925424575806,
      "loss": 1.9105,
      "nll_loss": 1.8488125801086426,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1758495569229126,
      "rewards/margins": 0.015129687264561653,
      "rewards/rejected": -0.1909792423248291,
      "step": 227
    },
    {
      "epoch": 0.35933806146572106,
      "grad_norm": 0.26918303966522217,
      "learning_rate": 3.924929123462447e-06,
      "log_odds_chosen": 0.33828845620155334,
      "log_odds_ratio": -0.5443422794342041,
      "logits/chosen": 0.2773420810699463,
      "logits/rejected": -0.8576438426971436,
      "logps/chosen": -1.7860910892486572,
      "logps/rejected": -2.0751004219055176,
      "loss": 1.9309,
      "nll_loss": 1.876507043838501,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17860911786556244,
      "rewards/margins": 0.028900934383273125,
      "rewards/rejected": -0.20751003921031952,
      "step": 228
    },
    {
      "epoch": 0.3609141055949567,
      "grad_norm": 0.27900397777557373,
      "learning_rate": 3.923429900855468e-06,
      "log_odds_chosen": 0.26649370789527893,
      "log_odds_ratio": -0.5749764442443848,
      "logits/chosen": 0.2404632568359375,
      "logits/rejected": -0.6364312767982483,
      "logps/chosen": -1.857352375984192,
      "logps/rejected": -2.0843093395233154,
      "loss": 2.0045,
      "nll_loss": 1.9470313787460327,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18573524057865143,
      "rewards/margins": 0.02269568480551243,
      "rewards/rejected": -0.2084309309720993,
      "step": 229
    },
    {
      "epoch": 0.3624901497241923,
      "grad_norm": 0.2622164189815521,
      "learning_rate": 3.921916147654287e-06,
      "log_odds_chosen": 0.5030225515365601,
      "log_odds_ratio": -0.4795306324958801,
      "logits/chosen": 0.19567719101905823,
      "logits/rejected": -0.8917854428291321,
      "logps/chosen": -1.693268895149231,
      "logps/rejected": -2.1187849044799805,
      "loss": 1.829,
      "nll_loss": 1.7810728549957275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16932690143585205,
      "rewards/margins": 0.04255159944295883,
      "rewards/rejected": -0.21187849342823029,
      "step": 230
    },
    {
      "epoch": 0.3640661938534279,
      "grad_norm": 0.25981763005256653,
      "learning_rate": 3.920387875294588e-06,
      "log_odds_chosen": 0.34487566351890564,
      "log_odds_ratio": -0.5391297936439514,
      "logits/chosen": 0.22495216131210327,
      "logits/rejected": -0.7459501028060913,
      "logps/chosen": -1.6685864925384521,
      "logps/rejected": -1.9565632343292236,
      "loss": 1.8216,
      "nll_loss": 1.7676771879196167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16685864329338074,
      "rewards/margins": 0.028797684237360954,
      "rewards/rejected": -0.19565632939338684,
      "step": 231
    },
    {
      "epoch": 0.3656422379826635,
      "grad_norm": 0.2565964460372925,
      "learning_rate": 3.918845095321737e-06,
      "log_odds_chosen": 0.32233738899230957,
      "log_odds_ratio": -0.5548774003982544,
      "logits/chosen": 0.19658507406711578,
      "logits/rejected": -0.9954935908317566,
      "logps/chosen": -1.6999584436416626,
      "logps/rejected": -1.9643486738204956,
      "loss": 1.8453,
      "nll_loss": 1.7898164987564087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16999585926532745,
      "rewards/margins": 0.026439011096954346,
      "rewards/rejected": -0.1964348405599594,
      "step": 232
    },
    {
      "epoch": 0.3672182821118991,
      "grad_norm": 0.26710182428359985,
      "learning_rate": 3.9172878193907004e-06,
      "log_odds_chosen": 0.36338526010513306,
      "log_odds_ratio": -0.5375514030456543,
      "logits/chosen": 0.13183602690696716,
      "logits/rejected": -0.9046310782432556,
      "logps/chosen": -1.772655963897705,
      "logps/rejected": -2.0851950645446777,
      "loss": 1.9319,
      "nll_loss": 1.8781368732452393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17726561427116394,
      "rewards/margins": 0.03125389292836189,
      "rewards/rejected": -0.20851948857307434,
      "step": 233
    },
    {
      "epoch": 0.36879432624113473,
      "grad_norm": 0.252542108297348,
      "learning_rate": 3.915716059265955e-06,
      "log_odds_chosen": 0.3764590620994568,
      "log_odds_ratio": -0.5301459431648254,
      "logits/chosen": 0.2072755992412567,
      "logits/rejected": -0.9679795503616333,
      "logps/chosen": -1.7244305610656738,
      "logps/rejected": -2.0450494289398193,
      "loss": 1.8711,
      "nll_loss": 1.8181140422821045,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17244306206703186,
      "rewards/margins": 0.032061897218227386,
      "rewards/rejected": -0.20450495183467865,
      "step": 234
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 0.23418840765953064,
      "learning_rate": 3.9141298268213966e-06,
      "log_odds_chosen": 0.47056907415390015,
      "log_odds_ratio": -0.48991110920906067,
      "logits/chosen": 0.2516101002693176,
      "logits/rejected": -0.8004311323165894,
      "logps/chosen": -1.643180251121521,
      "logps/rejected": -2.04105806350708,
      "loss": 1.7967,
      "nll_loss": 1.7476621866226196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1643180400133133,
      "rewards/margins": 0.039787761867046356,
      "rewards/rejected": -0.20410577952861786,
      "step": 235
    },
    {
      "epoch": 0.37194641449960597,
      "grad_norm": 0.26616281270980835,
      "learning_rate": 3.912529134040255e-06,
      "log_odds_chosen": 0.3272354304790497,
      "log_odds_ratio": -0.5463756918907166,
      "logits/chosen": 0.23501628637313843,
      "logits/rejected": -0.7225480079650879,
      "logps/chosen": -1.7243411540985107,
      "logps/rejected": -2.002100706100464,
      "loss": 1.8846,
      "nll_loss": 1.8299776315689087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17243412137031555,
      "rewards/margins": 0.027775948867201805,
      "rewards/rejected": -0.2002100646495819,
      "step": 236
    },
    {
      "epoch": 0.3735224586288416,
      "grad_norm": 0.2716814875602722,
      "learning_rate": 3.910913993014998e-06,
      "log_odds_chosen": 0.48933014273643494,
      "log_odds_ratio": -0.48118287324905396,
      "logits/chosen": 0.3107871413230896,
      "logits/rejected": -0.8831154704093933,
      "logps/chosen": -1.7238764762878418,
      "logps/rejected": -2.1423754692077637,
      "loss": 1.8692,
      "nll_loss": 1.8211053609848022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17238764464855194,
      "rewards/margins": 0.041849926114082336,
      "rewards/rejected": -0.21423755586147308,
      "step": 237
    },
    {
      "epoch": 0.3750985027580772,
      "grad_norm": 0.2618771493434906,
      "learning_rate": 3.909284415947246e-06,
      "log_odds_chosen": 0.47179070115089417,
      "log_odds_ratio": -0.4907287359237671,
      "logits/chosen": 0.3078673183917999,
      "logits/rejected": -0.9788669347763062,
      "logps/chosen": -1.7702430486679077,
      "logps/rejected": -2.173584461212158,
      "loss": 1.9009,
      "nll_loss": 1.851858139038086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17702430486679077,
      "rewards/margins": 0.04033412039279938,
      "rewards/rejected": -0.21735844016075134,
      "step": 238
    },
    {
      "epoch": 0.37667454688731283,
      "grad_norm": 0.24487651884555817,
      "learning_rate": 3.907640415147674e-06,
      "log_odds_chosen": 0.5142653584480286,
      "log_odds_ratio": -0.47312235832214355,
      "logits/chosen": 0.17608575522899628,
      "logits/rejected": -1.0495761632919312,
      "logps/chosen": -1.6641782522201538,
      "logps/rejected": -2.098402738571167,
      "loss": 1.8178,
      "nll_loss": 1.7705227136611938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16641782224178314,
      "rewards/margins": 0.04342243820428848,
      "rewards/rejected": -0.20984025299549103,
      "step": 239
    },
    {
      "epoch": 0.37825059101654845,
      "grad_norm": 0.2650536000728607,
      "learning_rate": 3.905982003035924e-06,
      "log_odds_chosen": 0.4236319363117218,
      "log_odds_ratio": -0.5087918043136597,
      "logits/chosen": 0.20158736407756805,
      "logits/rejected": -0.9927594661712646,
      "logps/chosen": -1.7691205739974976,
      "logps/rejected": -2.132444143295288,
      "loss": 1.9045,
      "nll_loss": 1.853621006011963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17691205441951752,
      "rewards/margins": 0.03633236512541771,
      "rewards/rejected": -0.21324442327022552,
      "step": 240
    },
    {
      "epoch": 0.3798266351457841,
      "grad_norm": 0.26103925704956055,
      "learning_rate": 3.904309192140506e-06,
      "log_odds_chosen": 0.2944376468658447,
      "log_odds_ratio": -0.564617931842804,
      "logits/chosen": 0.21214058995246887,
      "logits/rejected": -0.6803040504455566,
      "logps/chosen": -1.6910583972930908,
      "logps/rejected": -1.9392961263656616,
      "loss": 1.8418,
      "nll_loss": 1.7853529453277588,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16910582780838013,
      "rewards/margins": 0.024823768064379692,
      "rewards/rejected": -0.19392961263656616,
      "step": 241
    },
    {
      "epoch": 0.3814026792750197,
      "grad_norm": 0.2756460905075073,
      "learning_rate": 3.90262199509871e-06,
      "log_odds_chosen": 0.31087297201156616,
      "log_odds_ratio": -0.5524182319641113,
      "logits/chosen": 0.2239079773426056,
      "logits/rejected": -0.7127591371536255,
      "logps/chosen": -1.7215018272399902,
      "logps/rejected": -1.9822158813476562,
      "loss": 1.8763,
      "nll_loss": 1.8210104703903198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17215019464492798,
      "rewards/margins": 0.026071399450302124,
      "rewards/rejected": -0.1982215791940689,
      "step": 242
    },
    {
      "epoch": 0.3829787234042553,
      "grad_norm": 0.25208762288093567,
      "learning_rate": 3.900920424656501e-06,
      "log_odds_chosen": 0.4542803168296814,
      "log_odds_ratio": -0.4980185031890869,
      "logits/chosen": 0.19997750222682953,
      "logits/rejected": -0.8956844806671143,
      "logps/chosen": -1.6321688890457153,
      "logps/rejected": -2.0129523277282715,
      "loss": 1.777,
      "nll_loss": 1.7271740436553955,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16321690380573273,
      "rewards/margins": 0.038078341633081436,
      "rewards/rejected": -0.20129524171352386,
      "step": 243
    },
    {
      "epoch": 0.38455476753349094,
      "grad_norm": 0.275621235370636,
      "learning_rate": 3.899204493668432e-06,
      "log_odds_chosen": 0.3811621069908142,
      "log_odds_ratio": -0.5312854647636414,
      "logits/chosen": 0.27042973041534424,
      "logits/rejected": -0.8135743141174316,
      "logps/chosen": -1.7932356595993042,
      "logps/rejected": -2.1242425441741943,
      "loss": 1.9275,
      "nll_loss": 1.8743923902511597,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17932356894016266,
      "rewards/margins": 0.0331006720662117,
      "rewards/rejected": -0.21242424845695496,
      "step": 244
    },
    {
      "epoch": 0.38613081166272656,
      "grad_norm": 0.269351065158844,
      "learning_rate": 3.897474215097542e-06,
      "log_odds_chosen": 0.37562695145606995,
      "log_odds_ratio": -0.5268362760543823,
      "logits/chosen": 0.30195605754852295,
      "logits/rejected": -1.0125755071640015,
      "logps/chosen": -1.7352933883666992,
      "logps/rejected": -2.0556907653808594,
      "loss": 1.891,
      "nll_loss": 1.8383123874664307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17352935671806335,
      "rewards/margins": 0.03203972801566124,
      "rewards/rejected": -0.2055690735578537,
      "step": 245
    },
    {
      "epoch": 0.3877068557919622,
      "grad_norm": 0.24685640633106232,
      "learning_rate": 3.8957296020152596e-06,
      "log_odds_chosen": 0.4886430501937866,
      "log_odds_ratio": -0.48388606309890747,
      "logits/chosen": 0.29761719703674316,
      "logits/rejected": -0.8692208528518677,
      "logps/chosen": -1.6982948780059814,
      "logps/rejected": -2.1153006553649902,
      "loss": 1.8491,
      "nll_loss": 1.8006938695907593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16982948780059814,
      "rewards/margins": 0.04170059412717819,
      "rewards/rejected": -0.21153007447719574,
      "step": 246
    },
    {
      "epoch": 0.3892828999211978,
      "grad_norm": 0.2664787173271179,
      "learning_rate": 3.893970667601303e-06,
      "log_odds_chosen": 0.3381783068180084,
      "log_odds_ratio": -0.5428158044815063,
      "logits/chosen": 0.19423866271972656,
      "logits/rejected": -1.1145470142364502,
      "logps/chosen": -1.783911943435669,
      "logps/rejected": -2.072521924972534,
      "loss": 1.9215,
      "nll_loss": 1.8672170639038086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1783912032842636,
      "rewards/margins": 0.02886100485920906,
      "rewards/rejected": -0.20725220441818237,
      "step": 247
    },
    {
      "epoch": 0.3908589440504334,
      "grad_norm": 0.263099730014801,
      "learning_rate": 3.892197425143581e-06,
      "log_odds_chosen": 0.50048828125,
      "log_odds_ratio": -0.4782467186450958,
      "logits/chosen": 0.2478259652853012,
      "logits/rejected": -0.7359952926635742,
      "logps/chosen": -1.708917498588562,
      "logps/rejected": -2.134807825088501,
      "loss": 1.8431,
      "nll_loss": 1.7952405214309692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17089174687862396,
      "rewards/margins": 0.042589038610458374,
      "rewards/rejected": -0.21348077058792114,
      "step": 248
    },
    {
      "epoch": 0.39243498817966904,
      "grad_norm": 0.2791616916656494,
      "learning_rate": 3.890409888038094e-06,
      "log_odds_chosen": 0.387483149766922,
      "log_odds_ratio": -0.5229809284210205,
      "logits/chosen": 0.2733336091041565,
      "logits/rejected": -0.7646632194519043,
      "logps/chosen": -1.7522411346435547,
      "logps/rejected": -2.0853376388549805,
      "loss": 1.8984,
      "nll_loss": 1.8461503982543945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17522412538528442,
      "rewards/margins": 0.03330962732434273,
      "rewards/rejected": -0.20853373408317566,
      "step": 249
    },
    {
      "epoch": 0.39401103230890466,
      "grad_norm": 0.30953434109687805,
      "learning_rate": 3.888608069788831e-06,
      "log_odds_chosen": 0.35872530937194824,
      "log_odds_ratio": -0.5424368381500244,
      "logits/chosen": 0.2008129060268402,
      "logits/rejected": -0.6498639583587646,
      "logps/chosen": -1.7074558734893799,
      "logps/rejected": -2.015054702758789,
      "loss": 1.8569,
      "nll_loss": 1.8026765584945679,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1707455813884735,
      "rewards/margins": 0.030759908258914948,
      "rewards/rejected": -0.20150548219680786,
      "step": 250
    },
    {
      "epoch": 0.3955870764381403,
      "grad_norm": 0.24580441415309906,
      "learning_rate": 3.8867919840076685e-06,
      "log_odds_chosen": 0.44406554102897644,
      "log_odds_ratio": -0.49969637393951416,
      "logits/chosen": 0.23540851473808289,
      "logits/rejected": -0.936524510383606,
      "logps/chosen": -1.6349210739135742,
      "logps/rejected": -2.0072009563446045,
      "loss": 1.7722,
      "nll_loss": 1.7221910953521729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1634921282529831,
      "rewards/margins": 0.03722797706723213,
      "rewards/rejected": -0.20072008669376373,
      "step": 251
    },
    {
      "epoch": 0.3971631205673759,
      "grad_norm": 0.2601582407951355,
      "learning_rate": 3.884961644414267e-06,
      "log_odds_chosen": 0.35995495319366455,
      "log_odds_ratio": -0.5312166213989258,
      "logits/chosen": 0.2544477581977844,
      "logits/rejected": -0.4497869908809662,
      "logps/chosen": -1.7571417093276978,
      "logps/rejected": -2.0635859966278076,
      "loss": 1.8765,
      "nll_loss": 1.8233567476272583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17571419477462769,
      "rewards/margins": 0.030644403770565987,
      "rewards/rejected": -0.20635858178138733,
      "step": 252
    },
    {
      "epoch": 0.3987391646966115,
      "grad_norm": 0.25563278794288635,
      "learning_rate": 3.883117064835967e-06,
      "log_odds_chosen": 0.42896199226379395,
      "log_odds_ratio": -0.5130375027656555,
      "logits/chosen": 0.2095140814781189,
      "logits/rejected": -0.7717199325561523,
      "logps/chosen": -1.6954660415649414,
      "logps/rejected": -2.0621328353881836,
      "loss": 1.8451,
      "nll_loss": 1.7938411235809326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16954661905765533,
      "rewards/margins": 0.03666667640209198,
      "rewards/rejected": -0.20621328055858612,
      "step": 253
    },
    {
      "epoch": 0.40031520882584715,
      "grad_norm": 0.25425294041633606,
      "learning_rate": 3.881258259207688e-06,
      "log_odds_chosen": 0.3588080406188965,
      "log_odds_ratio": -0.5354359149932861,
      "logits/chosen": 0.3078593313694,
      "logits/rejected": -0.8438656330108643,
      "logps/chosen": -1.6679356098175049,
      "logps/rejected": -1.9688389301300049,
      "loss": 1.7965,
      "nll_loss": 1.7429429292678833,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1667935699224472,
      "rewards/margins": 0.030090318992733955,
      "rewards/rejected": -0.196883887052536,
      "step": 254
    },
    {
      "epoch": 0.40189125295508277,
      "grad_norm": 0.2434053122997284,
      "learning_rate": 3.8793852415718165e-06,
      "log_odds_chosen": 0.4722547233104706,
      "log_odds_ratio": -0.4863814413547516,
      "logits/chosen": 0.23671665787696838,
      "logits/rejected": -1.1288728713989258,
      "logps/chosen": -1.5909440517425537,
      "logps/rejected": -1.9825053215026855,
      "loss": 1.7232,
      "nll_loss": 1.6745716333389282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1590944230556488,
      "rewards/margins": 0.039156101644039154,
      "rewards/rejected": -0.19825051724910736,
      "step": 255
    },
    {
      "epoch": 0.4034672970843184,
      "grad_norm": 0.24645063281059265,
      "learning_rate": 3.877498026078107e-06,
      "log_odds_chosen": 0.28664523363113403,
      "log_odds_ratio": -0.5660127401351929,
      "logits/chosen": 0.24130457639694214,
      "logits/rejected": -0.9756399393081665,
      "logps/chosen": -1.7283720970153809,
      "logps/rejected": -1.97283935546875,
      "loss": 1.8639,
      "nll_loss": 1.8072566986083984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17283721268177032,
      "rewards/margins": 0.02444673888385296,
      "rewards/rejected": -0.19728393852710724,
      "step": 256
    },
    {
      "epoch": 0.40504334121355395,
      "grad_norm": 0.24935108423233032,
      "learning_rate": 3.875596626983573e-06,
      "log_odds_chosen": 0.4969295561313629,
      "log_odds_ratio": -0.4879858195781708,
      "logits/chosen": 0.2646740972995758,
      "logits/rejected": -0.6229531764984131,
      "logps/chosen": -1.6504206657409668,
      "logps/rejected": -2.0679101943969727,
      "loss": 1.7856,
      "nll_loss": 1.7367937564849854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16504207253456116,
      "rewards/margins": 0.041748929768800735,
      "rewards/rejected": -0.2067909985780716,
      "step": 257
    },
    {
      "epoch": 0.4066193853427896,
      "grad_norm": 0.25852248072624207,
      "learning_rate": 3.873681058652374e-06,
      "log_odds_chosen": 0.3059096038341522,
      "log_odds_ratio": -0.5598926544189453,
      "logits/chosen": 0.2584393620491028,
      "logits/rejected": -0.7289465665817261,
      "logps/chosen": -1.7720271348953247,
      "logps/rejected": -2.033935308456421,
      "loss": 1.905,
      "nll_loss": 1.8490136861801147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1772027164697647,
      "rewards/margins": 0.026190834119915962,
      "rewards/rejected": -0.20339354872703552,
      "step": 258
    },
    {
      "epoch": 0.4081954294720252,
      "grad_norm": 0.2675853967666626,
      "learning_rate": 3.871751335555715e-06,
      "log_odds_chosen": 0.28866899013519287,
      "log_odds_ratio": -0.5614006519317627,
      "logits/chosen": 0.23755401372909546,
      "logits/rejected": -0.8612145781517029,
      "logps/chosen": -1.7061153650283813,
      "logps/rejected": -1.9488041400909424,
      "loss": 1.8347,
      "nll_loss": 1.7785265445709229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17061154544353485,
      "rewards/margins": 0.024268897250294685,
      "rewards/rejected": -0.1948804259300232,
      "step": 259
    },
    {
      "epoch": 0.4097714736012608,
      "grad_norm": 0.23187151551246643,
      "learning_rate": 3.869807472271731e-06,
      "log_odds_chosen": 0.36366868019104004,
      "log_odds_ratio": -0.5309762358665466,
      "logits/chosen": 0.2599530816078186,
      "logits/rejected": -0.7904383540153503,
      "logps/chosen": -1.6728236675262451,
      "logps/rejected": -1.9797537326812744,
      "loss": 1.834,
      "nll_loss": 1.7808889150619507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.167282372713089,
      "rewards/margins": 0.03069300577044487,
      "rewards/rejected": -0.19797536730766296,
      "step": 260
    },
    {
      "epoch": 0.41134751773049644,
      "grad_norm": 0.23739495873451233,
      "learning_rate": 3.8678494834853826e-06,
      "log_odds_chosen": 0.3631543219089508,
      "log_odds_ratio": -0.534823477268219,
      "logits/chosen": 0.23579055070877075,
      "logits/rejected": -0.7249607443809509,
      "logps/chosen": -1.6771705150604248,
      "logps/rejected": -1.9804139137268066,
      "loss": 1.8111,
      "nll_loss": 1.7576665878295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1677170693874359,
      "rewards/margins": 0.03032434731721878,
      "rewards/rejected": -0.1980414092540741,
      "step": 261
    },
    {
      "epoch": 0.41292356185973206,
      "grad_norm": 0.2685432434082031,
      "learning_rate": 3.865877383988339e-06,
      "log_odds_chosen": 0.45002633333206177,
      "log_odds_ratio": -0.4997020959854126,
      "logits/chosen": 0.24803175032138824,
      "logits/rejected": -0.6998909711837769,
      "logps/chosen": -1.7094076871871948,
      "logps/rejected": -2.0885047912597656,
      "loss": 1.8626,
      "nll_loss": 1.8126096725463867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17094077169895172,
      "rewards/margins": 0.03790968656539917,
      "rewards/rejected": -0.2088504433631897,
      "step": 262
    },
    {
      "epoch": 0.4144996059889677,
      "grad_norm": 0.25303518772125244,
      "learning_rate": 3.863891188678869e-06,
      "log_odds_chosen": 0.18005381524562836,
      "log_odds_ratio": -0.6181609630584717,
      "logits/chosen": 0.20075806975364685,
      "logits/rejected": -0.8267990946769714,
      "logps/chosen": -1.6197738647460938,
      "logps/rejected": -1.7719405889511108,
      "loss": 1.7674,
      "nll_loss": 1.7055647373199463,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16197741031646729,
      "rewards/margins": 0.015216664411127567,
      "rewards/rejected": -0.17719407379627228,
      "step": 263
    },
    {
      "epoch": 0.4160756501182033,
      "grad_norm": 0.241813063621521,
      "learning_rate": 3.8618909125617305e-06,
      "log_odds_chosen": 0.5345890522003174,
      "log_odds_ratio": -0.464138925075531,
      "logits/chosen": 0.2237018644809723,
      "logits/rejected": -1.0322611331939697,
      "logps/chosen": -1.6182307004928589,
      "logps/rejected": -2.0676519870758057,
      "loss": 1.7704,
      "nll_loss": 1.7239396572113037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1618230640888214,
      "rewards/margins": 0.04494212567806244,
      "rewards/rejected": -0.20676518976688385,
      "step": 264
    },
    {
      "epoch": 0.4176516942474389,
      "grad_norm": 0.23744823038578033,
      "learning_rate": 3.859876570748054e-06,
      "log_odds_chosen": 0.36543500423431396,
      "log_odds_ratio": -0.5293257832527161,
      "logits/chosen": 0.2739250063896179,
      "logits/rejected": -0.3711114525794983,
      "logps/chosen": -1.666553020477295,
      "logps/rejected": -1.9728542566299438,
      "loss": 1.818,
      "nll_loss": 1.7650859355926514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16665531694889069,
      "rewards/margins": 0.03063010796904564,
      "rewards/rejected": -0.19728542864322662,
      "step": 265
    },
    {
      "epoch": 0.41922773837667454,
      "grad_norm": 0.23207531869411469,
      "learning_rate": 3.857848178455231e-06,
      "log_odds_chosen": 0.14971187710762024,
      "log_odds_ratio": -0.6274988055229187,
      "logits/chosen": 0.2907387912273407,
      "logits/rejected": -1.0335592031478882,
      "logps/chosen": -1.684551477432251,
      "logps/rejected": -1.8142600059509277,
      "loss": 1.8511,
      "nll_loss": 1.788321614265442,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16845515370368958,
      "rewards/margins": 0.012970849871635437,
      "rewards/rejected": -0.1814260184764862,
      "step": 266
    },
    {
      "epoch": 0.42080378250591016,
      "grad_norm": 0.26370444893836975,
      "learning_rate": 3.855805751006794e-06,
      "log_odds_chosen": 0.5131804943084717,
      "log_odds_ratio": -0.4791393280029297,
      "logits/chosen": 0.27102935314178467,
      "logits/rejected": -0.9959600567817688,
      "logps/chosen": -1.7070389986038208,
      "logps/rejected": -2.1433634757995605,
      "loss": 1.8335,
      "nll_loss": 1.7856334447860718,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17070390284061432,
      "rewards/margins": 0.043632443994283676,
      "rewards/rejected": -0.2143363505601883,
      "step": 267
    },
    {
      "epoch": 0.4223798266351458,
      "grad_norm": 0.22576405107975006,
      "learning_rate": 3.853749303832308e-06,
      "log_odds_chosen": 0.3635735809803009,
      "log_odds_ratio": -0.5336201190948486,
      "logits/chosen": 0.23854336142539978,
      "logits/rejected": -0.5814566016197205,
      "logps/chosen": -1.6284022331237793,
      "logps/rejected": -1.9309828281402588,
      "loss": 1.7683,
      "nll_loss": 1.7149466276168823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16284021735191345,
      "rewards/margins": 0.030258052051067352,
      "rewards/rejected": -0.1930982619524002,
      "step": 268
    },
    {
      "epoch": 0.4239558707643814,
      "grad_norm": 0.25254911184310913,
      "learning_rate": 3.8516788524672495e-06,
      "log_odds_chosen": 0.2377062886953354,
      "log_odds_ratio": -0.5871192216873169,
      "logits/chosen": 0.2799081802368164,
      "logits/rejected": -0.9253388047218323,
      "logps/chosen": -1.7134755849838257,
      "logps/rejected": -1.9147106409072876,
      "loss": 1.8458,
      "nll_loss": 1.7871187925338745,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17134755849838257,
      "rewards/margins": 0.020123496651649475,
      "rewards/rejected": -0.19147105515003204,
      "step": 269
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 0.2403043806552887,
      "learning_rate": 3.849594412552889e-06,
      "log_odds_chosen": 0.3609575927257538,
      "log_odds_ratio": -0.53483647108078,
      "logits/chosen": 0.3030107021331787,
      "logits/rejected": -0.6186437606811523,
      "logps/chosen": -1.614844560623169,
      "logps/rejected": -1.9084908962249756,
      "loss": 1.7544,
      "nll_loss": 1.7009284496307373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16148445010185242,
      "rewards/margins": 0.029364656656980515,
      "rewards/rejected": -0.19084911048412323,
      "step": 270
    },
    {
      "epoch": 0.42710795902285265,
      "grad_norm": 0.2465418577194214,
      "learning_rate": 3.847495999836175e-06,
      "log_odds_chosen": 0.36459216475486755,
      "log_odds_ratio": -0.5316565036773682,
      "logits/chosen": 0.32157933712005615,
      "logits/rejected": -0.9023212194442749,
      "logps/chosen": -1.6977217197418213,
      "logps/rejected": -2.005786895751953,
      "loss": 1.8233,
      "nll_loss": 1.77018404006958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1697721779346466,
      "rewards/margins": 0.030806517228484154,
      "rewards/rejected": -0.2005787044763565,
      "step": 271
    },
    {
      "epoch": 0.42868400315208827,
      "grad_norm": 0.2286456674337387,
      "learning_rate": 3.845383630169613e-06,
      "log_odds_chosen": 0.33313167095184326,
      "log_odds_ratio": -0.5462765693664551,
      "logits/chosen": 0.19532737135887146,
      "logits/rejected": -0.9047931432723999,
      "logps/chosen": -1.6047083139419556,
      "logps/rejected": -1.8802075386047363,
      "loss": 1.7327,
      "nll_loss": 1.6780540943145752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1604708433151245,
      "rewards/margins": 0.02754993550479412,
      "rewards/rejected": -0.1880207657814026,
      "step": 272
    },
    {
      "epoch": 0.4302600472813239,
      "grad_norm": 0.2520315647125244,
      "learning_rate": 3.843257319511147e-06,
      "log_odds_chosen": 0.5703607201576233,
      "log_odds_ratio": -0.4557495713233948,
      "logits/chosen": 0.24878323078155518,
      "logits/rejected": -0.9858651161193848,
      "logps/chosen": -1.6043668985366821,
      "logps/rejected": -2.0794129371643066,
      "loss": 1.7324,
      "nll_loss": 1.6867859363555908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1604366898536682,
      "rewards/margins": 0.04750461503863335,
      "rewards/rejected": -0.20794130861759186,
      "step": 273
    },
    {
      "epoch": 0.4318360914105595,
      "grad_norm": 0.24630145728588104,
      "learning_rate": 3.841117083924039e-06,
      "log_odds_chosen": 0.3485129773616791,
      "log_odds_ratio": -0.5395157337188721,
      "logits/chosen": 0.23884254693984985,
      "logits/rejected": -0.9991633296012878,
      "logps/chosen": -1.644177794456482,
      "logps/rejected": -1.936130166053772,
      "loss": 1.7767,
      "nll_loss": 1.7227089405059814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1644177883863449,
      "rewards/margins": 0.029195237904787064,
      "rewards/rejected": -0.19361303746700287,
      "step": 274
    },
    {
      "epoch": 0.43341213553979513,
      "grad_norm": 0.24250437319278717,
      "learning_rate": 3.838962939576746e-06,
      "log_odds_chosen": 0.4249970316886902,
      "log_odds_ratio": -0.5112988352775574,
      "logits/chosen": 0.20201551914215088,
      "logits/rejected": -0.8135443925857544,
      "logps/chosen": -1.5921757221221924,
      "logps/rejected": -1.9463852643966675,
      "loss": 1.7457,
      "nll_loss": 1.6946040391921997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15921758115291595,
      "rewards/margins": 0.035420943051576614,
      "rewards/rejected": -0.19463855028152466,
      "step": 275
    },
    {
      "epoch": 0.43498817966903075,
      "grad_norm": 0.24594025313854218,
      "learning_rate": 3.8367949027427985e-06,
      "log_odds_chosen": 0.3127194046974182,
      "log_odds_ratio": -0.5534557104110718,
      "logits/chosen": 0.27988913655281067,
      "logits/rejected": -0.676584005355835,
      "logps/chosen": -1.7097309827804565,
      "logps/rejected": -1.9752788543701172,
      "loss": 1.8678,
      "nll_loss": 1.812432050704956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17097310721874237,
      "rewards/margins": 0.026554781943559647,
      "rewards/rejected": -0.19752787053585052,
      "step": 276
    },
    {
      "epoch": 0.43656422379826637,
      "grad_norm": 0.2602437138557434,
      "learning_rate": 3.834612989800681e-06,
      "log_odds_chosen": 0.5669816136360168,
      "log_odds_ratio": -0.45552071928977966,
      "logits/chosen": 0.2907956540584564,
      "logits/rejected": -1.2319718599319458,
      "logps/chosen": -1.6959583759307861,
      "logps/rejected": -2.179285764694214,
      "loss": 1.8138,
      "nll_loss": 1.7682132720947266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1695958375930786,
      "rewards/margins": 0.04833271726965904,
      "rewards/rejected": -0.21792855858802795,
      "step": 277
    },
    {
      "epoch": 0.438140267927502,
      "grad_norm": 0.25470709800720215,
      "learning_rate": 3.832417217233703e-06,
      "log_odds_chosen": 0.3171738386154175,
      "log_odds_ratio": -0.5497609376907349,
      "logits/chosen": 0.24686628580093384,
      "logits/rejected": -0.7249422669410706,
      "logps/chosen": -1.6841633319854736,
      "logps/rejected": -1.9515080451965332,
      "loss": 1.8215,
      "nll_loss": 1.7665718793869019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1684163361787796,
      "rewards/margins": 0.02673446759581566,
      "rewards/rejected": -0.19515080749988556,
      "step": 278
    },
    {
      "epoch": 0.4397163120567376,
      "grad_norm": 0.24597591161727905,
      "learning_rate": 3.8302076016298775e-06,
      "log_odds_chosen": 0.45453959703445435,
      "log_odds_ratio": -0.4950627088546753,
      "logits/chosen": 0.2077048420906067,
      "logits/rejected": -0.9508089423179626,
      "logps/chosen": -1.6135221719741821,
      "logps/rejected": -1.992347240447998,
      "loss": 1.7562,
      "nll_loss": 1.7066751718521118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1613522171974182,
      "rewards/margins": 0.03788250684738159,
      "rewards/rejected": -0.1992347240447998,
      "step": 279
    },
    {
      "epoch": 0.44129235618597323,
      "grad_norm": 0.25559622049331665,
      "learning_rate": 3.827984159681796e-06,
      "log_odds_chosen": 0.3496161699295044,
      "log_odds_ratio": -0.5389132499694824,
      "logits/chosen": 0.19033017754554749,
      "logits/rejected": -0.7456286549568176,
      "logps/chosen": -1.668264389038086,
      "logps/rejected": -1.9605971574783325,
      "loss": 1.7978,
      "nll_loss": 1.743927240371704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16682645678520203,
      "rewards/margins": 0.02923327311873436,
      "rewards/rejected": -0.1960597187280655,
      "step": 280
    },
    {
      "epoch": 0.4428684003152088,
      "grad_norm": 0.2457672506570816,
      "learning_rate": 3.825746908186498e-06,
      "log_odds_chosen": 0.40318137407302856,
      "log_odds_ratio": -0.5202803015708923,
      "logits/chosen": 0.32850149273872375,
      "logits/rejected": -1.0219409465789795,
      "logps/chosen": -1.7146260738372803,
      "logps/rejected": -2.056997299194336,
      "loss": 1.8254,
      "nll_loss": 1.7733912467956543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17146262526512146,
      "rewards/margins": 0.03423710912466049,
      "rewards/rejected": -0.20569972693920135,
      "step": 281
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.26707932353019714,
      "learning_rate": 3.823495864045352e-06,
      "log_odds_chosen": 0.4439901113510132,
      "log_odds_ratio": -0.49818819761276245,
      "logits/chosen": 0.3058054447174072,
      "logits/rejected": -0.41550758481025696,
      "logps/chosen": -1.666752576828003,
      "logps/rejected": -2.037712812423706,
      "loss": 1.7985,
      "nll_loss": 1.748701810836792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16667526960372925,
      "rewards/margins": 0.037096016108989716,
      "rewards/rejected": -0.20377127826213837,
      "step": 282
    },
    {
      "epoch": 0.44602048857368004,
      "grad_norm": 0.2250865399837494,
      "learning_rate": 3.8212310442639205e-06,
      "log_odds_chosen": 0.34101033210754395,
      "log_odds_ratio": -0.5409380793571472,
      "logits/chosen": 0.3130001425743103,
      "logits/rejected": -0.6116319894790649,
      "logps/chosen": -1.6088491678237915,
      "logps/rejected": -1.8918440341949463,
      "loss": 1.7535,
      "nll_loss": 1.6994398832321167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16088491678237915,
      "rewards/margins": 0.028299514204263687,
      "rewards/rejected": -0.18918441236019135,
      "step": 283
    },
    {
      "epoch": 0.44759653270291566,
      "grad_norm": 0.23142075538635254,
      "learning_rate": 3.8189524659518355e-06,
      "log_odds_chosen": 0.40798282623291016,
      "log_odds_ratio": -0.5132877826690674,
      "logits/chosen": 0.3497146964073181,
      "logits/rejected": -0.7581396102905273,
      "logps/chosen": -1.6762844324111938,
      "logps/rejected": -2.017564296722412,
      "loss": 1.8035,
      "nll_loss": 1.752134084701538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1676284223794937,
      "rewards/margins": 0.034127987921237946,
      "rewards/rejected": -0.20175644755363464,
      "step": 284
    },
    {
      "epoch": 0.4491725768321513,
      "grad_norm": 0.2335939258337021,
      "learning_rate": 3.816660146322667e-06,
      "log_odds_chosen": 0.38592225313186646,
      "log_odds_ratio": -0.522415816783905,
      "logits/chosen": 0.28406161069869995,
      "logits/rejected": -0.5710610151290894,
      "logps/chosen": -1.569653868675232,
      "logps/rejected": -1.8868498802185059,
      "loss": 1.7028,
      "nll_loss": 1.6505882740020752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15696537494659424,
      "rewards/margins": 0.03171960636973381,
      "rewards/rejected": -0.18868498504161835,
      "step": 285
    },
    {
      "epoch": 0.4507486209613869,
      "grad_norm": 0.23405610024929047,
      "learning_rate": 3.814354102693797e-06,
      "log_odds_chosen": 0.40164172649383545,
      "log_odds_ratio": -0.514717698097229,
      "logits/chosen": 0.3134067952632904,
      "logits/rejected": -0.824557900428772,
      "logps/chosen": -1.643754005432129,
      "logps/rejected": -1.9766446352005005,
      "loss": 1.7743,
      "nll_loss": 1.7228494882583618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1643753945827484,
      "rewards/margins": 0.033289071172475815,
      "rewards/rejected": -0.19766446948051453,
      "step": 286
    },
    {
      "epoch": 0.4523246650906225,
      "grad_norm": 0.237734854221344,
      "learning_rate": 3.8120343524862814e-06,
      "log_odds_chosen": 0.34567391872406006,
      "log_odds_ratio": -0.5369825959205627,
      "logits/chosen": 0.28361567854881287,
      "logits/rejected": -0.7294724583625793,
      "logps/chosen": -1.6829057931900024,
      "logps/rejected": -1.970367193222046,
      "loss": 1.796,
      "nll_loss": 1.742274522781372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16829057037830353,
      "rewards/margins": 0.02874615229666233,
      "rewards/rejected": -0.1970367133617401,
      "step": 287
    },
    {
      "epoch": 0.45390070921985815,
      "grad_norm": 0.22408141195774078,
      "learning_rate": 3.809700913224726e-06,
      "log_odds_chosen": 0.3844645619392395,
      "log_odds_ratio": -0.5258805751800537,
      "logits/chosen": 0.24493148922920227,
      "logits/rejected": -1.1642123460769653,
      "logps/chosen": -1.6014727354049683,
      "logps/rejected": -1.9189801216125488,
      "loss": 1.7282,
      "nll_loss": 1.6755775213241577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1601472645998001,
      "rewards/margins": 0.03175073489546776,
      "rewards/rejected": -0.19189801812171936,
      "step": 288
    },
    {
      "epoch": 0.45547675334909377,
      "grad_norm": 0.22789910435676575,
      "learning_rate": 3.8073538025371494e-06,
      "log_odds_chosen": 0.48448646068573,
      "log_odds_ratio": -0.48281943798065186,
      "logits/chosen": 0.36718764901161194,
      "logits/rejected": -0.9315266609191895,
      "logps/chosen": -1.579671025276184,
      "logps/rejected": -1.9810744524002075,
      "loss": 1.7167,
      "nll_loss": 1.6684261560440063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15796710550785065,
      "rewards/margins": 0.04014034569263458,
      "rewards/rejected": -0.19810745120048523,
      "step": 289
    },
    {
      "epoch": 0.4570527974783294,
      "grad_norm": 0.24670805037021637,
      "learning_rate": 3.804993038154852e-06,
      "log_odds_chosen": 0.36597153544425964,
      "log_odds_ratio": -0.5341177582740784,
      "logits/chosen": 0.3026730716228485,
      "logits/rejected": -0.8013948202133179,
      "logps/chosen": -1.6119648218154907,
      "logps/rejected": -1.9177809953689575,
      "loss": 1.7591,
      "nll_loss": 1.7057002782821655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1611964851617813,
      "rewards/margins": 0.03058161959052086,
      "rewards/rejected": -0.19177812337875366,
      "step": 290
    },
    {
      "epoch": 0.458628841607565,
      "grad_norm": 0.2390255182981491,
      "learning_rate": 3.8026186379122816e-06,
      "log_odds_chosen": 0.2855335474014282,
      "log_odds_ratio": -0.5630379915237427,
      "logits/chosen": 0.25951528549194336,
      "logits/rejected": -0.8107087016105652,
      "logps/chosen": -1.688565731048584,
      "logps/rejected": -1.9271503686904907,
      "loss": 1.8113,
      "nll_loss": 1.754982590675354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16885659098625183,
      "rewards/margins": 0.023858455941081047,
      "rewards/rejected": -0.19271504878997803,
      "step": 291
    },
    {
      "epoch": 0.46020488573680063,
      "grad_norm": 0.23229312896728516,
      "learning_rate": 3.8002306197468983e-06,
      "log_odds_chosen": 0.17798058688640594,
      "log_odds_ratio": -0.6102758049964905,
      "logits/chosen": 0.275474488735199,
      "logits/rejected": -0.8618179559707642,
      "logps/chosen": -1.6651320457458496,
      "logps/rejected": -1.8112871646881104,
      "loss": 1.806,
      "nll_loss": 1.7449393272399902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16651323437690735,
      "rewards/margins": 0.014615494757890701,
      "rewards/rejected": -0.18112872540950775,
      "step": 292
    },
    {
      "epoch": 0.46178092986603625,
      "grad_norm": 0.2197176069021225,
      "learning_rate": 3.7978290016990367e-06,
      "log_odds_chosen": 0.49173322319984436,
      "log_odds_ratio": -0.4813576638698578,
      "logits/chosen": 0.24893192946910858,
      "logits/rejected": -0.7819869518280029,
      "logps/chosen": -1.6695976257324219,
      "logps/rejected": -2.0870187282562256,
      "loss": 1.7923,
      "nll_loss": 1.744187831878662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1669597625732422,
      "rewards/margins": 0.04174211993813515,
      "rewards/rejected": -0.20870187878608704,
      "step": 293
    },
    {
      "epoch": 0.46335697399527187,
      "grad_norm": 0.24719281494617462,
      "learning_rate": 3.795413801911776e-06,
      "log_odds_chosen": 0.5599408745765686,
      "log_odds_ratio": -0.4586997926235199,
      "logits/chosen": 0.24123334884643555,
      "logits/rejected": -0.8821290135383606,
      "logps/chosen": -1.597358226776123,
      "logps/rejected": -2.068033456802368,
      "loss": 1.741,
      "nll_loss": 1.69508957862854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1597358137369156,
      "rewards/margins": 0.0470675528049469,
      "rewards/rejected": -0.2068033665418625,
      "step": 294
    },
    {
      "epoch": 0.4649330181245075,
      "grad_norm": 0.25565704703330994,
      "learning_rate": 3.7929850386307965e-06,
      "log_odds_chosen": 0.327176958322525,
      "log_odds_ratio": -0.548469066619873,
      "logits/chosen": 0.34679511189460754,
      "logits/rejected": -0.684309720993042,
      "logps/chosen": -1.806984782218933,
      "logps/rejected": -2.0878827571868896,
      "loss": 1.9209,
      "nll_loss": 1.8660597801208496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1806984692811966,
      "rewards/margins": 0.028089802712202072,
      "rewards/rejected": -0.20878829061985016,
      "step": 295
    },
    {
      "epoch": 0.4665090622537431,
      "grad_norm": 0.24570108950138092,
      "learning_rate": 3.790542730204245e-06,
      "log_odds_chosen": 0.3950555920600891,
      "log_odds_ratio": -0.5176329612731934,
      "logits/chosen": 0.29106709361076355,
      "logits/rejected": -0.8974196910858154,
      "logps/chosen": -1.7347073554992676,
      "logps/rejected": -2.0715324878692627,
      "loss": 1.843,
      "nll_loss": 1.7912408113479614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17347073554992676,
      "rewards/margins": 0.033682532608509064,
      "rewards/rejected": -0.20715326070785522,
      "step": 296
    },
    {
      "epoch": 0.46808510638297873,
      "grad_norm": 0.23746472597122192,
      "learning_rate": 3.7880868950825935e-06,
      "log_odds_chosen": 0.4195549488067627,
      "log_odds_ratio": -0.5058358907699585,
      "logits/chosen": 0.26286160945892334,
      "logits/rejected": -0.9936229586601257,
      "logps/chosen": -1.690929889678955,
      "logps/rejected": -2.044651508331299,
      "loss": 1.7908,
      "nll_loss": 1.740168571472168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16909299790859222,
      "rewards/margins": 0.03537215292453766,
      "rewards/rejected": -0.20446515083312988,
      "step": 297
    },
    {
      "epoch": 0.46966115051221435,
      "grad_norm": 0.25234460830688477,
      "learning_rate": 3.7856175518185058e-06,
      "log_odds_chosen": 0.420246422290802,
      "log_odds_ratio": -0.510624349117279,
      "logits/chosen": 0.1483045071363449,
      "logits/rejected": -1.0136771202087402,
      "logps/chosen": -1.6671572923660278,
      "logps/rejected": -2.019413709640503,
      "loss": 1.795,
      "nll_loss": 1.7439560890197754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16671572625637054,
      "rewards/margins": 0.03522564470767975,
      "rewards/rejected": -0.2019413709640503,
      "step": 298
    },
    {
      "epoch": 0.47123719464145,
      "grad_norm": 0.24835272133350372,
      "learning_rate": 3.7831347190666883e-06,
      "log_odds_chosen": 0.4697123169898987,
      "log_odds_ratio": -0.49107876420021057,
      "logits/chosen": 0.20852668583393097,
      "logits/rejected": -0.9135799407958984,
      "logps/chosen": -1.662784457206726,
      "logps/rejected": -2.059032917022705,
      "loss": 1.7932,
      "nll_loss": 1.7441176176071167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16627845168113708,
      "rewards/margins": 0.039624836295843124,
      "rewards/rejected": -0.20590327680110931,
      "step": 299
    },
    {
      "epoch": 0.4728132387706856,
      "grad_norm": 0.23449158668518066,
      "learning_rate": 3.780638415583759e-06,
      "log_odds_chosen": 0.2897394299507141,
      "log_odds_ratio": -0.568331241607666,
      "logits/chosen": 0.32292455434799194,
      "logits/rejected": -0.7160006761550903,
      "logps/chosen": -1.6466126441955566,
      "logps/rejected": -1.8909659385681152,
      "loss": 1.7859,
      "nll_loss": 1.7290998697280884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16466127336025238,
      "rewards/margins": 0.024435309693217278,
      "rewards/rejected": -0.1890965849161148,
      "step": 300
    },
    {
      "epoch": 0.4743892828999212,
      "grad_norm": 0.23816445469856262,
      "learning_rate": 3.7781286602280967e-06,
      "log_odds_chosen": 0.168609157204628,
      "log_odds_ratio": -0.6210045218467712,
      "logits/chosen": 0.28317171335220337,
      "logits/rejected": -0.8950541019439697,
      "logps/chosen": -1.6320608854293823,
      "logps/rejected": -1.7655320167541504,
      "loss": 1.7803,
      "nll_loss": 1.7181925773620605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.163206085562706,
      "rewards/margins": 0.013347111642360687,
      "rewards/rejected": -0.17655321955680847,
      "step": 301
    },
    {
      "epoch": 0.47596532702915684,
      "grad_norm": 0.2272704839706421,
      "learning_rate": 3.7756054719597044e-06,
      "log_odds_chosen": 0.28053295612335205,
      "log_odds_ratio": -0.5875728726387024,
      "logits/chosen": 0.16442207992076874,
      "logits/rejected": -0.9546319246292114,
      "logps/chosen": -1.6610065698623657,
      "logps/rejected": -1.903661847114563,
      "loss": 1.7791,
      "nll_loss": 1.7202967405319214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16610068082809448,
      "rewards/margins": 0.024265505373477936,
      "rewards/rejected": -0.19036617875099182,
      "step": 302
    },
    {
      "epoch": 0.47754137115839246,
      "grad_norm": 0.21994216740131378,
      "learning_rate": 3.773068869840066e-06,
      "log_odds_chosen": 0.21547146141529083,
      "log_odds_ratio": -0.6008081436157227,
      "logits/chosen": 0.37992700934410095,
      "logits/rejected": -0.9604979157447815,
      "logps/chosen": -1.6873159408569336,
      "logps/rejected": -1.8643107414245605,
      "loss": 1.8079,
      "nll_loss": 1.7478294372558594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16873158514499664,
      "rewards/margins": 0.017699476331472397,
      "rewards/rejected": -0.18643106520175934,
      "step": 303
    },
    {
      "epoch": 0.4791174152876281,
      "grad_norm": 0.22366014122962952,
      "learning_rate": 3.770518873031997e-06,
      "log_odds_chosen": 0.4537414312362671,
      "log_odds_ratio": -0.4942602813243866,
      "logits/chosen": 0.15717966854572296,
      "logits/rejected": -0.6233416199684143,
      "logps/chosen": -1.5575112104415894,
      "logps/rejected": -1.9306727647781372,
      "loss": 1.7018,
      "nll_loss": 1.6523317098617554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15575110912322998,
      "rewards/margins": 0.037316154688596725,
      "rewards/rejected": -0.193067267537117,
      "step": 304
    },
    {
      "epoch": 0.4806934594168637,
      "grad_norm": 0.24693521857261658,
      "learning_rate": 3.7679555007995065e-06,
      "log_odds_chosen": 0.4173615872859955,
      "log_odds_ratio": -0.5234081745147705,
      "logits/chosen": 0.291871577501297,
      "logits/rejected": -0.8078293204307556,
      "logps/chosen": -1.6800901889801025,
      "logps/rejected": -2.0314533710479736,
      "loss": 1.8116,
      "nll_loss": 1.7592403888702393,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16800902783870697,
      "rewards/margins": 0.03513630852103233,
      "rewards/rejected": -0.2031453251838684,
      "step": 305
    },
    {
      "epoch": 0.48226950354609927,
      "grad_norm": 0.2487160563468933,
      "learning_rate": 3.7653787725076464e-06,
      "log_odds_chosen": 0.26628607511520386,
      "log_odds_ratio": -0.5737897157669067,
      "logits/chosen": 0.2887229025363922,
      "logits/rejected": -0.7761309146881104,
      "logps/chosen": -1.644072413444519,
      "logps/rejected": -1.8666337728500366,
      "loss": 1.7694,
      "nll_loss": 1.7120240926742554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16440723836421967,
      "rewards/margins": 0.02225613035261631,
      "rewards/rejected": -0.18666337430477142,
      "step": 306
    },
    {
      "epoch": 0.4838455476753349,
      "grad_norm": 0.2717806398868561,
      "learning_rate": 3.7627887076223685e-06,
      "log_odds_chosen": 0.3717851936817169,
      "log_odds_ratio": -0.5291674733161926,
      "logits/chosen": 0.3130492866039276,
      "logits/rejected": -0.5744433999061584,
      "logps/chosen": -1.7094042301177979,
      "logps/rejected": -2.023301124572754,
      "loss": 1.8175,
      "nll_loss": 1.7645916938781738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17094042897224426,
      "rewards/margins": 0.03138966113328934,
      "rewards/rejected": -0.2023300975561142,
      "step": 307
    },
    {
      "epoch": 0.4854215918045705,
      "grad_norm": 0.3177509605884552,
      "learning_rate": 3.7601853257103765e-06,
      "log_odds_chosen": 0.23472489416599274,
      "log_odds_ratio": -0.5888123512268066,
      "logits/chosen": 0.21074482798576355,
      "logits/rejected": -0.8820545077323914,
      "logps/chosen": -1.6076140403747559,
      "logps/rejected": -1.8040858507156372,
      "loss": 1.7182,
      "nll_loss": 1.6593618392944336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16076140105724335,
      "rewards/margins": 0.019647175446152687,
      "rewards/rejected": -0.18040858209133148,
      "step": 308
    },
    {
      "epoch": 0.48699763593380613,
      "grad_norm": 0.22378169000148773,
      "learning_rate": 3.7575686464389767e-06,
      "log_odds_chosen": 0.3512086570262909,
      "log_odds_ratio": -0.5429125428199768,
      "logits/chosen": 0.23526576161384583,
      "logits/rejected": -0.8768814206123352,
      "logps/chosen": -1.589032769203186,
      "logps/rejected": -1.880218267440796,
      "loss": 1.7234,
      "nll_loss": 1.6690884828567505,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1589033007621765,
      "rewards/margins": 0.029118528589606285,
      "rewards/rejected": -0.18802182376384735,
      "step": 309
    },
    {
      "epoch": 0.48857368006304175,
      "grad_norm": 0.21849550306797028,
      "learning_rate": 3.7549386895759315e-06,
      "log_odds_chosen": 0.3859555721282959,
      "log_odds_ratio": -0.5214402079582214,
      "logits/chosen": 0.2112809419631958,
      "logits/rejected": -0.8602145910263062,
      "logps/chosen": -1.5440895557403564,
      "logps/rejected": -1.8589766025543213,
      "loss": 1.6701,
      "nll_loss": 1.61797034740448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15440896153450012,
      "rewards/margins": 0.03148870915174484,
      "rewards/rejected": -0.18589766323566437,
      "step": 310
    },
    {
      "epoch": 0.49014972419227737,
      "grad_norm": 0.21237994730472565,
      "learning_rate": 3.7522954749893086e-06,
      "log_odds_chosen": 0.5937802195549011,
      "log_odds_ratio": -0.4507063925266266,
      "logits/chosen": 0.14280448853969574,
      "logits/rejected": -0.8862166404724121,
      "logps/chosen": -1.571141242980957,
      "logps/rejected": -2.0660626888275146,
      "loss": 1.7067,
      "nll_loss": 1.6616342067718506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15711411833763123,
      "rewards/margins": 0.04949216917157173,
      "rewards/rejected": -0.20660629868507385,
      "step": 311
    },
    {
      "epoch": 0.491725768321513,
      "grad_norm": 0.23776353895664215,
      "learning_rate": 3.749639022647332e-06,
      "log_odds_chosen": 0.3704267144203186,
      "log_odds_ratio": -0.5295000076293945,
      "logits/chosen": 0.21290373802185059,
      "logits/rejected": -0.8199090957641602,
      "logps/chosen": -1.6614030599594116,
      "logps/rejected": -1.9728885889053345,
      "loss": 1.7854,
      "nll_loss": 1.7324740886688232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1661403328180313,
      "rewards/margins": 0.031148536130785942,
      "rewards/rejected": -0.1972888708114624,
      "step": 312
    },
    {
      "epoch": 0.4933018124507486,
      "grad_norm": 0.21524956822395325,
      "learning_rate": 3.7469693526182304e-06,
      "log_odds_chosen": 0.5271570682525635,
      "log_odds_ratio": -0.4816766381263733,
      "logits/chosen": 0.17095641791820526,
      "logits/rejected": -1.1865314245224,
      "logps/chosen": -1.605276346206665,
      "logps/rejected": -2.045314073562622,
      "loss": 1.7166,
      "nll_loss": 1.668409824371338,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16052761673927307,
      "rewards/margins": 0.04400378465652466,
      "rewards/rejected": -0.20453140139579773,
      "step": 313
    },
    {
      "epoch": 0.49487785657998423,
      "grad_norm": 0.22467942535877228,
      "learning_rate": 3.744286485070085e-06,
      "log_odds_chosen": 0.5159242749214172,
      "log_odds_ratio": -0.4703598618507385,
      "logits/chosen": 0.19940146803855896,
      "logits/rejected": -0.9005659818649292,
      "logps/chosen": -1.5551189184188843,
      "logps/rejected": -1.9823968410491943,
      "loss": 1.6713,
      "nll_loss": 1.6242244243621826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15551190078258514,
      "rewards/margins": 0.04272779822349548,
      "rewards/rejected": -0.19823968410491943,
      "step": 314
    },
    {
      "epoch": 0.49645390070921985,
      "grad_norm": 0.24698135256767273,
      "learning_rate": 3.7415904402706795e-06,
      "log_odds_chosen": 0.43156135082244873,
      "log_odds_ratio": -0.508359432220459,
      "logits/chosen": 0.22156882286071777,
      "logits/rejected": -0.8898169994354248,
      "logps/chosen": -1.7517122030258179,
      "logps/rejected": -2.120948076248169,
      "loss": 1.8642,
      "nll_loss": 1.813354253768921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17517122626304626,
      "rewards/margins": 0.036923572421073914,
      "rewards/rejected": -0.21209479868412018,
      "step": 315
    },
    {
      "epoch": 0.4980299448384555,
      "grad_norm": 0.21681058406829834,
      "learning_rate": 3.7388812385873435e-06,
      "log_odds_chosen": 0.36917757987976074,
      "log_odds_ratio": -0.527721643447876,
      "logits/chosen": 0.1808536797761917,
      "logits/rejected": -0.9747561812400818,
      "logps/chosen": -1.5873167514801025,
      "logps/rejected": -1.8916015625,
      "loss": 1.7039,
      "nll_loss": 1.6511293649673462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15873166918754578,
      "rewards/margins": 0.030428504571318626,
      "rewards/rejected": -0.18916018307209015,
      "step": 316
    },
    {
      "epoch": 0.4996059889676911,
      "grad_norm": 0.2535804212093353,
      "learning_rate": 3.7361589004868033e-06,
      "log_odds_chosen": 0.5016902685165405,
      "log_odds_ratio": -0.4793998599052429,
      "logits/chosen": 0.3036730885505676,
      "logits/rejected": -0.9837846159934998,
      "logps/chosen": -1.6742297410964966,
      "logps/rejected": -2.1013123989105225,
      "loss": 1.7751,
      "nll_loss": 1.727131724357605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16742298007011414,
      "rewards/margins": 0.04270825535058975,
      "rewards/rejected": -0.2101312130689621,
      "step": 317
    },
    {
      "epoch": 0.5011820330969267,
      "grad_norm": 0.22353678941726685,
      "learning_rate": 3.733423446535022e-06,
      "log_odds_chosen": 0.5217798352241516,
      "log_odds_ratio": -0.47520411014556885,
      "logits/chosen": 0.2759425938129425,
      "logits/rejected": -1.0702801942825317,
      "logps/chosen": -1.6638113260269165,
      "logps/rejected": -2.1103484630584717,
      "loss": 1.7661,
      "nll_loss": 1.7186000347137451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1663811355829239,
      "rewards/margins": 0.04465372860431671,
      "rewards/rejected": -0.2110348641872406,
      "step": 318
    },
    {
      "epoch": 0.5027580772261623,
      "grad_norm": 0.2162550985813141,
      "learning_rate": 3.7306748973970476e-06,
      "log_odds_chosen": 0.39420896768569946,
      "log_odds_ratio": -0.5247294902801514,
      "logits/chosen": 0.19344046711921692,
      "logits/rejected": -1.2027227878570557,
      "logps/chosen": -1.531542420387268,
      "logps/rejected": -1.8519684076309204,
      "loss": 1.6788,
      "nll_loss": 1.626314640045166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15315423905849457,
      "rewards/margins": 0.03204261139035225,
      "rewards/rejected": -0.18519684672355652,
      "step": 319
    },
    {
      "epoch": 0.5043341213553979,
      "grad_norm": 0.2854284942150116,
      "learning_rate": 3.7279132738368564e-06,
      "log_odds_chosen": 0.19533756375312805,
      "log_odds_ratio": -0.6046186089515686,
      "logits/chosen": 0.25027167797088623,
      "logits/rejected": -0.9639004468917847,
      "logps/chosen": -1.727016568183899,
      "logps/rejected": -1.8893370628356934,
      "loss": 1.8393,
      "nll_loss": 1.778852939605713,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1727016568183899,
      "rewards/margins": 0.016232045367360115,
      "rewards/rejected": -0.18893368542194366,
      "step": 320
    },
    {
      "epoch": 0.5059101654846335,
      "grad_norm": 0.22148051857948303,
      "learning_rate": 3.725138596717195e-06,
      "log_odds_chosen": 0.4484974443912506,
      "log_odds_ratio": -0.4975241720676422,
      "logits/chosen": 0.17572706937789917,
      "logits/rejected": -0.9954120516777039,
      "logps/chosen": -1.577749252319336,
      "logps/rejected": -1.949328899383545,
      "loss": 1.6904,
      "nll_loss": 1.640618085861206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1577749252319336,
      "rewards/margins": 0.03715795278549194,
      "rewards/rejected": -0.19493287801742554,
      "step": 321
    },
    {
      "epoch": 0.5074862096138691,
      "grad_norm": 0.3961973786354065,
      "learning_rate": 3.7223508869994244e-06,
      "log_odds_chosen": 0.4676430821418762,
      "log_odds_ratio": -0.4938008785247803,
      "logits/chosen": 0.21728691458702087,
      "logits/rejected": -1.0851279497146606,
      "logps/chosen": -1.6289960145950317,
      "logps/rejected": -2.0200228691101074,
      "loss": 1.7432,
      "nll_loss": 1.6938555240631104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16289958357810974,
      "rewards/margins": 0.03910268098115921,
      "rewards/rejected": -0.20200228691101074,
      "step": 322
    },
    {
      "epoch": 0.5090622537431048,
      "grad_norm": 0.2561986744403839,
      "learning_rate": 3.7195501657433594e-06,
      "log_odds_chosen": 0.3967494070529938,
      "log_odds_ratio": -0.518577516078949,
      "logits/chosen": 0.16213250160217285,
      "logits/rejected": -1.014851450920105,
      "logps/chosen": -1.6295907497406006,
      "logps/rejected": -1.9586544036865234,
      "loss": 1.7525,
      "nll_loss": 1.700607180595398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16295908391475677,
      "rewards/margins": 0.03290637210011482,
      "rewards/rejected": -0.1958654522895813,
      "step": 323
    },
    {
      "epoch": 0.5106382978723404,
      "grad_norm": 0.2168796956539154,
      "learning_rate": 3.716736454107111e-06,
      "log_odds_chosen": 0.5052562355995178,
      "log_odds_ratio": -0.47744137048721313,
      "logits/chosen": 0.16427958011627197,
      "logits/rejected": -0.8152539134025574,
      "logps/chosen": -1.5257859230041504,
      "logps/rejected": -1.938908576965332,
      "loss": 1.6411,
      "nll_loss": 1.5933436155319214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15257860720157623,
      "rewards/margins": 0.041312284767627716,
      "rewards/rejected": -0.19389088451862335,
      "step": 324
    },
    {
      "epoch": 0.512214342001576,
      "grad_norm": 0.2312452644109726,
      "learning_rate": 3.7139097733469277e-06,
      "log_odds_chosen": 0.5160778760910034,
      "log_odds_ratio": -0.47250521183013916,
      "logits/chosen": 0.18574649095535278,
      "logits/rejected": -0.9940904974937439,
      "logps/chosen": -1.5720903873443604,
      "logps/rejected": -2.0003862380981445,
      "loss": 1.6905,
      "nll_loss": 1.6432417631149292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15720903873443604,
      "rewards/margins": 0.04282958805561066,
      "rewards/rejected": -0.20003864169120789,
      "step": 325
    },
    {
      "epoch": 0.5137903861308116,
      "grad_norm": 0.2351461946964264,
      "learning_rate": 3.711070144817032e-06,
      "log_odds_chosen": 0.4174703359603882,
      "log_odds_ratio": -0.5126141905784607,
      "logits/chosen": 0.16775144636631012,
      "logits/rejected": -0.7969292998313904,
      "logps/chosen": -1.6593265533447266,
      "logps/rejected": -2.0132079124450684,
      "loss": 1.7821,
      "nll_loss": 1.730817437171936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16593264043331146,
      "rewards/margins": 0.03538812696933746,
      "rewards/rejected": -0.20132076740264893,
      "step": 326
    },
    {
      "epoch": 0.5153664302600472,
      "grad_norm": 0.2421269714832306,
      "learning_rate": 3.708217589969461e-06,
      "log_odds_chosen": 0.5103427171707153,
      "log_odds_ratio": -0.4785231649875641,
      "logits/chosen": 0.1360926777124405,
      "logits/rejected": -1.0719431638717651,
      "logps/chosen": -1.636336088180542,
      "logps/rejected": -2.0653886795043945,
      "loss": 1.7628,
      "nll_loss": 1.7149783372879028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16363361477851868,
      "rewards/margins": 0.04290526732802391,
      "rewards/rejected": -0.20653888583183289,
      "step": 327
    },
    {
      "epoch": 0.5169424743892829,
      "grad_norm": 0.2216341197490692,
      "learning_rate": 3.705352130353904e-06,
      "log_odds_chosen": 0.4580025374889374,
      "log_odds_ratio": -0.5018777251243591,
      "logits/chosen": 0.1973736435174942,
      "logits/rejected": -1.0065253973007202,
      "logps/chosen": -1.5510623455047607,
      "logps/rejected": -1.9285662174224854,
      "loss": 1.6825,
      "nll_loss": 1.6323111057281494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15510624647140503,
      "rewards/margins": 0.03775038942694664,
      "rewards/rejected": -0.19285663962364197,
      "step": 328
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 0.23714834451675415,
      "learning_rate": 3.7024737876175404e-06,
      "log_odds_chosen": 0.41599202156066895,
      "log_odds_ratio": -0.5087955594062805,
      "logits/chosen": 0.15757359564304352,
      "logits/rejected": -0.8292059302330017,
      "logps/chosen": -1.6072828769683838,
      "logps/rejected": -1.949842929840088,
      "loss": 1.7153,
      "nll_loss": 1.664394736289978,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16072829067707062,
      "rewards/margins": 0.03425602242350578,
      "rewards/rejected": -0.1949843019247055,
      "step": 329
    },
    {
      "epoch": 0.5200945626477541,
      "grad_norm": 0.23693764209747314,
      "learning_rate": 3.699582583504874e-06,
      "log_odds_chosen": 0.2892705500125885,
      "log_odds_ratio": -0.5659085512161255,
      "logits/chosen": 0.2140631228685379,
      "logits/rejected": -1.0034198760986328,
      "logps/chosen": -1.7431427240371704,
      "logps/rejected": -1.9875526428222656,
      "loss": 1.8448,
      "nll_loss": 1.788230061531067,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17431427538394928,
      "rewards/margins": 0.024440988898277283,
      "rewards/rejected": -0.19875526428222656,
      "step": 330
    },
    {
      "epoch": 0.5216706067769897,
      "grad_norm": 0.2290606051683426,
      "learning_rate": 3.696678539857571e-06,
      "log_odds_chosen": 0.39967021346092224,
      "log_odds_ratio": -0.5233641862869263,
      "logits/chosen": 0.1767245978116989,
      "logits/rejected": -0.9841853380203247,
      "logps/chosen": -1.723608374595642,
      "logps/rejected": -2.065523862838745,
      "loss": 1.8305,
      "nll_loss": 1.7781703472137451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1723608523607254,
      "rewards/margins": 0.0341915525496006,
      "rewards/rejected": -0.2065524011850357,
      "step": 331
    },
    {
      "epoch": 0.5232466509062254,
      "grad_norm": 0.22988593578338623,
      "learning_rate": 3.6937616786142956e-06,
      "log_odds_chosen": 0.526243269443512,
      "log_odds_ratio": -0.4684247672557831,
      "logits/chosen": 0.18895483016967773,
      "logits/rejected": -0.7783670425415039,
      "logps/chosen": -1.5386149883270264,
      "logps/rejected": -1.9738643169403076,
      "loss": 1.6716,
      "nll_loss": 1.6247152090072632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15386147797107697,
      "rewards/margins": 0.043524954468011856,
      "rewards/rejected": -0.19738642871379852,
      "step": 332
    },
    {
      "epoch": 0.524822695035461,
      "grad_norm": 0.2145272046327591,
      "learning_rate": 3.6908320218105393e-06,
      "log_odds_chosen": 0.3370119333267212,
      "log_odds_ratio": -0.5480267405509949,
      "logits/chosen": 0.16891886293888092,
      "logits/rejected": -0.9655537009239197,
      "logps/chosen": -1.5358006954193115,
      "logps/rejected": -1.8141486644744873,
      "loss": 1.6492,
      "nll_loss": 1.5944433212280273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15358006954193115,
      "rewards/margins": 0.0278348159044981,
      "rewards/rejected": -0.1814148873090744,
      "step": 333
    },
    {
      "epoch": 0.5263987391646966,
      "grad_norm": 0.20287442207336426,
      "learning_rate": 3.6878895915784607e-06,
      "log_odds_chosen": 0.48463141918182373,
      "log_odds_ratio": -0.4846850335597992,
      "logits/chosen": 0.2023809552192688,
      "logits/rejected": -0.9322280287742615,
      "logps/chosen": -1.497266411781311,
      "logps/rejected": -1.8917622566223145,
      "loss": 1.6087,
      "nll_loss": 1.560238242149353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14972662925720215,
      "rewards/margins": 0.039449580013751984,
      "rewards/rejected": -0.18917621672153473,
      "step": 334
    },
    {
      "epoch": 0.5279747832939322,
      "grad_norm": 0.21218512952327728,
      "learning_rate": 3.6849344101467147e-06,
      "log_odds_chosen": 0.3986428380012512,
      "log_odds_ratio": -0.520935595035553,
      "logits/chosen": 0.19497668743133545,
      "logits/rejected": -1.0536717176437378,
      "logps/chosen": -1.5831056833267212,
      "logps/rejected": -1.9150642156600952,
      "loss": 1.7035,
      "nll_loss": 1.65139639377594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15831057727336884,
      "rewards/margins": 0.033195845782756805,
      "rewards/rejected": -0.19150641560554504,
      "step": 335
    },
    {
      "epoch": 0.5295508274231678,
      "grad_norm": 0.2202080488204956,
      "learning_rate": 3.6819664998402857e-06,
      "log_odds_chosen": 0.3745194673538208,
      "log_odds_ratio": -0.5271171927452087,
      "logits/chosen": 0.1852867603302002,
      "logits/rejected": -0.9111910462379456,
      "logps/chosen": -1.6216871738433838,
      "logps/rejected": -1.9346219301223755,
      "loss": 1.7383,
      "nll_loss": 1.6856300830841064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1621687263250351,
      "rewards/margins": 0.03129347786307335,
      "rewards/rejected": -0.19346219301223755,
      "step": 336
    },
    {
      "epoch": 0.5311268715524035,
      "grad_norm": 0.2444775551557541,
      "learning_rate": 3.6789858830803186e-06,
      "log_odds_chosen": 0.40843021869659424,
      "log_odds_ratio": -0.5149621963500977,
      "logits/chosen": 0.1969577968120575,
      "logits/rejected": -0.9371582269668579,
      "logps/chosen": -1.6738759279251099,
      "logps/rejected": -2.0160107612609863,
      "loss": 1.7778,
      "nll_loss": 1.7263498306274414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16738761961460114,
      "rewards/margins": 0.03421346843242645,
      "rewards/rejected": -0.2016010731458664,
      "step": 337
    },
    {
      "epoch": 0.5327029156816391,
      "grad_norm": 0.23307378590106964,
      "learning_rate": 3.6759925823839486e-06,
      "log_odds_chosen": 0.31894347071647644,
      "log_odds_ratio": -0.5511658787727356,
      "logits/chosen": 0.16060388088226318,
      "logits/rejected": -0.8142496943473816,
      "logps/chosen": -1.5587881803512573,
      "logps/rejected": -1.8193809986114502,
      "loss": 1.6951,
      "nll_loss": 1.6400277614593506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15587882697582245,
      "rewards/margins": 0.026059266179800034,
      "rewards/rejected": -0.1819380819797516,
      "step": 338
    },
    {
      "epoch": 0.5342789598108747,
      "grad_norm": 0.21192984282970428,
      "learning_rate": 3.672986620364134e-06,
      "log_odds_chosen": 0.4383492171764374,
      "log_odds_ratio": -0.5006621479988098,
      "logits/chosen": 0.19966988265514374,
      "logits/rejected": -0.8241013884544373,
      "logps/chosen": -1.6338273286819458,
      "logps/rejected": -1.9998819828033447,
      "loss": 1.7404,
      "nll_loss": 1.6902958154678345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16338275372982025,
      "rewards/margins": 0.03660544753074646,
      "rewards/rejected": -0.1999882012605667,
      "step": 339
    },
    {
      "epoch": 0.5358550039401103,
      "grad_norm": 0.20865905284881592,
      "learning_rate": 3.669968019729481e-06,
      "log_odds_chosen": 0.5389569997787476,
      "log_odds_ratio": -0.4687870442867279,
      "logits/chosen": 0.14506393671035767,
      "logits/rejected": -1.0461050271987915,
      "logps/chosen": -1.507906198501587,
      "logps/rejected": -1.9527502059936523,
      "loss": 1.6252,
      "nll_loss": 1.5782867670059204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15079061686992645,
      "rewards/margins": 0.044484399259090424,
      "rewards/rejected": -0.19527502357959747,
      "step": 340
    },
    {
      "epoch": 0.5374310480693459,
      "grad_norm": 0.22792787849903107,
      "learning_rate": 3.666936803284076e-06,
      "log_odds_chosen": 0.4631209373474121,
      "log_odds_ratio": -0.4909701347351074,
      "logits/chosen": 0.18787729740142822,
      "logits/rejected": -0.8647685647010803,
      "logps/chosen": -1.6495139598846436,
      "logps/rejected": -2.0405564308166504,
      "loss": 1.7452,
      "nll_loss": 1.6961357593536377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1649513989686966,
      "rewards/margins": 0.039104264229536057,
      "rewards/rejected": -0.20405566692352295,
      "step": 341
    },
    {
      "epoch": 0.5390070921985816,
      "grad_norm": 0.22127310931682587,
      "learning_rate": 3.663892993927312e-06,
      "log_odds_chosen": 0.5525428652763367,
      "log_odds_ratio": -0.45917612314224243,
      "logits/chosen": 0.11797277629375458,
      "logits/rejected": -1.0334317684173584,
      "logps/chosen": -1.596346139907837,
      "logps/rejected": -2.0591015815734863,
      "loss": 1.7139,
      "nll_loss": 1.6679694652557373,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15963461995124817,
      "rewards/margins": 0.04627552628517151,
      "rewards/rejected": -0.20591016113758087,
      "step": 342
    },
    {
      "epoch": 0.5405831363278172,
      "grad_norm": 0.21761682629585266,
      "learning_rate": 3.6608366146537136e-06,
      "log_odds_chosen": 0.5936369299888611,
      "log_odds_ratio": -0.4582027494907379,
      "logits/chosen": 0.09912507981061935,
      "logits/rejected": -0.8791636824607849,
      "logps/chosen": -1.5269497632980347,
      "logps/rejected": -2.0259313583374023,
      "loss": 1.6475,
      "nll_loss": 1.601715087890625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15269500017166138,
      "rewards/margins": 0.04989815130829811,
      "rewards/rejected": -0.2025931477546692,
      "step": 343
    },
    {
      "epoch": 0.5421591804570528,
      "grad_norm": 0.2286389023065567,
      "learning_rate": 3.6577676885527674e-06,
      "log_odds_chosen": 0.46725213527679443,
      "log_odds_ratio": -0.49099448323249817,
      "logits/chosen": 0.12012003362178802,
      "logits/rejected": -1.079842448234558,
      "logps/chosen": -1.5879201889038086,
      "logps/rejected": -1.9761006832122803,
      "loss": 1.6901,
      "nll_loss": 1.6409858465194702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15879201889038086,
      "rewards/margins": 0.03881806507706642,
      "rewards/rejected": -0.1976100653409958,
      "step": 344
    },
    {
      "epoch": 0.5437352245862884,
      "grad_norm": 0.2085123509168625,
      "learning_rate": 3.654686238808744e-06,
      "log_odds_chosen": 0.4555826783180237,
      "log_odds_ratio": -0.49895185232162476,
      "logits/chosen": 0.08309884369373322,
      "logits/rejected": -1.0625288486480713,
      "logps/chosen": -1.6775835752487183,
      "logps/rejected": -2.0619888305664062,
      "loss": 1.7808,
      "nll_loss": 1.7309454679489136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16775836050510406,
      "rewards/margins": 0.038440532982349396,
      "rewards/rejected": -0.20619890093803406,
      "step": 345
    },
    {
      "epoch": 0.545311268715524,
      "grad_norm": 0.19838877022266388,
      "learning_rate": 3.6515922887005245e-06,
      "log_odds_chosen": 0.658145010471344,
      "log_odds_ratio": -0.4212106168270111,
      "logits/chosen": -0.022555839270353317,
      "logits/rejected": -1.0948203802108765,
      "logps/chosen": -1.516296148300171,
      "logps/rejected": -2.05875825881958,
      "loss": 1.6283,
      "nll_loss": 1.5861799716949463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15162962675094604,
      "rewards/margins": 0.054246217012405396,
      "rewards/rejected": -0.20587582886219025,
      "step": 346
    },
    {
      "epoch": 0.5468873128447597,
      "grad_norm": 0.20286299288272858,
      "learning_rate": 3.6484858616014236e-06,
      "log_odds_chosen": 0.2818338871002197,
      "log_odds_ratio": -0.564333438873291,
      "logits/chosen": 0.1583118885755539,
      "logits/rejected": -1.0933054685592651,
      "logps/chosen": -1.6371684074401855,
      "logps/rejected": -1.8697998523712158,
      "loss": 1.7386,
      "nll_loss": 1.6821991205215454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1637168526649475,
      "rewards/margins": 0.02326314151287079,
      "rewards/rejected": -0.1869799643754959,
      "step": 347
    },
    {
      "epoch": 0.5484633569739953,
      "grad_norm": 0.29366299510002136,
      "learning_rate": 3.6453669809790154e-06,
      "log_odds_chosen": 0.32832571864128113,
      "log_odds_ratio": -0.5527113676071167,
      "logits/chosen": 0.1874067187309265,
      "logits/rejected": -0.7553760409355164,
      "logps/chosen": -1.5719748735427856,
      "logps/rejected": -1.8433563709259033,
      "loss": 1.6811,
      "nll_loss": 1.6258182525634766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1571974903345108,
      "rewards/margins": 0.027138162404298782,
      "rewards/rejected": -0.1843356490135193,
      "step": 348
    },
    {
      "epoch": 0.5500394011032309,
      "grad_norm": 0.21022526919841766,
      "learning_rate": 3.642235670394952e-06,
      "log_odds_chosen": 0.4171709418296814,
      "log_odds_ratio": -0.5099711418151855,
      "logits/chosen": 0.164518341422081,
      "logits/rejected": -0.7692568898200989,
      "logps/chosen": -1.5778452157974243,
      "logps/rejected": -1.9177711009979248,
      "loss": 1.6972,
      "nll_loss": 1.646168828010559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15778450667858124,
      "rewards/margins": 0.03399258852005005,
      "rewards/rejected": -0.19177711009979248,
      "step": 349
    },
    {
      "epoch": 0.5516154452324665,
      "grad_norm": 0.199528306722641,
      "learning_rate": 3.63909195350479e-06,
      "log_odds_chosen": 0.39780351519584656,
      "log_odds_ratio": -0.5235289335250854,
      "logits/chosen": 0.21239252388477325,
      "logits/rejected": -0.9313897490501404,
      "logps/chosen": -1.6653385162353516,
      "logps/rejected": -1.996529459953308,
      "loss": 1.7686,
      "nll_loss": 1.7162431478500366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16653385758399963,
      "rewards/margins": 0.033119089901447296,
      "rewards/rejected": -0.19965295493602753,
      "step": 350
    },
    {
      "epoch": 0.5531914893617021,
      "grad_norm": 0.21873773634433746,
      "learning_rate": 3.635935854057809e-06,
      "log_odds_chosen": 0.3728238642215729,
      "log_odds_ratio": -0.5321751236915588,
      "logits/chosen": 0.04262760281562805,
      "logits/rejected": -0.6860441565513611,
      "logps/chosen": -1.6196409463882446,
      "logps/rejected": -1.930510401725769,
      "loss": 1.731,
      "nll_loss": 1.6777571439743042,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16196408867835999,
      "rewards/margins": 0.031086942180991173,
      "rewards/rejected": -0.1930510252714157,
      "step": 351
    },
    {
      "epoch": 0.5547675334909378,
      "grad_norm": 0.20314662158489227,
      "learning_rate": 3.6327673958968327e-06,
      "log_odds_chosen": 0.49560269713401794,
      "log_odds_ratio": -0.4786853790283203,
      "logits/chosen": 0.07317562401294708,
      "logits/rejected": -0.9375707507133484,
      "logps/chosen": -1.498767375946045,
      "logps/rejected": -1.9013431072235107,
      "loss": 1.6284,
      "nll_loss": 1.5804861783981323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14987674355506897,
      "rewards/margins": 0.04025757685303688,
      "rewards/rejected": -0.19013431668281555,
      "step": 352
    },
    {
      "epoch": 0.5563435776201734,
      "grad_norm": 0.2797704339027405,
      "learning_rate": 3.6295866029580483e-06,
      "log_odds_chosen": 0.4891226589679718,
      "log_odds_ratio": -0.481170117855072,
      "logits/chosen": 0.07735095918178558,
      "logits/rejected": -1.0302777290344238,
      "logps/chosen": -1.5447524785995483,
      "logps/rejected": -1.9455407857894897,
      "loss": 1.6431,
      "nll_loss": 1.5949952602386475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15447524189949036,
      "rewards/margins": 0.040078844875097275,
      "rewards/rejected": -0.19455409049987793,
      "step": 353
    },
    {
      "epoch": 0.557919621749409,
      "grad_norm": 0.2015417069196701,
      "learning_rate": 3.626393499270829e-06,
      "log_odds_chosen": 0.4410100281238556,
      "log_odds_ratio": -0.5045400261878967,
      "logits/chosen": 0.08361411094665527,
      "logits/rejected": -0.9850642681121826,
      "logps/chosen": -1.4818400144577026,
      "logps/rejected": -1.8396446704864502,
      "loss": 1.6143,
      "nll_loss": 1.5638469457626343,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14818400144577026,
      "rewards/margins": 0.03578047454357147,
      "rewards/rejected": -0.18396447598934174,
      "step": 354
    },
    {
      "epoch": 0.5594956658786446,
      "grad_norm": 0.20118573307991028,
      "learning_rate": 3.6231881089575466e-06,
      "log_odds_chosen": 0.6061029434204102,
      "log_odds_ratio": -0.4385111927986145,
      "logits/chosen": 0.02086889185011387,
      "logits/rejected": -1.1207969188690186,
      "logps/chosen": -1.5292540788650513,
      "logps/rejected": -2.0326905250549316,
      "loss": 1.612,
      "nll_loss": 1.5681171417236328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15292541682720184,
      "rewards/margins": 0.05034365504980087,
      "rewards/rejected": -0.20326906442642212,
      "step": 355
    },
    {
      "epoch": 0.5610717100078803,
      "grad_norm": 0.21071591973304749,
      "learning_rate": 3.6199704562333945e-06,
      "log_odds_chosen": 0.45574647188186646,
      "log_odds_ratio": -0.49790191650390625,
      "logits/chosen": 0.054458700120449066,
      "logits/rejected": -0.9820718169212341,
      "logps/chosen": -1.5057026147842407,
      "logps/rejected": -1.8732877969741821,
      "loss": 1.6173,
      "nll_loss": 1.5674865245819092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15057027339935303,
      "rewards/margins": 0.03675851225852966,
      "rewards/rejected": -0.1873287856578827,
      "step": 356
    },
    {
      "epoch": 0.5626477541371159,
      "grad_norm": 0.20148809254169464,
      "learning_rate": 3.6167405654062024e-06,
      "log_odds_chosen": 0.4633074402809143,
      "log_odds_ratio": -0.5014227032661438,
      "logits/chosen": 0.11039365828037262,
      "logits/rejected": -0.9207887649536133,
      "logps/chosen": -1.556457757949829,
      "logps/rejected": -1.9362834692001343,
      "loss": 1.6571,
      "nll_loss": 1.606947660446167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15564577281475067,
      "rewards/margins": 0.03798258304595947,
      "rewards/rejected": -0.19362835586071014,
      "step": 357
    },
    {
      "epoch": 0.5642237982663515,
      "grad_norm": 0.18568812310695648,
      "learning_rate": 3.6134984608762515e-06,
      "log_odds_chosen": 0.4947914183139801,
      "log_odds_ratio": -0.47895732522010803,
      "logits/chosen": 0.09608149528503418,
      "logits/rejected": -1.0143461227416992,
      "logps/chosen": -1.4812129735946655,
      "logps/rejected": -1.881905436515808,
      "loss": 1.6029,
      "nll_loss": 1.5549757480621338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14812129735946655,
      "rewards/margins": 0.04006926715373993,
      "rewards/rejected": -0.18819056451320648,
      "step": 358
    },
    {
      "epoch": 0.5657998423955871,
      "grad_norm": 0.2126130759716034,
      "learning_rate": 3.6102441671360945e-06,
      "log_odds_chosen": 0.47413334250450134,
      "log_odds_ratio": -0.4936099946498871,
      "logits/chosen": 0.06261715292930603,
      "logits/rejected": -0.8793343305587769,
      "logps/chosen": -1.6251752376556396,
      "logps/rejected": -2.0224335193634033,
      "loss": 1.7191,
      "nll_loss": 1.6696956157684326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1625175178050995,
      "rewards/margins": 0.03972584009170532,
      "rewards/rejected": -0.2022433578968048,
      "step": 359
    },
    {
      "epoch": 0.5673758865248227,
      "grad_norm": 0.20976465940475464,
      "learning_rate": 3.6069777087703654e-06,
      "log_odds_chosen": 0.4963104724884033,
      "log_odds_ratio": -0.48178282380104065,
      "logits/chosen": 0.0887591764330864,
      "logits/rejected": -0.8758858442306519,
      "logps/chosen": -1.5486314296722412,
      "logps/rejected": -1.9577083587646484,
      "loss": 1.6676,
      "nll_loss": 1.619391679763794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548631489276886,
      "rewards/margins": 0.04090770334005356,
      "rewards/rejected": -0.19577085971832275,
      "step": 360
    },
    {
      "epoch": 0.5689519306540584,
      "grad_norm": 0.20991873741149902,
      "learning_rate": 3.6036991104555973e-06,
      "log_odds_chosen": 0.4287150204181671,
      "log_odds_ratio": -0.5141339898109436,
      "logits/chosen": 0.09619182348251343,
      "logits/rejected": -0.939784824848175,
      "logps/chosen": -1.6915943622589111,
      "logps/rejected": -2.055938720703125,
      "loss": 1.7819,
      "nll_loss": 1.730458378791809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16915945708751678,
      "rewards/margins": 0.03643442690372467,
      "rewards/rejected": -0.20559386909008026,
      "step": 361
    },
    {
      "epoch": 0.570527974783294,
      "grad_norm": 0.2047056257724762,
      "learning_rate": 3.600408396960034e-06,
      "log_odds_chosen": 0.48920494318008423,
      "log_odds_ratio": -0.4854881465435028,
      "logits/chosen": 0.11697036772966385,
      "logits/rejected": -0.6390340328216553,
      "logps/chosen": -1.4950973987579346,
      "logps/rejected": -1.8872171640396118,
      "loss": 1.5913,
      "nll_loss": 1.5427401065826416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495097428560257,
      "rewards/margins": 0.03921198099851608,
      "rewards/rejected": -0.18872171640396118,
      "step": 362
    },
    {
      "epoch": 0.5721040189125296,
      "grad_norm": 0.1965114176273346,
      "learning_rate": 3.5971055931434447e-06,
      "log_odds_chosen": 0.43638381361961365,
      "log_odds_ratio": -0.5067461729049683,
      "logits/chosen": 0.13429589569568634,
      "logits/rejected": -0.8910205364227295,
      "logps/chosen": -1.5401540994644165,
      "logps/rejected": -1.9012306928634644,
      "loss": 1.6487,
      "nll_loss": 1.5980687141418457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1540154069662094,
      "rewards/margins": 0.03610766679048538,
      "rewards/rejected": -0.1901230663061142,
      "step": 363
    },
    {
      "epoch": 0.5736800630417652,
      "grad_norm": 0.19536329805850983,
      "learning_rate": 3.5937907239569343e-06,
      "log_odds_chosen": 0.45536214113235474,
      "log_odds_ratio": -0.5008538365364075,
      "logits/chosen": 0.04754452407360077,
      "logits/rejected": -0.9931653141975403,
      "logps/chosen": -1.592581868171692,
      "logps/rejected": -1.9675819873809814,
      "loss": 1.7023,
      "nll_loss": 1.65225350856781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15925820171833038,
      "rewards/margins": 0.03749999776482582,
      "rewards/rejected": -0.1967581808567047,
      "step": 364
    },
    {
      "epoch": 0.5752561071710008,
      "grad_norm": 0.21669350564479828,
      "learning_rate": 3.5904638144427572e-06,
      "log_odds_chosen": 0.2608916759490967,
      "log_odds_ratio": -0.5785377621650696,
      "logits/chosen": 0.10151364654302597,
      "logits/rejected": -0.7654220461845398,
      "logps/chosen": -1.651747703552246,
      "logps/rejected": -1.8675367832183838,
      "loss": 1.7496,
      "nll_loss": 1.6917589902877808,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16517476737499237,
      "rewards/margins": 0.021578924730420113,
      "rewards/rejected": -0.18675370514392853,
      "step": 365
    },
    {
      "epoch": 0.5768321513002365,
      "grad_norm": 0.20522025227546692,
      "learning_rate": 3.5871248897341246e-06,
      "log_odds_chosen": 0.48859813809394836,
      "log_odds_ratio": -0.4839872121810913,
      "logits/chosen": 0.040113721042871475,
      "logits/rejected": -0.7136132121086121,
      "logps/chosen": -1.4968351125717163,
      "logps/rejected": -1.9003262519836426,
      "loss": 1.6213,
      "nll_loss": 1.572914958000183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14968350529670715,
      "rewards/margins": 0.04034912586212158,
      "rewards/rejected": -0.19003261625766754,
      "step": 366
    },
    {
      "epoch": 0.578408195429472,
      "grad_norm": 0.20791637897491455,
      "learning_rate": 3.5837739750550182e-06,
      "log_odds_chosen": 0.47811540961265564,
      "log_odds_ratio": -0.4912639856338501,
      "logits/chosen": 0.032442331314086914,
      "logits/rejected": -1.2634479999542236,
      "logps/chosen": -1.5675827264785767,
      "logps/rejected": -1.966813325881958,
      "loss": 1.6687,
      "nll_loss": 1.619585394859314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15675827860832214,
      "rewards/margins": 0.03992307186126709,
      "rewards/rejected": -0.19668133556842804,
      "step": 367
    },
    {
      "epoch": 0.5799842395587076,
      "grad_norm": 0.18128971755504608,
      "learning_rate": 3.5804110957199977e-06,
      "log_odds_chosen": 0.5172293782234192,
      "log_odds_ratio": -0.47360023856163025,
      "logits/chosen": 0.12969666719436646,
      "logits/rejected": -0.8731107711791992,
      "logps/chosen": -1.549896001815796,
      "logps/rejected": -1.9799461364746094,
      "loss": 1.6502,
      "nll_loss": 1.6028105020523071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1549895852804184,
      "rewards/margins": 0.043005019426345825,
      "rewards/rejected": -0.19799461960792542,
      "step": 368
    },
    {
      "epoch": 0.5815602836879432,
      "grad_norm": 0.1900000423192978,
      "learning_rate": 3.577036277134011e-06,
      "log_odds_chosen": 0.5826700925827026,
      "log_odds_ratio": -0.4474554657936096,
      "logits/chosen": 0.06492473185062408,
      "logits/rejected": -1.0637844800949097,
      "logps/chosen": -1.5324748754501343,
      "logps/rejected": -2.014566659927368,
      "loss": 1.6468,
      "nll_loss": 1.60209059715271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15324749052524567,
      "rewards/margins": 0.04820918291807175,
      "rewards/rejected": -0.20145665109157562,
      "step": 369
    },
    {
      "epoch": 0.5831363278171788,
      "grad_norm": 0.20140865445137024,
      "learning_rate": 3.5736495447922e-06,
      "log_odds_chosen": 0.36809372901916504,
      "log_odds_ratio": -0.5302509069442749,
      "logits/chosen": 0.06567880511283875,
      "logits/rejected": -1.0636937618255615,
      "logps/chosen": -1.659379243850708,
      "logps/rejected": -1.9658015966415405,
      "loss": 1.7622,
      "nll_loss": 1.709184169769287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16593793034553528,
      "rewards/margins": 0.03064224123954773,
      "rewards/rejected": -0.196580171585083,
      "step": 370
    },
    {
      "epoch": 0.5847123719464145,
      "grad_norm": 0.1792723536491394,
      "learning_rate": 3.5702509242797096e-06,
      "log_odds_chosen": 0.6814740300178528,
      "log_odds_ratio": -0.4248891770839691,
      "logits/chosen": 0.039288341999053955,
      "logits/rejected": -1.0555591583251953,
      "logps/chosen": -1.4857667684555054,
      "logps/rejected": -2.0495104789733887,
      "loss": 1.5995,
      "nll_loss": 1.5570186376571655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14857669174671173,
      "rewards/margins": 0.05637436360120773,
      "rewards/rejected": -0.20495106279850006,
      "step": 371
    },
    {
      "epoch": 0.5862884160756501,
      "grad_norm": 0.1895635426044464,
      "learning_rate": 3.566840441271495e-06,
      "log_odds_chosen": 0.6294078230857849,
      "log_odds_ratio": -0.4429580569267273,
      "logits/chosen": 0.07919707894325256,
      "logits/rejected": -1.0324056148529053,
      "logps/chosen": -1.5050973892211914,
      "logps/rejected": -2.0217623710632324,
      "loss": 1.5956,
      "nll_loss": 1.5513163805007935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15050974488258362,
      "rewards/margins": 0.05166648328304291,
      "rewards/rejected": -0.20217622816562653,
      "step": 372
    },
    {
      "epoch": 0.5878644602048857,
      "grad_norm": 0.1982351839542389,
      "learning_rate": 3.5634181215321265e-06,
      "log_odds_chosen": 0.5948350429534912,
      "log_odds_ratio": -0.4424865245819092,
      "logits/chosen": 0.10533066838979721,
      "logits/rejected": -0.964317798614502,
      "logps/chosen": -1.5590019226074219,
      "logps/rejected": -2.0552213191986084,
      "loss": 1.6722,
      "nll_loss": 1.6279126405715942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15590018033981323,
      "rewards/margins": 0.049621954560279846,
      "rewards/rejected": -0.20552213490009308,
      "step": 373
    },
    {
      "epoch": 0.5894405043341213,
      "grad_norm": 0.18995660543441772,
      "learning_rate": 3.5599839909155947e-06,
      "log_odds_chosen": 0.48201611638069153,
      "log_odds_ratio": -0.4901425540447235,
      "logits/chosen": 0.06920170038938522,
      "logits/rejected": -0.9864767789840698,
      "logps/chosen": -1.5232574939727783,
      "logps/rejected": -1.920086145401001,
      "loss": 1.6269,
      "nll_loss": 1.5779118537902832,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15232573449611664,
      "rewards/margins": 0.03968288004398346,
      "rewards/rejected": -0.1920086145401001,
      "step": 374
    },
    {
      "epoch": 0.5910165484633569,
      "grad_norm": 0.1883167177438736,
      "learning_rate": 3.556538075365116e-06,
      "log_odds_chosen": 0.5346459150314331,
      "log_odds_ratio": -0.4716908633708954,
      "logits/chosen": 0.10186448693275452,
      "logits/rejected": -0.9781060814857483,
      "logps/chosen": -1.5155980587005615,
      "logps/rejected": -1.952977180480957,
      "loss": 1.6202,
      "nll_loss": 1.5730595588684082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15155981481075287,
      "rewards/margins": 0.043737903237342834,
      "rewards/rejected": -0.1952977180480957,
      "step": 375
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 0.1945546567440033,
      "learning_rate": 3.5530804009129367e-06,
      "log_odds_chosen": 0.4805316925048828,
      "log_odds_ratio": -0.4836026430130005,
      "logits/chosen": 0.059399835765361786,
      "logits/rejected": -0.7888279557228088,
      "logps/chosen": -1.5551681518554688,
      "logps/rejected": -1.9524855613708496,
      "loss": 1.6452,
      "nll_loss": 1.5968666076660156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15551680326461792,
      "rewards/margins": 0.039731740951538086,
      "rewards/rejected": -0.195248544216156,
      "step": 376
    },
    {
      "epoch": 0.5941686367218282,
      "grad_norm": 0.1976991593837738,
      "learning_rate": 3.5496109936801368e-06,
      "log_odds_chosen": 0.4780104160308838,
      "log_odds_ratio": -0.4899000823497772,
      "logits/chosen": 0.014288029633462429,
      "logits/rejected": -1.0679166316986084,
      "logps/chosen": -1.5730230808258057,
      "logps/rejected": -1.9747998714447021,
      "loss": 1.6721,
      "nll_loss": 1.6231590509414673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15730230510234833,
      "rewards/margins": 0.040177665650844574,
      "rewards/rejected": -0.1974799782037735,
      "step": 377
    },
    {
      "epoch": 0.5957446808510638,
      "grad_norm": 0.20086143910884857,
      "learning_rate": 3.546129879876429e-06,
      "log_odds_chosen": 0.345024049282074,
      "log_odds_ratio": -0.547674298286438,
      "logits/chosen": 0.11364105343818665,
      "logits/rejected": -0.8114718794822693,
      "logps/chosen": -1.6343905925750732,
      "logps/rejected": -1.9218847751617432,
      "loss": 1.73,
      "nll_loss": 1.6752517223358154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16343903541564941,
      "rewards/margins": 0.028749439865350723,
      "rewards/rejected": -0.19218848645687103,
      "step": 378
    },
    {
      "epoch": 0.5973207249802994,
      "grad_norm": 0.21694695949554443,
      "learning_rate": 3.5426370857999662e-06,
      "log_odds_chosen": 0.3613109886646271,
      "log_odds_ratio": -0.5329811573028564,
      "logits/chosen": 0.05423086881637573,
      "logits/rejected": -0.89935302734375,
      "logps/chosen": -1.6174565553665161,
      "logps/rejected": -1.9179959297180176,
      "loss": 1.7236,
      "nll_loss": 1.6703437566757202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16174568235874176,
      "rewards/margins": 0.030053915455937386,
      "rewards/rejected": -0.191799595952034,
      "step": 379
    },
    {
      "epoch": 0.598896769109535,
      "grad_norm": 0.20389492809772491,
      "learning_rate": 3.53913263783714e-06,
      "log_odds_chosen": 0.3913048505783081,
      "log_odds_ratio": -0.5298315286636353,
      "logits/chosen": 0.03343074396252632,
      "logits/rejected": -0.9829487800598145,
      "logps/chosen": -1.5685036182403564,
      "logps/rejected": -1.8965883255004883,
      "loss": 1.6643,
      "nll_loss": 1.6112816333770752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15685035288333893,
      "rewards/margins": 0.032808490097522736,
      "rewards/rejected": -0.18965885043144226,
      "step": 380
    },
    {
      "epoch": 0.6004728132387707,
      "grad_norm": 0.18716998398303986,
      "learning_rate": 3.53561656246238e-06,
      "log_odds_chosen": 0.6300040483474731,
      "log_odds_ratio": -0.4305461347103119,
      "logits/chosen": -0.0723160058259964,
      "logits/rejected": -1.210856556892395,
      "logps/chosen": -1.5864709615707397,
      "logps/rejected": -2.1141700744628906,
      "loss": 1.6812,
      "nll_loss": 1.638156771659851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1586471050977707,
      "rewards/margins": 0.052769921720027924,
      "rewards/rejected": -0.2114170342683792,
      "step": 381
    },
    {
      "epoch": 0.6020488573680063,
      "grad_norm": 0.18554647266864777,
      "learning_rate": 3.532088886237956e-06,
      "log_odds_chosen": 0.5184412002563477,
      "log_odds_ratio": -0.47040432691574097,
      "logits/chosen": 0.008511025458574295,
      "logits/rejected": -1.1571906805038452,
      "logps/chosen": -1.6124807596206665,
      "logps/rejected": -2.04634428024292,
      "loss": 1.6949,
      "nll_loss": 1.6479085683822632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1612480878829956,
      "rewards/margins": 0.04338634014129639,
      "rewards/rejected": -0.2046344131231308,
      "step": 382
    },
    {
      "epoch": 0.6036249014972419,
      "grad_norm": 0.17985667288303375,
      "learning_rate": 3.528549635813778e-06,
      "log_odds_chosen": 0.5406981706619263,
      "log_odds_ratio": -0.46587079763412476,
      "logits/chosen": -0.046166982501745224,
      "logits/rejected": -0.9862831830978394,
      "logps/chosen": -1.4486740827560425,
      "logps/rejected": -1.8883919715881348,
      "loss": 1.5544,
      "nll_loss": 1.507815957069397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.144867405295372,
      "rewards/margins": 0.04397178441286087,
      "rewards/rejected": -0.18883919715881348,
      "step": 383
    },
    {
      "epoch": 0.6052009456264775,
      "grad_norm": 0.19946593046188354,
      "learning_rate": 3.524998837927192e-06,
      "log_odds_chosen": 0.5522040128707886,
      "log_odds_ratio": -0.45645731687545776,
      "logits/chosen": 0.0009910385124385357,
      "logits/rejected": -0.9322502017021179,
      "logps/chosen": -1.5654282569885254,
      "logps/rejected": -2.024646759033203,
      "loss": 1.6474,
      "nll_loss": 1.601776361465454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1565428376197815,
      "rewards/margins": 0.04592186585068703,
      "rewards/rejected": -0.20246468484401703,
      "step": 384
    },
    {
      "epoch": 0.6067769897557131,
      "grad_norm": 0.18796052038669586,
      "learning_rate": 3.5214365194027797e-06,
      "log_odds_chosen": 0.565650463104248,
      "log_odds_ratio": -0.4533955752849579,
      "logits/chosen": -0.014517467468976974,
      "logits/rejected": -1.0332865715026855,
      "logps/chosen": -1.492905855178833,
      "logps/rejected": -1.9580696821212769,
      "loss": 1.5881,
      "nll_loss": 1.5427998304367065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14929059147834778,
      "rewards/margins": 0.04651637375354767,
      "rewards/rejected": -0.19580696523189545,
      "step": 385
    },
    {
      "epoch": 0.6083530338849488,
      "grad_norm": 0.19329367578029633,
      "learning_rate": 3.517862707152157e-06,
      "log_odds_chosen": 0.4329968988895416,
      "log_odds_ratio": -0.5075357556343079,
      "logits/chosen": 0.07324811816215515,
      "logits/rejected": -0.848515510559082,
      "logps/chosen": -1.6198889017105103,
      "logps/rejected": -1.980916976928711,
      "loss": 1.7035,
      "nll_loss": 1.6527172327041626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16198891401290894,
      "rewards/margins": 0.03610279783606529,
      "rewards/rejected": -0.19809171557426453,
      "step": 386
    },
    {
      "epoch": 0.6099290780141844,
      "grad_norm": 0.2003830224275589,
      "learning_rate": 3.5142774281737674e-06,
      "log_odds_chosen": 0.6210008859634399,
      "log_odds_ratio": -0.4322190284729004,
      "logits/chosen": -0.0007459288462996483,
      "logits/rejected": -0.9446620345115662,
      "logps/chosen": -1.6110001802444458,
      "logps/rejected": -2.1332759857177734,
      "loss": 1.6895,
      "nll_loss": 1.6462812423706055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16110001504421234,
      "rewards/margins": 0.052227575331926346,
      "rewards/rejected": -0.2133275866508484,
      "step": 387
    },
    {
      "epoch": 0.61150512214342,
      "grad_norm": 0.17635990679264069,
      "learning_rate": 3.5106807095526817e-06,
      "log_odds_chosen": 0.5996285080909729,
      "log_odds_ratio": -0.44902303814888,
      "logits/chosen": -0.02553224191069603,
      "logits/rejected": -1.1442608833312988,
      "logps/chosen": -1.5787206888198853,
      "logps/rejected": -2.0769317150115967,
      "loss": 1.673,
      "nll_loss": 1.6281229257583618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1578720659017563,
      "rewards/margins": 0.04982110857963562,
      "rewards/rejected": -0.2076931744813919,
      "step": 388
    },
    {
      "epoch": 0.6130811662726556,
      "grad_norm": 0.17955166101455688,
      "learning_rate": 3.5070725784603905e-06,
      "log_odds_chosen": 0.5273114442825317,
      "log_odds_ratio": -0.4709566831588745,
      "logits/chosen": -0.12750433385372162,
      "logits/rejected": -0.8868665099143982,
      "logps/chosen": -1.450034737586975,
      "logps/rejected": -1.878908634185791,
      "loss": 1.5397,
      "nll_loss": 1.4926133155822754,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14500348269939423,
      "rewards/margins": 0.0428873747587204,
      "rewards/rejected": -0.18789085745811462,
      "step": 389
    },
    {
      "epoch": 0.6146572104018913,
      "grad_norm": 0.206462100148201,
      "learning_rate": 3.503453062154602e-06,
      "log_odds_chosen": 0.4430813491344452,
      "log_odds_ratio": -0.5067101120948792,
      "logits/chosen": -0.01937907561659813,
      "logits/rejected": -0.8744063973426819,
      "logps/chosen": -1.6130276918411255,
      "logps/rejected": -1.989070177078247,
      "loss": 1.7124,
      "nll_loss": 1.6617623567581177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16130277514457703,
      "rewards/margins": 0.037604257464408875,
      "rewards/rejected": -0.1989070326089859,
      "step": 390
    },
    {
      "epoch": 0.6162332545311269,
      "grad_norm": 0.1900879293680191,
      "learning_rate": 3.499822187979032e-06,
      "log_odds_chosen": 0.42407649755477905,
      "log_odds_ratio": -0.5121719837188721,
      "logits/chosen": 0.0466628335416317,
      "logits/rejected": -1.085059404373169,
      "logps/chosen": -1.5678186416625977,
      "logps/rejected": -1.919469952583313,
      "loss": 1.675,
      "nll_loss": 1.6237623691558838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1567818522453308,
      "rewards/margins": 0.03516513854265213,
      "rewards/rejected": -0.19194699823856354,
      "step": 391
    },
    {
      "epoch": 0.6178092986603625,
      "grad_norm": 0.17979402840137482,
      "learning_rate": 3.496179983363202e-06,
      "log_odds_chosen": 0.39420366287231445,
      "log_odds_ratio": -0.5192743539810181,
      "logits/chosen": 0.009703043848276138,
      "logits/rejected": -0.9731006026268005,
      "logps/chosen": -1.5731898546218872,
      "logps/rejected": -1.8994686603546143,
      "loss": 1.66,
      "nll_loss": 1.6080539226531982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15731897950172424,
      "rewards/margins": 0.0326278880238533,
      "rewards/rejected": -0.18994688987731934,
      "step": 392
    },
    {
      "epoch": 0.6193853427895981,
      "grad_norm": 0.20772549510002136,
      "learning_rate": 3.4925264758222268e-06,
      "log_odds_chosen": 0.6100749373435974,
      "log_odds_ratio": -0.4368566870689392,
      "logits/chosen": -0.04344996437430382,
      "logits/rejected": -0.844353199005127,
      "logps/chosen": -1.5049924850463867,
      "logps/rejected": -2.0098416805267334,
      "loss": 1.6044,
      "nll_loss": 1.5606787204742432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15049925446510315,
      "rewards/margins": 0.05048491805791855,
      "rewards/rejected": -0.2009841501712799,
      "step": 393
    },
    {
      "epoch": 0.6209613869188337,
      "grad_norm": 0.1882377713918686,
      "learning_rate": 3.488861692956611e-06,
      "log_odds_chosen": 0.5299686789512634,
      "log_odds_ratio": -0.46948546171188354,
      "logits/chosen": -0.05664820969104767,
      "logits/rejected": -0.9505389928817749,
      "logps/chosen": -1.5510681867599487,
      "logps/rejected": -1.9924825429916382,
      "loss": 1.6432,
      "nll_loss": 1.5962103605270386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1551068127155304,
      "rewards/margins": 0.04414144158363342,
      "rewards/rejected": -0.19924825429916382,
      "step": 394
    },
    {
      "epoch": 0.6225374310480694,
      "grad_norm": 0.18782253563404083,
      "learning_rate": 3.4851856624520394e-06,
      "log_odds_chosen": 0.5067974925041199,
      "log_odds_ratio": -0.4746589660644531,
      "logits/chosen": -0.008503247052431107,
      "logits/rejected": -1.0330562591552734,
      "logps/chosen": -1.5813007354736328,
      "logps/rejected": -2.0009989738464355,
      "loss": 1.668,
      "nll_loss": 1.6205734014511108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15813006460666656,
      "rewards/margins": 0.04196983203291893,
      "rewards/rejected": -0.200099915266037,
      "step": 395
    },
    {
      "epoch": 0.624113475177305,
      "grad_norm": 0.19832897186279297,
      "learning_rate": 3.4814984120791664e-06,
      "log_odds_chosen": 0.49104994535446167,
      "log_odds_ratio": -0.48074740171432495,
      "logits/chosen": -0.014758365228772163,
      "logits/rejected": -0.8711364269256592,
      "logps/chosen": -1.5502197742462158,
      "logps/rejected": -1.954229712486267,
      "loss": 1.6486,
      "nll_loss": 1.6004936695098877,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15502198040485382,
      "rewards/margins": 0.040400996804237366,
      "rewards/rejected": -0.1954229772090912,
      "step": 396
    },
    {
      "epoch": 0.6256895193065406,
      "grad_norm": 0.17359870672225952,
      "learning_rate": 3.477799969693407e-06,
      "log_odds_chosen": 0.446225643157959,
      "log_odds_ratio": -0.5032547116279602,
      "logits/chosen": 0.005189519375562668,
      "logits/rejected": -1.031449317932129,
      "logps/chosen": -1.5504294633865356,
      "logps/rejected": -1.9127092361450195,
      "loss": 1.6418,
      "nll_loss": 1.5914617776870728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15504296123981476,
      "rewards/margins": 0.036227963864803314,
      "rewards/rejected": -0.19127091765403748,
      "step": 397
    },
    {
      "epoch": 0.6272655634357762,
      "grad_norm": 0.19309338927268982,
      "learning_rate": 3.474090363234728e-06,
      "log_odds_chosen": 0.7335612177848816,
      "log_odds_ratio": -0.4062363803386688,
      "logits/chosen": -0.12859651446342468,
      "logits/rejected": -1.0809268951416016,
      "logps/chosen": -1.487412929534912,
      "logps/rejected": -2.097137928009033,
      "loss": 1.5731,
      "nll_loss": 1.5325067043304443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14874127507209778,
      "rewards/margins": 0.060972537845373154,
      "rewards/rejected": -0.20971381664276123,
      "step": 398
    },
    {
      "epoch": 0.6288416075650118,
      "grad_norm": 0.18933425843715668,
      "learning_rate": 3.4703696207274325e-06,
      "log_odds_chosen": 0.5037076473236084,
      "log_odds_ratio": -0.4797070622444153,
      "logits/chosen": -0.0356442891061306,
      "logits/rejected": -0.9325816035270691,
      "logps/chosen": -1.566871166229248,
      "logps/rejected": -1.9823353290557861,
      "loss": 1.6422,
      "nll_loss": 1.5941991806030273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15668712556362152,
      "rewards/margins": 0.04154641181230545,
      "rewards/rejected": -0.19823354482650757,
      "step": 399
    },
    {
      "epoch": 0.6304176516942475,
      "grad_norm": 0.1794310212135315,
      "learning_rate": 3.4666377702799545e-06,
      "log_odds_chosen": 0.507260799407959,
      "log_odds_ratio": -0.47620829939842224,
      "logits/chosen": -0.1437523066997528,
      "logits/rejected": -1.181861400604248,
      "logps/chosen": -1.4786607027053833,
      "logps/rejected": -1.8904798030853271,
      "loss": 1.5826,
      "nll_loss": 1.5349457263946533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14786607027053833,
      "rewards/margins": 0.04118192940950394,
      "rewards/rejected": -0.18904799222946167,
      "step": 400
    },
    {
      "epoch": 0.6319936958234831,
      "grad_norm": 0.19182448089122772,
      "learning_rate": 3.4628948400846417e-06,
      "log_odds_chosen": 0.6221880316734314,
      "log_odds_ratio": -0.43973517417907715,
      "logits/chosen": -0.07098089158535004,
      "logits/rejected": -1.0959513187408447,
      "logps/chosen": -1.5599616765975952,
      "logps/rejected": -2.0811495780944824,
      "loss": 1.6506,
      "nll_loss": 1.6066694259643555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1559961885213852,
      "rewards/margins": 0.052118781954050064,
      "rewards/rejected": -0.20811496675014496,
      "step": 401
    },
    {
      "epoch": 0.6335697399527187,
      "grad_norm": 0.19639141857624054,
      "learning_rate": 3.4591408584175426e-06,
      "log_odds_chosen": 0.5058379173278809,
      "log_odds_ratio": -0.48095616698265076,
      "logits/chosen": -0.0667698010802269,
      "logits/rejected": -1.0430485010147095,
      "logps/chosen": -1.5944137573242188,
      "logps/rejected": -2.0188231468200684,
      "loss": 1.6883,
      "nll_loss": 1.6401551961898804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15944138169288635,
      "rewards/margins": 0.04244093596935272,
      "rewards/rejected": -0.20188231766223907,
      "step": 402
    },
    {
      "epoch": 0.6351457840819543,
      "grad_norm": 0.21676738560199738,
      "learning_rate": 3.4553758536381974e-06,
      "log_odds_chosen": 0.5158869028091431,
      "log_odds_ratio": -0.4791540205478668,
      "logits/chosen": -0.07078143954277039,
      "logits/rejected": -1.0287364721298218,
      "logps/chosen": -1.5488569736480713,
      "logps/rejected": -1.9769119024276733,
      "loss": 1.6391,
      "nll_loss": 1.591202974319458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15488570928573608,
      "rewards/margins": 0.04280547425150871,
      "rewards/rejected": -0.1976911723613739,
      "step": 403
    },
    {
      "epoch": 0.6367218282111899,
      "grad_norm": 0.19120246171951294,
      "learning_rate": 3.451599854189418e-06,
      "log_odds_chosen": 0.5294864177703857,
      "log_odds_ratio": -0.4672943651676178,
      "logits/chosen": -0.02905140444636345,
      "logits/rejected": -0.7433152198791504,
      "logps/chosen": -1.5466922521591187,
      "logps/rejected": -1.9846346378326416,
      "loss": 1.6494,
      "nll_loss": 1.6026866436004639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15466921031475067,
      "rewards/margins": 0.04379424825310707,
      "rewards/rejected": -0.19846348464488983,
      "step": 404
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 0.19179391860961914,
      "learning_rate": 3.4478128885970765e-06,
      "log_odds_chosen": 0.587192177772522,
      "log_odds_ratio": -0.4434540867805481,
      "logits/chosen": -0.07451428472995758,
      "logits/rejected": -1.0500125885009766,
      "logps/chosen": -1.6142433881759644,
      "logps/rejected": -2.1082777976989746,
      "loss": 1.702,
      "nll_loss": 1.657612919807434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16142433881759644,
      "rewards/margins": 0.049403443932533264,
      "rewards/rejected": -0.2108277678489685,
      "step": 405
    },
    {
      "epoch": 0.6398739164696612,
      "grad_norm": 0.18501067161560059,
      "learning_rate": 3.44401498546989e-06,
      "log_odds_chosen": 0.40521109104156494,
      "log_odds_ratio": -0.5183945298194885,
      "logits/chosen": -0.03797990828752518,
      "logits/rejected": -1.0899728536605835,
      "logps/chosen": -1.5686931610107422,
      "logps/rejected": -1.90311861038208,
      "loss": 1.6415,
      "nll_loss": 1.5896741151809692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1568693071603775,
      "rewards/margins": 0.03344256803393364,
      "rewards/rejected": -0.19031187891960144,
      "step": 406
    },
    {
      "epoch": 0.6414499605988968,
      "grad_norm": 0.1799306869506836,
      "learning_rate": 3.4402061734992005e-06,
      "log_odds_chosen": 0.4967467188835144,
      "log_odds_ratio": -0.48000407218933105,
      "logits/chosen": -0.05448009818792343,
      "logits/rejected": -1.0589975118637085,
      "logps/chosen": -1.5077265501022339,
      "logps/rejected": -1.9145967960357666,
      "loss": 1.6056,
      "nll_loss": 1.5576308965682983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15077266097068787,
      "rewards/margins": 0.04068702459335327,
      "rewards/rejected": -0.19145967066287994,
      "step": 407
    },
    {
      "epoch": 0.6430260047281324,
      "grad_norm": 0.20216302573680878,
      "learning_rate": 3.4363864814587656e-06,
      "log_odds_chosen": 0.47775065898895264,
      "log_odds_ratio": -0.48624420166015625,
      "logits/chosen": -0.17656797170639038,
      "logits/rejected": -0.6637123823165894,
      "logps/chosen": -1.4837757349014282,
      "logps/rejected": -1.8745161294937134,
      "loss": 1.5846,
      "nll_loss": 1.5359528064727783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14837758243083954,
      "rewards/margins": 0.039074040949344635,
      "rewards/rejected": -0.18745161592960358,
      "step": 408
    },
    {
      "epoch": 0.644602048857368,
      "grad_norm": 0.20102708041667938,
      "learning_rate": 3.4325559382045343e-06,
      "log_odds_chosen": 0.36611372232437134,
      "log_odds_ratio": -0.5313414335250854,
      "logits/chosen": -0.1553874909877777,
      "logits/rejected": -1.0996140241622925,
      "logps/chosen": -1.4935764074325562,
      "logps/rejected": -1.7868804931640625,
      "loss": 1.5879,
      "nll_loss": 1.5347291231155396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1493576467037201,
      "rewards/margins": 0.029330415651202202,
      "rewards/rejected": -0.17868804931640625,
      "step": 409
    },
    {
      "epoch": 0.6461780929866037,
      "grad_norm": 0.19717082381248474,
      "learning_rate": 3.4287145726744295e-06,
      "log_odds_chosen": 0.4966202974319458,
      "log_odds_ratio": -0.4785189628601074,
      "logits/chosen": -0.17511916160583496,
      "logits/rejected": -1.0572038888931274,
      "logps/chosen": -1.4797816276550293,
      "logps/rejected": -1.8831043243408203,
      "loss": 1.5853,
      "nll_loss": 1.5374045372009277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14797815680503845,
      "rewards/margins": 0.04033227264881134,
      "rewards/rejected": -0.1883104145526886,
      "step": 410
    },
    {
      "epoch": 0.6477541371158393,
      "grad_norm": 0.19743499159812927,
      "learning_rate": 3.4248624138881335e-06,
      "log_odds_chosen": 0.41479527950286865,
      "log_odds_ratio": -0.5294526815414429,
      "logits/chosen": -0.07476592063903809,
      "logits/rejected": -0.8440074920654297,
      "logps/chosen": -1.6108230352401733,
      "logps/rejected": -1.9684065580368042,
      "loss": 1.6982,
      "nll_loss": 1.6452078819274902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16108231246471405,
      "rewards/margins": 0.035758357495069504,
      "rewards/rejected": -0.19684067368507385,
      "step": 411
    },
    {
      "epoch": 0.6493301812450749,
      "grad_norm": 0.22961333394050598,
      "learning_rate": 3.4209994909468672e-06,
      "log_odds_chosen": 0.6204558610916138,
      "log_odds_ratio": -0.4397663474082947,
      "logits/chosen": -0.19274577498435974,
      "logits/rejected": -0.7418732643127441,
      "logps/chosen": -1.5166682004928589,
      "logps/rejected": -2.030907392501831,
      "loss": 1.6063,
      "nll_loss": 1.5622737407684326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15166683495044708,
      "rewards/margins": 0.051423899829387665,
      "rewards/rejected": -0.20309074223041534,
      "step": 412
    },
    {
      "epoch": 0.6509062253743105,
      "grad_norm": 0.18674862384796143,
      "learning_rate": 3.4171258330331667e-06,
      "log_odds_chosen": 0.4268144369125366,
      "log_odds_ratio": -0.5047621130943298,
      "logits/chosen": -0.06827862560749054,
      "logits/rejected": -0.8055958151817322,
      "logps/chosen": -1.5982739925384521,
      "logps/rejected": -1.9501354694366455,
      "loss": 1.6845,
      "nll_loss": 1.6340434551239014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15982739627361298,
      "rewards/margins": 0.03518615663051605,
      "rewards/rejected": -0.19501356780529022,
      "step": 413
    },
    {
      "epoch": 0.6524822695035462,
      "grad_norm": 0.1905805468559265,
      "learning_rate": 3.4132414694106684e-06,
      "log_odds_chosen": 0.6537319421768188,
      "log_odds_ratio": -0.42581480741500854,
      "logits/chosen": -0.09001298248767853,
      "logits/rejected": -1.1602773666381836,
      "logps/chosen": -1.5066035985946655,
      "logps/rejected": -2.0486607551574707,
      "loss": 1.5912,
      "nll_loss": 1.5486054420471191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15066036581993103,
      "rewards/margins": 0.05420570820569992,
      "rewards/rejected": -0.20486608147621155,
      "step": 414
    },
    {
      "epoch": 0.6540583136327817,
      "grad_norm": 0.19903519749641418,
      "learning_rate": 3.409346429423884e-06,
      "log_odds_chosen": 0.43221455812454224,
      "log_odds_ratio": -0.5025917887687683,
      "logits/chosen": 0.04223542660474777,
      "logits/rejected": -1.0005658864974976,
      "logps/chosen": -1.5795127153396606,
      "logps/rejected": -1.9357554912567139,
      "loss": 1.6597,
      "nll_loss": 1.6094313859939575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15795128047466278,
      "rewards/margins": 0.03562428802251816,
      "rewards/rejected": -0.19357556104660034,
      "step": 415
    },
    {
      "epoch": 0.6556343577620173,
      "grad_norm": 0.1885460466146469,
      "learning_rate": 3.40544074249798e-06,
      "log_odds_chosen": 0.5970582365989685,
      "log_odds_ratio": -0.44192248582839966,
      "logits/chosen": -0.13446246087551117,
      "logits/rejected": -1.157158374786377,
      "logps/chosen": -1.5702476501464844,
      "logps/rejected": -2.0696332454681396,
      "loss": 1.6555,
      "nll_loss": 1.6113184690475464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15702477097511292,
      "rewards/margins": 0.049938566982746124,
      "rewards/rejected": -0.20696333050727844,
      "step": 416
    },
    {
      "epoch": 0.6572104018912529,
      "grad_norm": 0.20453964173793793,
      "learning_rate": 3.401524438138556e-06,
      "log_odds_chosen": 0.447142630815506,
      "log_odds_ratio": -0.4999522864818573,
      "logits/chosen": -0.06529423594474792,
      "logits/rejected": -0.8071571588516235,
      "logps/chosen": -1.5707049369812012,
      "logps/rejected": -1.9413305521011353,
      "loss": 1.6382,
      "nll_loss": 1.5881924629211426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15707048773765564,
      "rewards/margins": 0.037062570452690125,
      "rewards/rejected": -0.19413305819034576,
      "step": 417
    },
    {
      "epoch": 0.6587864460204885,
      "grad_norm": 0.1998985856771469,
      "learning_rate": 3.39759754593142e-06,
      "log_odds_chosen": 0.49144208431243896,
      "log_odds_ratio": -0.4852937161922455,
      "logits/chosen": -0.12646208703517914,
      "logits/rejected": -0.9786322116851807,
      "logps/chosen": -1.6269687414169312,
      "logps/rejected": -2.0418336391448975,
      "loss": 1.7024,
      "nll_loss": 1.6539154052734375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16269688308238983,
      "rewards/margins": 0.041486479341983795,
      "rewards/rejected": -0.20418335497379303,
      "step": 418
    },
    {
      "epoch": 0.6603624901497241,
      "grad_norm": 0.19601181149482727,
      "learning_rate": 3.3936600955423683e-06,
      "log_odds_chosen": 0.6909330487251282,
      "log_odds_ratio": -0.42103973031044006,
      "logits/chosen": -0.05681590735912323,
      "logits/rejected": -0.7964982986450195,
      "logps/chosen": -1.524495005607605,
      "logps/rejected": -2.098771333694458,
      "loss": 1.614,
      "nll_loss": 1.5719231367111206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15244948863983154,
      "rewards/margins": 0.05742764472961426,
      "rewards/rejected": -0.209877148270607,
      "step": 419
    },
    {
      "epoch": 0.6619385342789598,
      "grad_norm": 0.1980346292257309,
      "learning_rate": 3.3897121167169573e-06,
      "log_odds_chosen": 0.3761827051639557,
      "log_odds_ratio": -0.5326985120773315,
      "logits/chosen": -0.1706826090812683,
      "logits/rejected": -1.0160996913909912,
      "logps/chosen": -1.4991317987442017,
      "logps/rejected": -1.805385947227478,
      "loss": 1.6005,
      "nll_loss": 1.5472354888916016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14991319179534912,
      "rewards/margins": 0.030625399202108383,
      "rewards/rejected": -0.1805385798215866,
      "step": 420
    },
    {
      "epoch": 0.6635145784081954,
      "grad_norm": 0.19532448053359985,
      "learning_rate": 3.38575363928028e-06,
      "log_odds_chosen": 0.5968228578567505,
      "log_odds_ratio": -0.4450194239616394,
      "logits/chosen": -0.09741874039173126,
      "logits/rejected": -0.8825951814651489,
      "logps/chosen": -1.5054579973220825,
      "logps/rejected": -1.9978173971176147,
      "loss": 1.5798,
      "nll_loss": 1.5352727174758911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15054580569267273,
      "rewards/margins": 0.049235932528972626,
      "rewards/rejected": -0.19978173077106476,
      "step": 421
    },
    {
      "epoch": 0.665090622537431,
      "grad_norm": 0.19437773525714874,
      "learning_rate": 3.3817846931367452e-06,
      "log_odds_chosen": 0.3979894518852234,
      "log_odds_ratio": -0.5184385180473328,
      "logits/chosen": -0.19602851569652557,
      "logits/rejected": -0.9046583771705627,
      "logps/chosen": -1.492729902267456,
      "logps/rejected": -1.8125101327896118,
      "loss": 1.5839,
      "nll_loss": 1.5320786237716675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14927297830581665,
      "rewards/margins": 0.031978022307157516,
      "rewards/rejected": -0.18125101923942566,
      "step": 422
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 0.18563054502010345,
      "learning_rate": 3.377805308269844e-06,
      "log_odds_chosen": 0.6691169738769531,
      "log_odds_ratio": -0.4283203184604645,
      "logits/chosen": -0.06355718523263931,
      "logits/rejected": -1.2140281200408936,
      "logps/chosen": -1.549073338508606,
      "logps/rejected": -2.1106374263763428,
      "loss": 1.6271,
      "nll_loss": 1.5843141078948975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15490733087062836,
      "rewards/margins": 0.05615641176700592,
      "rewards/rejected": -0.21106374263763428,
      "step": 423
    },
    {
      "epoch": 0.6682427107959023,
      "grad_norm": 0.1859610229730606,
      "learning_rate": 3.3738155147419275e-06,
      "log_odds_chosen": 0.6023603677749634,
      "log_odds_ratio": -0.44674554467201233,
      "logits/chosen": -0.18490374088287354,
      "logits/rejected": -0.9434022307395935,
      "logps/chosen": -1.5427931547164917,
      "logps/rejected": -2.0453150272369385,
      "loss": 1.6059,
      "nll_loss": 1.5612272024154663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15427932143211365,
      "rewards/margins": 0.05025219917297363,
      "rewards/rejected": -0.20453152060508728,
      "step": 424
    },
    {
      "epoch": 0.6698187549251379,
      "grad_norm": 0.18988454341888428,
      "learning_rate": 3.3698153426939824e-06,
      "log_odds_chosen": 0.6990432739257812,
      "log_odds_ratio": -0.41350850462913513,
      "logits/chosen": -0.17698457837104797,
      "logits/rejected": -0.7861905694007874,
      "logps/chosen": -1.4711315631866455,
      "logps/rejected": -2.045083999633789,
      "loss": 1.5487,
      "nll_loss": 1.5073806047439575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1471131592988968,
      "rewards/margins": 0.05739524960517883,
      "rewards/rejected": -0.20450839400291443,
      "step": 425
    },
    {
      "epoch": 0.6713947990543735,
      "grad_norm": 0.19885441660881042,
      "learning_rate": 3.3658048223453954e-06,
      "log_odds_chosen": 0.6381431221961975,
      "log_odds_ratio": -0.4310988783836365,
      "logits/chosen": -0.14545650780200958,
      "logits/rejected": -1.0610605478286743,
      "logps/chosen": -1.6127598285675049,
      "logps/rejected": -2.1517128944396973,
      "loss": 1.661,
      "nll_loss": 1.6179263591766357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1612759828567505,
      "rewards/margins": 0.05389528349041939,
      "rewards/rejected": -0.21517126262187958,
      "step": 426
    },
    {
      "epoch": 0.6729708431836091,
      "grad_norm": 0.19337302446365356,
      "learning_rate": 3.3617839839937337e-06,
      "log_odds_chosen": 0.6319109201431274,
      "log_odds_ratio": -0.4360054135322571,
      "logits/chosen": -0.08238127827644348,
      "logits/rejected": -1.1307555437088013,
      "logps/chosen": -1.5771963596343994,
      "logps/rejected": -2.1106276512145996,
      "loss": 1.6469,
      "nll_loss": 1.6032705307006836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15771962702274323,
      "rewards/margins": 0.05334313213825226,
      "rewards/rejected": -0.21106277406215668,
      "step": 427
    },
    {
      "epoch": 0.6745468873128447,
      "grad_norm": 0.19824667274951935,
      "learning_rate": 3.3577528580145107e-06,
      "log_odds_chosen": 0.3213936984539032,
      "log_odds_ratio": -0.548486053943634,
      "logits/chosen": -0.09538719803094864,
      "logits/rejected": -1.006461501121521,
      "logps/chosen": -1.5969069004058838,
      "logps/rejected": -1.8635402917861938,
      "loss": 1.6813,
      "nll_loss": 1.6264036893844604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15969069302082062,
      "rewards/margins": 0.026663340628147125,
      "rewards/rejected": -0.18635404109954834,
      "step": 428
    },
    {
      "epoch": 0.6761229314420804,
      "grad_norm": 0.1969161182641983,
      "learning_rate": 3.353711474860956e-06,
      "log_odds_chosen": 0.5900525450706482,
      "log_odds_ratio": -0.45637646317481995,
      "logits/chosen": -0.154756560921669,
      "logits/rejected": -0.9149386882781982,
      "logps/chosen": -1.5449559688568115,
      "logps/rejected": -2.042635202407837,
      "loss": 1.6083,
      "nll_loss": 1.5626716613769531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15449561178684235,
      "rewards/margins": 0.049767911434173584,
      "rewards/rejected": -0.20426350831985474,
      "step": 429
    },
    {
      "epoch": 0.677698975571316,
      "grad_norm": 0.1858314871788025,
      "learning_rate": 3.3496598650637916e-06,
      "log_odds_chosen": 0.555091917514801,
      "log_odds_ratio": -0.4606228470802307,
      "logits/chosen": -0.17273341119289398,
      "logits/rejected": -0.8560744524002075,
      "logps/chosen": -1.4441226720809937,
      "logps/rejected": -1.892524242401123,
      "loss": 1.5294,
      "nll_loss": 1.4833111763000488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14441227912902832,
      "rewards/margins": 0.04484015703201294,
      "rewards/rejected": -0.18925242125988007,
      "step": 430
    },
    {
      "epoch": 0.6792750197005516,
      "grad_norm": 0.18842041492462158,
      "learning_rate": 3.3455980592309923e-06,
      "log_odds_chosen": 0.6005351543426514,
      "log_odds_ratio": -0.44451576471328735,
      "logits/chosen": -0.2088775932788849,
      "logits/rejected": -0.9120292663574219,
      "logps/chosen": -1.4238063097000122,
      "logps/rejected": -1.9125840663909912,
      "loss": 1.5182,
      "nll_loss": 1.4737814664840698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14238062500953674,
      "rewards/margins": 0.0488777831196785,
      "rewards/rejected": -0.19125841557979584,
      "step": 431
    },
    {
      "epoch": 0.6808510638297872,
      "grad_norm": 0.19271476566791534,
      "learning_rate": 3.341526088047562e-06,
      "log_odds_chosen": 0.5448278188705444,
      "log_odds_ratio": -0.46626630425453186,
      "logits/chosen": -0.15303638577461243,
      "logits/rejected": -1.0090641975402832,
      "logps/chosen": -1.6087162494659424,
      "logps/rejected": -2.0677130222320557,
      "loss": 1.6696,
      "nll_loss": 1.622945785522461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16087162494659424,
      "rewards/margins": 0.04589966684579849,
      "rewards/rejected": -0.20677129924297333,
      "step": 432
    },
    {
      "epoch": 0.6824271079590228,
      "grad_norm": 0.21036098897457123,
      "learning_rate": 3.3374439822752972e-06,
      "log_odds_chosen": 0.419351726770401,
      "log_odds_ratio": -0.5088696479797363,
      "logits/chosen": -0.10887278616428375,
      "logits/rejected": -0.8606767654418945,
      "logps/chosen": -1.572546362876892,
      "logps/rejected": -1.9210548400878906,
      "loss": 1.6632,
      "nll_loss": 1.612297534942627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1572546511888504,
      "rewards/margins": 0.03485086187720299,
      "rewards/rejected": -0.1921055018901825,
      "step": 433
    },
    {
      "epoch": 0.6840031520882585,
      "grad_norm": 0.20627471804618835,
      "learning_rate": 3.333351772752559e-06,
      "log_odds_chosen": 0.566561222076416,
      "log_odds_ratio": -0.46442872285842896,
      "logits/chosen": -0.21569103002548218,
      "logits/rejected": -0.9650353789329529,
      "logps/chosen": -1.639683723449707,
      "logps/rejected": -2.114182949066162,
      "loss": 1.7037,
      "nll_loss": 1.6572954654693604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16396838426589966,
      "rewards/margins": 0.04744991660118103,
      "rewards/rejected": -0.2114183008670807,
      "step": 434
    },
    {
      "epoch": 0.6855791962174941,
      "grad_norm": 0.20955084264278412,
      "learning_rate": 3.3292494903940338e-06,
      "log_odds_chosen": 0.6968019604682922,
      "log_odds_ratio": -0.41255873441696167,
      "logits/chosen": -0.1339864879846573,
      "logits/rejected": -0.9979945421218872,
      "logps/chosen": -1.54659104347229,
      "logps/rejected": -2.1267032623291016,
      "loss": 1.6187,
      "nll_loss": 1.5774283409118652,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15465912222862244,
      "rewards/margins": 0.05801120400428772,
      "rewards/rejected": -0.21267032623291016,
      "step": 435
    },
    {
      "epoch": 0.6871552403467297,
      "grad_norm": 0.18498389422893524,
      "learning_rate": 3.3251371661905063e-06,
      "log_odds_chosen": 0.5884695053100586,
      "log_odds_ratio": -0.45282870531082153,
      "logits/chosen": -0.174495130777359,
      "logits/rejected": -0.8306106925010681,
      "logps/chosen": -1.3706011772155762,
      "logps/rejected": -1.8502486944198608,
      "loss": 1.4589,
      "nll_loss": 1.413588523864746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13706013560295105,
      "rewards/margins": 0.047964759171009064,
      "rewards/rejected": -0.18502488732337952,
      "step": 436
    },
    {
      "epoch": 0.6887312844759653,
      "grad_norm": 0.1930173933506012,
      "learning_rate": 3.321014831208622e-06,
      "log_odds_chosen": 0.5815909504890442,
      "log_odds_ratio": -0.44832438230514526,
      "logits/chosen": -0.16663047671318054,
      "logits/rejected": -0.8158444762229919,
      "logps/chosen": -1.4361575841903687,
      "logps/rejected": -1.9045019149780273,
      "loss": 1.532,
      "nll_loss": 1.487160325050354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14361576735973358,
      "rewards/margins": 0.046834442764520645,
      "rewards/rejected": -0.19045020639896393,
      "step": 437
    },
    {
      "epoch": 0.6903073286052009,
      "grad_norm": 0.20272772014141083,
      "learning_rate": 3.316882516590652e-06,
      "log_odds_chosen": 0.6009421944618225,
      "log_odds_ratio": -0.43921971321105957,
      "logits/chosen": -0.16785867512226105,
      "logits/rejected": -0.9978471398353577,
      "logps/chosen": -1.4756765365600586,
      "logps/rejected": -1.96806001663208,
      "loss": 1.5594,
      "nll_loss": 1.5154640674591064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14756765961647034,
      "rewards/margins": 0.04923836141824722,
      "rewards/rejected": -0.19680601358413696,
      "step": 438
    },
    {
      "epoch": 0.6918833727344366,
      "grad_norm": 0.21401967108249664,
      "learning_rate": 3.31274025355426e-06,
      "log_odds_chosen": 0.5665644407272339,
      "log_odds_ratio": -0.45475664734840393,
      "logits/chosen": -0.20744185149669647,
      "logits/rejected": -0.8934562802314758,
      "logps/chosen": -1.5667637586593628,
      "logps/rejected": -2.0387802124023438,
      "loss": 1.6208,
      "nll_loss": 1.575301170349121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15667636692523956,
      "rewards/margins": 0.04720166325569153,
      "rewards/rejected": -0.2038780301809311,
      "step": 439
    },
    {
      "epoch": 0.6934594168636722,
      "grad_norm": 0.20656292140483856,
      "learning_rate": 3.308588073392265e-06,
      "log_odds_chosen": 0.5346047878265381,
      "log_odds_ratio": -0.4697275161743164,
      "logits/chosen": -0.19090476632118225,
      "logits/rejected": -0.910874605178833,
      "logps/chosen": -1.603625774383545,
      "logps/rejected": -2.054203748703003,
      "loss": 1.6598,
      "nll_loss": 1.612839937210083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16036257147789001,
      "rewards/margins": 0.04505779966711998,
      "rewards/rejected": -0.2054203748703003,
      "step": 440
    },
    {
      "epoch": 0.6950354609929078,
      "grad_norm": 0.18964466452598572,
      "learning_rate": 3.3044260074724035e-06,
      "log_odds_chosen": 0.6167492866516113,
      "log_odds_ratio": -0.4436300992965698,
      "logits/chosen": -0.19338998198509216,
      "logits/rejected": -1.0959054231643677,
      "logps/chosen": -1.5115933418273926,
      "logps/rejected": -2.0191221237182617,
      "loss": 1.5891,
      "nll_loss": 1.544729471206665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1511593461036682,
      "rewards/margins": 0.050752852112054825,
      "rewards/rejected": -0.20191219449043274,
      "step": 441
    },
    {
      "epoch": 0.6966115051221434,
      "grad_norm": 0.19798265397548676,
      "learning_rate": 3.300254087237097e-06,
      "log_odds_chosen": 0.536584198474884,
      "log_odds_ratio": -0.4647059440612793,
      "logits/chosen": -0.16768081486225128,
      "logits/rejected": -0.9846087098121643,
      "logps/chosen": -1.4075177907943726,
      "logps/rejected": -1.8397033214569092,
      "loss": 1.5153,
      "nll_loss": 1.4688141345977783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14075177907943726,
      "rewards/margins": 0.043218567967414856,
      "rewards/rejected": -0.1839703470468521,
      "step": 442
    },
    {
      "epoch": 0.698187549251379,
      "grad_norm": 0.19152890145778656,
      "learning_rate": 3.2960723442032105e-06,
      "log_odds_chosen": 0.6987941265106201,
      "log_odds_ratio": -0.4106602370738983,
      "logits/chosen": -0.2129349410533905,
      "logits/rejected": -1.2987092733383179,
      "logps/chosen": -1.5995932817459106,
      "logps/rejected": -2.192657947540283,
      "loss": 1.6557,
      "nll_loss": 1.614605188369751,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1599593311548233,
      "rewards/margins": 0.059306442737579346,
      "rewards/rejected": -0.21926578879356384,
      "step": 443
    },
    {
      "epoch": 0.6997635933806147,
      "grad_norm": 0.1959015130996704,
      "learning_rate": 3.291880809961814e-06,
      "log_odds_chosen": 0.6200303435325623,
      "log_odds_ratio": -0.44293415546417236,
      "logits/chosen": -0.13872218132019043,
      "logits/rejected": -0.9167954921722412,
      "logps/chosen": -1.5655865669250488,
      "logps/rejected": -2.0818004608154297,
      "loss": 1.6387,
      "nll_loss": 1.594357967376709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15655863285064697,
      "rewards/margins": 0.05162140727043152,
      "rewards/rejected": -0.20818006992340088,
      "step": 444
    },
    {
      "epoch": 0.7013396375098503,
      "grad_norm": 0.20035208761692047,
      "learning_rate": 3.2876795161779473e-06,
      "log_odds_chosen": 0.7111386060714722,
      "log_odds_ratio": -0.40192127227783203,
      "logits/chosen": -0.16324040293693542,
      "logits/rejected": -1.2092691659927368,
      "logps/chosen": -1.576241374015808,
      "logps/rejected": -2.177393674850464,
      "loss": 1.627,
      "nll_loss": 1.586832880973816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15762415528297424,
      "rewards/margins": 0.060115229338407516,
      "rewards/rejected": -0.21773935854434967,
      "step": 445
    },
    {
      "epoch": 0.7029156816390859,
      "grad_norm": 0.19885501265525818,
      "learning_rate": 3.2834684945903776e-06,
      "log_odds_chosen": 0.5507287979125977,
      "log_odds_ratio": -0.4596758782863617,
      "logits/chosen": -0.2190311998128891,
      "logits/rejected": -0.9251666069030762,
      "logps/chosen": -1.5019633769989014,
      "logps/rejected": -1.9505462646484375,
      "loss": 1.5836,
      "nll_loss": 1.5376709699630737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15019632875919342,
      "rewards/margins": 0.044858310371637344,
      "rewards/rejected": -0.19505466520786285,
      "step": 446
    },
    {
      "epoch": 0.7044917257683215,
      "grad_norm": 0.2020716369152069,
      "learning_rate": 3.2792477770113624e-06,
      "log_odds_chosen": 0.4936216175556183,
      "log_odds_ratio": -0.4818211495876312,
      "logits/chosen": -0.3026110827922821,
      "logits/rejected": -1.0682443380355835,
      "logps/chosen": -1.5848886966705322,
      "logps/rejected": -1.9951268434524536,
      "loss": 1.6426,
      "nll_loss": 1.5943942070007324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15848886966705322,
      "rewards/margins": 0.041023820638656616,
      "rewards/rejected": -0.19951270520687103,
      "step": 447
    },
    {
      "epoch": 0.7060677698975572,
      "grad_norm": 0.18542690575122833,
      "learning_rate": 3.275017395326407e-06,
      "log_odds_chosen": 0.48193085193634033,
      "log_odds_ratio": -0.4900546371936798,
      "logits/chosen": -0.1320614218711853,
      "logits/rejected": -0.8647894263267517,
      "logps/chosen": -1.4781405925750732,
      "logps/rejected": -1.8653026819229126,
      "loss": 1.5636,
      "nll_loss": 1.5146205425262451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14781403541564941,
      "rewards/margins": 0.03871622309088707,
      "rewards/rejected": -0.18653027713298798,
      "step": 448
    },
    {
      "epoch": 0.7076438140267928,
      "grad_norm": 0.1981741189956665,
      "learning_rate": 3.2707773814940244e-06,
      "log_odds_chosen": 0.6399044990539551,
      "log_odds_ratio": -0.43002036213874817,
      "logits/chosen": -0.1578344702720642,
      "logits/rejected": -1.0271568298339844,
      "logps/chosen": -1.509232997894287,
      "logps/rejected": -2.0302257537841797,
      "loss": 1.5915,
      "nll_loss": 1.5485467910766602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15092331171035767,
      "rewards/margins": 0.05209926515817642,
      "rewards/rejected": -0.2030225694179535,
      "step": 449
    },
    {
      "epoch": 0.7092198581560284,
      "grad_norm": 0.2161954790353775,
      "learning_rate": 3.2665277675454935e-06,
      "log_odds_chosen": 0.6585409641265869,
      "log_odds_ratio": -0.43546614050865173,
      "logits/chosen": -0.2146613448858261,
      "logits/rejected": -1.2824149131774902,
      "logps/chosen": -1.591871976852417,
      "logps/rejected": -2.1544575691223145,
      "loss": 1.6446,
      "nll_loss": 1.60104501247406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1591871976852417,
      "rewards/margins": 0.05625856667757034,
      "rewards/rejected": -0.21544577181339264,
      "step": 450
    },
    {
      "epoch": 0.710795902285264,
      "grad_norm": 0.19009153544902802,
      "learning_rate": 3.262268585584619e-06,
      "log_odds_chosen": 0.5836161971092224,
      "log_odds_ratio": -0.44612884521484375,
      "logits/chosen": -0.1468001902103424,
      "logits/rejected": -0.9669194221496582,
      "logps/chosen": -1.5377750396728516,
      "logps/rejected": -2.0233418941497803,
      "loss": 1.607,
      "nll_loss": 1.562421441078186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15377750992774963,
      "rewards/margins": 0.048556677997112274,
      "rewards/rejected": -0.2023341804742813,
      "step": 451
    },
    {
      "epoch": 0.7123719464144996,
      "grad_norm": 0.2047768086194992,
      "learning_rate": 3.2579998677874853e-06,
      "log_odds_chosen": 0.6704806685447693,
      "log_odds_ratio": -0.42241793870925903,
      "logits/chosen": -0.22139252722263336,
      "logits/rejected": -0.9740051627159119,
      "logps/chosen": -1.572437047958374,
      "logps/rejected": -2.138031244277954,
      "loss": 1.6272,
      "nll_loss": 1.5849249362945557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15724369883537292,
      "rewards/margins": 0.05655941367149353,
      "rewards/rejected": -0.21380311250686646,
      "step": 452
    },
    {
      "epoch": 0.7139479905437353,
      "grad_norm": 0.1976122111082077,
      "learning_rate": 3.2537216464022155e-06,
      "log_odds_chosen": 0.6525107026100159,
      "log_odds_ratio": -0.42794734239578247,
      "logits/chosen": -0.21664118766784668,
      "logits/rejected": -0.944098174571991,
      "logps/chosen": -1.5721834897994995,
      "logps/rejected": -2.1166865825653076,
      "loss": 1.638,
      "nll_loss": 1.5951955318450928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15721836686134338,
      "rewards/margins": 0.054450295865535736,
      "rewards/rejected": -0.21166865527629852,
      "step": 453
    },
    {
      "epoch": 0.7155240346729709,
      "grad_norm": 0.19175730645656586,
      "learning_rate": 3.2494339537487314e-06,
      "log_odds_chosen": 0.5195281505584717,
      "log_odds_ratio": -0.46826571226119995,
      "logits/chosen": -0.21530620753765106,
      "logits/rejected": -1.1608517169952393,
      "logps/chosen": -1.5761486291885376,
      "logps/rejected": -2.0063657760620117,
      "loss": 1.6517,
      "nll_loss": 1.6048378944396973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15761485695838928,
      "rewards/margins": 0.043021731078624725,
      "rewards/rejected": -0.2006365805864334,
      "step": 454
    },
    {
      "epoch": 0.7171000788022065,
      "grad_norm": 0.1911652535200119,
      "learning_rate": 3.2451368222185006e-06,
      "log_odds_chosen": 0.46220406889915466,
      "log_odds_ratio": -0.4957023561000824,
      "logits/chosen": -0.22534283995628357,
      "logits/rejected": -0.9358788132667542,
      "logps/chosen": -1.5175461769104004,
      "logps/rejected": -1.8928101062774658,
      "loss": 1.6,
      "nll_loss": 1.550462007522583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15175461769104004,
      "rewards/margins": 0.03752640262246132,
      "rewards/rejected": -0.18928101658821106,
      "step": 455
    },
    {
      "epoch": 0.7186761229314421,
      "grad_norm": 0.17351412773132324,
      "learning_rate": 3.2408302842743007e-06,
      "log_odds_chosen": 0.6546438932418823,
      "log_odds_ratio": -0.42687559127807617,
      "logits/chosen": -0.19083309173583984,
      "logits/rejected": -0.9766150116920471,
      "logps/chosen": -1.4680445194244385,
      "logps/rejected": -2.0059475898742676,
      "loss": 1.5574,
      "nll_loss": 1.5147082805633545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14680443704128265,
      "rewards/margins": 0.05379030108451843,
      "rewards/rejected": -0.2005947381258011,
      "step": 456
    },
    {
      "epoch": 0.7202521670606777,
      "grad_norm": 0.19053436815738678,
      "learning_rate": 3.2365143724499684e-06,
      "log_odds_chosen": 0.5713084936141968,
      "log_odds_ratio": -0.4496612548828125,
      "logits/chosen": -0.19255973398685455,
      "logits/rejected": -1.0097147226333618,
      "logps/chosen": -1.5191394090652466,
      "logps/rejected": -1.9902849197387695,
      "loss": 1.6076,
      "nll_loss": 1.5626832246780396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15191395580768585,
      "rewards/margins": 0.047114528715610504,
      "rewards/rejected": -0.19902849197387695,
      "step": 457
    },
    {
      "epoch": 0.7218282111899134,
      "grad_norm": 0.19614577293395996,
      "learning_rate": 3.2321891193501564e-06,
      "log_odds_chosen": 0.5764437913894653,
      "log_odds_ratio": -0.45762819051742554,
      "logits/chosen": -0.17019122838974,
      "logits/rejected": -0.6311085224151611,
      "logps/chosen": -1.525330901145935,
      "logps/rejected": -2.001309633255005,
      "loss": 1.5922,
      "nll_loss": 1.5463932752609253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15253308415412903,
      "rewards/margins": 0.04759787768125534,
      "rewards/rejected": -0.20013096928596497,
      "step": 458
    },
    {
      "epoch": 0.723404255319149,
      "grad_norm": 0.18243008852005005,
      "learning_rate": 3.2278545576500858e-06,
      "log_odds_chosen": 0.8040944337844849,
      "log_odds_ratio": -0.3783116936683655,
      "logits/chosen": -0.10738835483789444,
      "logits/rejected": -0.7439028024673462,
      "logps/chosen": -1.3702032566070557,
      "logps/rejected": -2.027367115020752,
      "loss": 1.4567,
      "nll_loss": 1.4188926219940186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1370203197002411,
      "rewards/margins": 0.06571639329195023,
      "rewards/rejected": -0.2027367204427719,
      "step": 459
    },
    {
      "epoch": 0.7249802994483846,
      "grad_norm": 0.19899946451187134,
      "learning_rate": 3.223510720095299e-06,
      "log_odds_chosen": 0.7631068825721741,
      "log_odds_ratio": -0.39056843519210815,
      "logits/chosen": -0.26606088876724243,
      "logits/rejected": -1.066392421722412,
      "logps/chosen": -1.5545616149902344,
      "logps/rejected": -2.1977691650390625,
      "loss": 1.6112,
      "nll_loss": 1.5720990896224976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15545617043972015,
      "rewards/margins": 0.06432076543569565,
      "rewards/rejected": -0.2197769284248352,
      "step": 460
    },
    {
      "epoch": 0.7265563435776202,
      "grad_norm": 0.18922847509384155,
      "learning_rate": 3.2191576395014158e-06,
      "log_odds_chosen": 0.70744788646698,
      "log_odds_ratio": -0.4035731256008148,
      "logits/chosen": -0.21413788199424744,
      "logits/rejected": -0.9344199299812317,
      "logps/chosen": -1.4856576919555664,
      "logps/rejected": -2.070754051208496,
      "loss": 1.5558,
      "nll_loss": 1.5154281854629517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14856578409671783,
      "rewards/margins": 0.058509618043899536,
      "rewards/rejected": -0.20707541704177856,
      "step": 461
    },
    {
      "epoch": 0.7281323877068558,
      "grad_norm": 0.1896039843559265,
      "learning_rate": 3.2147953487538794e-06,
      "log_odds_chosen": 0.6818787455558777,
      "log_odds_ratio": -0.4148927927017212,
      "logits/chosen": -0.15922857820987701,
      "logits/rejected": -1.004948377609253,
      "logps/chosen": -1.4554266929626465,
      "logps/rejected": -2.015336275100708,
      "loss": 1.5271,
      "nll_loss": 1.4856001138687134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1455426663160324,
      "rewards/margins": 0.05599096417427063,
      "rewards/rejected": -0.20153363049030304,
      "step": 462
    },
    {
      "epoch": 0.7297084318360915,
      "grad_norm": 0.19634757936000824,
      "learning_rate": 3.2104238808077133e-06,
      "log_odds_chosen": 0.5675202012062073,
      "log_odds_ratio": -0.4579786956310272,
      "logits/chosen": -0.16884222626686096,
      "logits/rejected": -0.785213828086853,
      "logps/chosen": -1.490207314491272,
      "logps/rejected": -1.9569623470306396,
      "loss": 1.5656,
      "nll_loss": 1.5198465585708618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1490207314491272,
      "rewards/margins": 0.04667549580335617,
      "rewards/rejected": -0.19569621980190277,
      "step": 463
    },
    {
      "epoch": 0.731284475965327,
      "grad_norm": 0.18298950791358948,
      "learning_rate": 3.2060432686872704e-06,
      "log_odds_chosen": 0.817018449306488,
      "log_odds_ratio": -0.37420207262039185,
      "logits/chosen": -0.25051018595695496,
      "logits/rejected": -0.8697549104690552,
      "logps/chosen": -1.399641513824463,
      "logps/rejected": -2.06581449508667,
      "loss": 1.4859,
      "nll_loss": 1.4484362602233887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13996414840221405,
      "rewards/margins": 0.06661731749773026,
      "rewards/rejected": -0.2065814733505249,
      "step": 464
    },
    {
      "epoch": 0.7328605200945626,
      "grad_norm": 0.18866463005542755,
      "learning_rate": 3.201653545485982e-06,
      "log_odds_chosen": 0.6448830962181091,
      "log_odds_ratio": -0.42958295345306396,
      "logits/chosen": -0.1599559634923935,
      "logits/rejected": -0.9195252060890198,
      "logps/chosen": -1.5451632738113403,
      "logps/rejected": -2.085179090499878,
      "loss": 1.6017,
      "nll_loss": 1.5587797164916992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1545163244009018,
      "rewards/margins": 0.0540015734732151,
      "rewards/rejected": -0.2085179090499878,
      "step": 465
    },
    {
      "epoch": 0.7344365642237982,
      "grad_norm": 0.20371563732624054,
      "learning_rate": 3.197254744366111e-06,
      "log_odds_chosen": 0.6497169733047485,
      "log_odds_ratio": -0.42687100172042847,
      "logits/chosen": -0.1431826502084732,
      "logits/rejected": -0.902421236038208,
      "logps/chosen": -1.4522693157196045,
      "logps/rejected": -1.9825525283813477,
      "loss": 1.5297,
      "nll_loss": 1.4870625734329224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14522692561149597,
      "rewards/margins": 0.05302830785512924,
      "rewards/rejected": -0.1982552409172058,
      "step": 466
    },
    {
      "epoch": 0.7360126083530338,
      "grad_norm": 0.1974330097436905,
      "learning_rate": 3.192846898558498e-06,
      "log_odds_chosen": 0.5328829288482666,
      "log_odds_ratio": -0.46320897340774536,
      "logits/chosen": -0.1656794548034668,
      "logits/rejected": -0.9588291645050049,
      "logps/chosen": -1.5812673568725586,
      "logps/rejected": -2.025834321975708,
      "loss": 1.6544,
      "nll_loss": 1.608088731765747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15812674164772034,
      "rewards/margins": 0.04445669800043106,
      "rewards/rejected": -0.2025834321975708,
      "step": 467
    },
    {
      "epoch": 0.7375886524822695,
      "grad_norm": 0.19795842468738556,
      "learning_rate": 3.188430041362313e-06,
      "log_odds_chosen": 0.47163689136505127,
      "log_odds_ratio": -0.49232786893844604,
      "logits/chosen": -0.12224957346916199,
      "logits/rejected": -0.909099280834198,
      "logps/chosen": -1.5807934999465942,
      "logps/rejected": -1.9740912914276123,
      "loss": 1.6398,
      "nll_loss": 1.5905786752700806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1580793559551239,
      "rewards/margins": 0.03932976722717285,
      "rewards/rejected": -0.19740912318229675,
      "step": 468
    },
    {
      "epoch": 0.7391646966115051,
      "grad_norm": 0.19603583216667175,
      "learning_rate": 3.184004206144803e-06,
      "log_odds_chosen": 0.7065101861953735,
      "log_odds_ratio": -0.41205504536628723,
      "logits/chosen": -0.2486933022737503,
      "logits/rejected": -0.9299865961074829,
      "logps/chosen": -1.4757022857666016,
      "logps/rejected": -2.054920196533203,
      "loss": 1.5464,
      "nll_loss": 1.5052430629730225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14757022261619568,
      "rewards/margins": 0.05792178213596344,
      "rewards/rejected": -0.20549200475215912,
      "step": 469
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 0.19463147222995758,
      "learning_rate": 3.1795694263410386e-06,
      "log_odds_chosen": 0.7694414258003235,
      "log_odds_ratio": -0.39305710792541504,
      "logits/chosen": -0.21214531362056732,
      "logits/rejected": -0.9676571488380432,
      "logps/chosen": -1.4680100679397583,
      "logps/rejected": -2.1116573810577393,
      "loss": 1.5324,
      "nll_loss": 1.4930670261383057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14680100977420807,
      "rewards/margins": 0.0643647089600563,
      "rewards/rejected": -0.21116572618484497,
      "step": 470
    },
    {
      "epoch": 0.7423167848699763,
      "grad_norm": 0.18510647118091583,
      "learning_rate": 3.1751257354536634e-06,
      "log_odds_chosen": 0.5473852753639221,
      "log_odds_ratio": -0.469069242477417,
      "logits/chosen": -0.16158902645111084,
      "logits/rejected": -1.0543768405914307,
      "logps/chosen": -1.458742618560791,
      "logps/rejected": -1.9098637104034424,
      "loss": 1.5407,
      "nll_loss": 1.4938390254974365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1458742618560791,
      "rewards/margins": 0.04511209577322006,
      "rewards/rejected": -0.19098637998104095,
      "step": 471
    },
    {
      "epoch": 0.7438928289992119,
      "grad_norm": 0.19235409796237946,
      "learning_rate": 3.1706731670526394e-06,
      "log_odds_chosen": 0.608568012714386,
      "log_odds_ratio": -0.43675366044044495,
      "logits/chosen": -0.26602888107299805,
      "logits/rejected": -0.9281780123710632,
      "logps/chosen": -1.5125672817230225,
      "logps/rejected": -2.013179063796997,
      "loss": 1.5683,
      "nll_loss": 1.5246038436889648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15125672519207,
      "rewards/margins": 0.0500611811876297,
      "rewards/rejected": -0.2013178914785385,
      "step": 472
    },
    {
      "epoch": 0.7454688731284476,
      "grad_norm": 0.19334271550178528,
      "learning_rate": 3.166211754774994e-06,
      "log_odds_chosen": 0.6511934995651245,
      "log_odds_ratio": -0.42712464928627014,
      "logits/chosen": -0.20964574813842773,
      "logits/rejected": -1.049457311630249,
      "logps/chosen": -1.5654706954956055,
      "logps/rejected": -2.1108522415161133,
      "loss": 1.6535,
      "nll_loss": 1.6107853651046753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15654708445072174,
      "rewards/margins": 0.05453815311193466,
      "rewards/rejected": -0.2110852152109146,
      "step": 473
    },
    {
      "epoch": 0.7470449172576832,
      "grad_norm": 0.21741212904453278,
      "learning_rate": 3.1617415323245665e-06,
      "log_odds_chosen": 0.6575350761413574,
      "log_odds_ratio": -0.4272666275501251,
      "logits/chosen": -0.24989920854568481,
      "logits/rejected": -1.0688942670822144,
      "logps/chosen": -1.4732602834701538,
      "logps/rejected": -2.006078004837036,
      "loss": 1.5346,
      "nll_loss": 1.491843581199646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1473260223865509,
      "rewards/margins": 0.05328178033232689,
      "rewards/rejected": -0.2006077915430069,
      "step": 474
    },
    {
      "epoch": 0.7486209613869188,
      "grad_norm": 0.18145692348480225,
      "learning_rate": 3.157262533471752e-06,
      "log_odds_chosen": 0.6887553930282593,
      "log_odds_ratio": -0.4109429717063904,
      "logits/chosen": -0.1704792082309723,
      "logits/rejected": -1.0367528200149536,
      "logps/chosen": -1.5262348651885986,
      "logps/rejected": -2.1029748916625977,
      "loss": 1.5964,
      "nll_loss": 1.5552966594696045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526234894990921,
      "rewards/margins": 0.057673998177051544,
      "rewards/rejected": -0.21029751002788544,
      "step": 475
    },
    {
      "epoch": 0.7501970055161544,
      "grad_norm": 0.20390117168426514,
      "learning_rate": 3.1527747920532468e-06,
      "log_odds_chosen": 0.5650341510772705,
      "log_odds_ratio": -0.4515586495399475,
      "logits/chosen": -0.17346344888210297,
      "logits/rejected": -0.88625168800354,
      "logps/chosen": -1.4861788749694824,
      "logps/rejected": -1.9488956928253174,
      "loss": 1.5617,
      "nll_loss": 1.516565203666687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14861789345741272,
      "rewards/margins": 0.0462716668844223,
      "rewards/rejected": -0.19488956034183502,
      "step": 476
    },
    {
      "epoch": 0.75177304964539,
      "grad_norm": 0.19172121584415436,
      "learning_rate": 3.148278341971795e-06,
      "log_odds_chosen": 0.6366340517997742,
      "log_odds_ratio": -0.4304071068763733,
      "logits/chosen": -0.2341357320547104,
      "logits/rejected": -0.8897933959960938,
      "logps/chosen": -1.5178066492080688,
      "logps/rejected": -2.045914888381958,
      "loss": 1.5838,
      "nll_loss": 1.5407583713531494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15178067982196808,
      "rewards/margins": 0.052810803055763245,
      "rewards/rejected": -0.20459145307540894,
      "step": 477
    },
    {
      "epoch": 0.7533490937746257,
      "grad_norm": 0.20052027702331543,
      "learning_rate": 3.143773217195929e-06,
      "log_odds_chosen": 0.7204963564872742,
      "log_odds_ratio": -0.40443986654281616,
      "logits/chosen": -0.24737702310085297,
      "logits/rejected": -1.0543692111968994,
      "logps/chosen": -1.5792121887207031,
      "logps/rejected": -2.187511920928955,
      "loss": 1.6363,
      "nll_loss": 1.5958765745162964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1579212099313736,
      "rewards/margins": 0.060829974710941315,
      "rewards/rejected": -0.2187511920928955,
      "step": 478
    },
    {
      "epoch": 0.7549251379038613,
      "grad_norm": 0.18838270008563995,
      "learning_rate": 3.139259451759714e-06,
      "log_odds_chosen": 0.5550141930580139,
      "log_odds_ratio": -0.4581400752067566,
      "logits/chosen": -0.2387973815202713,
      "logits/rejected": -0.9775084257125854,
      "logps/chosen": -1.5817269086837769,
      "logps/rejected": -2.044766426086426,
      "loss": 1.6553,
      "nll_loss": 1.6094508171081543,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15817268192768097,
      "rewards/margins": 0.046303942799568176,
      "rewards/rejected": -0.20447663962841034,
      "step": 479
    },
    {
      "epoch": 0.7565011820330969,
      "grad_norm": 0.1940333992242813,
      "learning_rate": 3.134737079762493e-06,
      "log_odds_chosen": 0.5709211230278015,
      "log_odds_ratio": -0.4525545537471771,
      "logits/chosen": -0.203268900513649,
      "logits/rejected": -0.9061958193778992,
      "logps/chosen": -1.5168800354003906,
      "logps/rejected": -1.9859740734100342,
      "loss": 1.5814,
      "nll_loss": 1.5361382961273193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15168799459934235,
      "rewards/margins": 0.046909406781196594,
      "rewards/rejected": -0.19859741628170013,
      "step": 480
    },
    {
      "epoch": 0.7580772261623325,
      "grad_norm": 0.19957613945007324,
      "learning_rate": 3.130206135368626e-06,
      "log_odds_chosen": 0.5962250232696533,
      "log_odds_ratio": -0.450898677110672,
      "logits/chosen": -0.24314109981060028,
      "logits/rejected": -0.7290536165237427,
      "logps/chosen": -1.4346656799316406,
      "logps/rejected": -1.9179528951644897,
      "loss": 1.5249,
      "nll_loss": 1.4798487424850464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14346656203269958,
      "rewards/margins": 0.048328716307878494,
      "rewards/rejected": -0.19179528951644897,
      "step": 481
    },
    {
      "epoch": 0.7596532702915682,
      "grad_norm": 0.267019122838974,
      "learning_rate": 3.1256666528072327e-06,
      "log_odds_chosen": 0.761562705039978,
      "log_odds_ratio": -0.38860708475112915,
      "logits/chosen": -0.23340564966201782,
      "logits/rejected": -0.7684799432754517,
      "logps/chosen": -1.4836543798446655,
      "logps/rejected": -2.114672899246216,
      "loss": 1.5491,
      "nll_loss": 1.5102040767669678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14836543798446655,
      "rewards/margins": 0.06310184299945831,
      "rewards/rejected": -0.21146729588508606,
      "step": 482
    },
    {
      "epoch": 0.7612293144208038,
      "grad_norm": 0.1857529729604721,
      "learning_rate": 3.121118666371937e-06,
      "log_odds_chosen": 0.5648155808448792,
      "log_odds_ratio": -0.456367164850235,
      "logits/chosen": -0.1727515161037445,
      "logits/rejected": -1.1376999616622925,
      "logps/chosen": -1.545863389968872,
      "logps/rejected": -2.0154378414154053,
      "loss": 1.6028,
      "nll_loss": 1.557152271270752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15458634495735168,
      "rewards/margins": 0.04695742577314377,
      "rewards/rejected": -0.20154377818107605,
      "step": 483
    },
    {
      "epoch": 0.7628053585500394,
      "grad_norm": 0.1998901218175888,
      "learning_rate": 3.1165622104206034e-06,
      "log_odds_chosen": 0.744139552116394,
      "log_odds_ratio": -0.3924621343612671,
      "logits/chosen": -0.3864690661430359,
      "logits/rejected": -0.8910558819770813,
      "logps/chosen": -1.4746683835983276,
      "logps/rejected": -2.0918891429901123,
      "loss": 1.5428,
      "nll_loss": 1.5035266876220703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14746685326099396,
      "rewards/margins": 0.061722077429294586,
      "rewards/rejected": -0.20918893814086914,
      "step": 484
    },
    {
      "epoch": 0.764381402679275,
      "grad_norm": 0.1890469342470169,
      "learning_rate": 3.1119973193750816e-06,
      "log_odds_chosen": 0.6397299766540527,
      "log_odds_ratio": -0.4284901022911072,
      "logits/chosen": -0.2745180130004883,
      "logits/rejected": -0.9615079164505005,
      "logps/chosen": -1.4832903146743774,
      "logps/rejected": -2.0098438262939453,
      "loss": 1.5453,
      "nll_loss": 1.502469778060913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14832903444766998,
      "rewards/margins": 0.052655380219221115,
      "rewards/rejected": -0.200984388589859,
      "step": 485
    },
    {
      "epoch": 0.7659574468085106,
      "grad_norm": 0.23916488885879517,
      "learning_rate": 3.1074240277209408e-06,
      "log_odds_chosen": 0.596796452999115,
      "log_odds_ratio": -0.4465484321117401,
      "logits/chosen": -0.27292871475219727,
      "logits/rejected": -0.8987700939178467,
      "logps/chosen": -1.5037661790847778,
      "logps/rejected": -1.9917497634887695,
      "loss": 1.5509,
      "nll_loss": 1.5062373876571655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15037663280963898,
      "rewards/margins": 0.04879835247993469,
      "rewards/rejected": -0.19917498528957367,
      "step": 486
    },
    {
      "epoch": 0.7675334909377463,
      "grad_norm": 0.21090711653232574,
      "learning_rate": 3.102842370007217e-06,
      "log_odds_chosen": 0.6544410586357117,
      "log_odds_ratio": -0.42719337344169617,
      "logits/chosen": -0.16714385151863098,
      "logits/rejected": -0.8268420100212097,
      "logps/chosen": -1.5153406858444214,
      "logps/rejected": -2.051680088043213,
      "loss": 1.5812,
      "nll_loss": 1.538484811782837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1515340805053711,
      "rewards/margins": 0.0536339245736599,
      "rewards/rejected": -0.2051679790019989,
      "step": 487
    },
    {
      "epoch": 0.7691095350669819,
      "grad_norm": 0.1895013153553009,
      "learning_rate": 3.0982523808461454e-06,
      "log_odds_chosen": 0.5242911577224731,
      "log_odds_ratio": -0.4711127281188965,
      "logits/chosen": -0.16803131997585297,
      "logits/rejected": -0.8569961786270142,
      "logps/chosen": -1.557231068611145,
      "logps/rejected": -1.9868173599243164,
      "loss": 1.6165,
      "nll_loss": 1.569408893585205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15572310984134674,
      "rewards/margins": 0.04295860975980759,
      "rewards/rejected": -0.19868171215057373,
      "step": 488
    },
    {
      "epoch": 0.7706855791962175,
      "grad_norm": 0.1873975694179535,
      "learning_rate": 3.0936540949129006e-06,
      "log_odds_chosen": 0.6198223829269409,
      "log_odds_ratio": -0.44185134768486023,
      "logits/chosen": -0.21959857642650604,
      "logits/rejected": -0.8527919054031372,
      "logps/chosen": -1.5346941947937012,
      "logps/rejected": -2.0513901710510254,
      "loss": 1.601,
      "nll_loss": 1.5567827224731445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15346942842006683,
      "rewards/margins": 0.05166961997747421,
      "rewards/rejected": -0.20513902604579926,
      "step": 489
    },
    {
      "epoch": 0.7722616233254531,
      "grad_norm": 0.1847822517156601,
      "learning_rate": 3.0890475469453378e-06,
      "log_odds_chosen": 0.7269415855407715,
      "log_odds_ratio": -0.4065399765968323,
      "logits/chosen": -0.2197483777999878,
      "logits/rejected": -0.9742456078529358,
      "logps/chosen": -1.4097678661346436,
      "logps/rejected": -2.001023054122925,
      "loss": 1.4795,
      "nll_loss": 1.4388233423233032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14097680151462555,
      "rewards/margins": 0.05912550166249275,
      "rewards/rejected": -0.2001022845506668,
      "step": 490
    },
    {
      "epoch": 0.7738376674546887,
      "grad_norm": 0.18551437556743622,
      "learning_rate": 3.0844327717437263e-06,
      "log_odds_chosen": 0.6703569889068604,
      "log_odds_ratio": -0.42353707551956177,
      "logits/chosen": -0.23779956996440887,
      "logits/rejected": -0.9448487758636475,
      "logps/chosen": -1.4378920793533325,
      "logps/rejected": -1.9892666339874268,
      "loss": 1.5099,
      "nll_loss": 1.4675414562225342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14378920197486877,
      "rewards/margins": 0.055137455463409424,
      "rewards/rejected": -0.198926642537117,
      "step": 491
    },
    {
      "epoch": 0.7754137115839244,
      "grad_norm": 0.17690856754779816,
      "learning_rate": 3.0798098041704892e-06,
      "log_odds_chosen": 0.4932956099510193,
      "log_odds_ratio": -0.48199495673179626,
      "logits/chosen": -0.18783992528915405,
      "logits/rejected": -0.9837126135826111,
      "logps/chosen": -1.36940336227417,
      "logps/rejected": -1.7628213167190552,
      "loss": 1.4616,
      "nll_loss": 1.4133892059326172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13694033026695251,
      "rewards/margins": 0.039341796189546585,
      "rewards/rejected": -0.17628213763237,
      "step": 492
    },
    {
      "epoch": 0.77698975571316,
      "grad_norm": 0.19573038816452026,
      "learning_rate": 3.0751786791499368e-06,
      "log_odds_chosen": 0.6043769717216492,
      "log_odds_ratio": -0.44333675503730774,
      "logits/chosen": -0.18606124818325043,
      "logits/rejected": -0.9217195510864258,
      "logps/chosen": -1.558120608329773,
      "logps/rejected": -2.0642759799957275,
      "loss": 1.6268,
      "nll_loss": 1.5824553966522217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15581203997135162,
      "rewards/margins": 0.050615549087524414,
      "rewards/rejected": -0.20642760396003723,
      "step": 493
    },
    {
      "epoch": 0.7785657998423956,
      "grad_norm": 0.18777020275592804,
      "learning_rate": 3.070539431668008e-06,
      "log_odds_chosen": 0.6096667647361755,
      "log_odds_ratio": -0.4396899938583374,
      "logits/chosen": -0.19759081304073334,
      "logits/rejected": -0.8988550305366516,
      "logps/chosen": -1.5582072734832764,
      "logps/rejected": -2.069912910461426,
      "loss": 1.6177,
      "nll_loss": 1.5736886262893677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15582072734832764,
      "rewards/margins": 0.051170557737350464,
      "rewards/rejected": -0.2069912701845169,
      "step": 494
    },
    {
      "epoch": 0.7801418439716312,
      "grad_norm": 0.19385822117328644,
      "learning_rate": 3.0658920967720018e-06,
      "log_odds_chosen": 0.777790904045105,
      "log_odds_ratio": -0.3871975541114807,
      "logits/chosen": -0.3274940252304077,
      "logits/rejected": -0.924656331539154,
      "logps/chosen": -1.495604395866394,
      "logps/rejected": -2.147104263305664,
      "loss": 1.5648,
      "nll_loss": 1.5260502099990845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14956045150756836,
      "rewards/margins": 0.06514997035264969,
      "rewards/rejected": -0.21471041440963745,
      "step": 495
    },
    {
      "epoch": 0.7817178881008668,
      "grad_norm": 0.19317972660064697,
      "learning_rate": 3.0612367095703116e-06,
      "log_odds_chosen": 0.7192878723144531,
      "log_odds_ratio": -0.4008732736110687,
      "logits/chosen": -0.1955651044845581,
      "logits/rejected": -1.0837275981903076,
      "logps/chosen": -1.5353162288665771,
      "logps/rejected": -2.139145851135254,
      "loss": 1.5897,
      "nll_loss": 1.5495661497116089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15353162586688995,
      "rewards/margins": 0.06038297340273857,
      "rewards/rejected": -0.21391460299491882,
      "step": 496
    },
    {
      "epoch": 0.7832939322301025,
      "grad_norm": 0.1901102215051651,
      "learning_rate": 3.056573305232167e-06,
      "log_odds_chosen": 0.7629727721214294,
      "log_odds_ratio": -0.38942158222198486,
      "logits/chosen": -0.2240157276391983,
      "logits/rejected": -1.0085515975952148,
      "logps/chosen": -1.4874334335327148,
      "logps/rejected": -2.1163125038146973,
      "loss": 1.554,
      "nll_loss": 1.5150222778320312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14874334633350372,
      "rewards/margins": 0.06288789212703705,
      "rewards/rejected": -0.21163123846054077,
      "step": 497
    },
    {
      "epoch": 0.7848699763593381,
      "grad_norm": 0.18616610765457153,
      "learning_rate": 3.051901918987359e-06,
      "log_odds_chosen": 0.7289243340492249,
      "log_odds_ratio": -0.3984372615814209,
      "logits/chosen": -0.3597657084465027,
      "logits/rejected": -1.0724691152572632,
      "logps/chosen": -1.43277108669281,
      "logps/rejected": -2.0301997661590576,
      "loss": 1.5008,
      "nll_loss": 1.4609586000442505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.143277108669281,
      "rewards/margins": 0.05974285304546356,
      "rewards/rejected": -0.20301997661590576,
      "step": 498
    },
    {
      "epoch": 0.7864460204885737,
      "grad_norm": 0.23125092685222626,
      "learning_rate": 3.047222586125979e-06,
      "log_odds_chosen": 0.7393041253089905,
      "log_odds_ratio": -0.39401498436927795,
      "logits/chosen": -0.16603292524814606,
      "logits/rejected": -0.6574579477310181,
      "logps/chosen": -1.4413788318634033,
      "logps/rejected": -2.049246072769165,
      "loss": 1.5197,
      "nll_loss": 1.4802521467208862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1441378891468048,
      "rewards/margins": 0.06078672781586647,
      "rewards/rejected": -0.20492461323738098,
      "step": 499
    },
    {
      "epoch": 0.7880220646178093,
      "grad_norm": 0.18493805825710297,
      "learning_rate": 3.042535341998152e-06,
      "log_odds_chosen": 0.5539823174476624,
      "log_odds_ratio": -0.4595508873462677,
      "logits/chosen": -0.08695846050977707,
      "logits/rejected": -0.9634714722633362,
      "logps/chosen": -1.5871978998184204,
      "logps/rejected": -2.052105665206909,
      "loss": 1.6325,
      "nll_loss": 1.5865356922149658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15871979296207428,
      "rewards/margins": 0.04649076238274574,
      "rewards/rejected": -0.20521055161952972,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 1270,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}