{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 1270,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015760441292356187,
      "grad_norm": 0.7906547784805298,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.391758531332016,
      "log_odds_ratio": -0.534308910369873,
      "logits/chosen": -0.8063042759895325,
      "logits/rejected": 0.05536898970603943,
      "logps/chosen": -2.0363523960113525,
      "logps/rejected": -2.3849008083343506,
      "loss": 2.2935,
      "nll_loss": 2.240062952041626,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20363523066043854,
      "rewards/margins": 0.03485485166311264,
      "rewards/rejected": -0.23849007487297058,
      "step": 1
    },
    {
      "epoch": 0.0031520882584712374,
      "grad_norm": 0.7304520606994629,
      "learning_rate": 3.1496062992125985e-08,
      "log_odds_chosen": 0.3922337293624878,
      "log_odds_ratio": -0.5238180756568909,
      "logits/chosen": -0.7810883522033691,
      "logits/rejected": 0.1618170589208603,
      "logps/chosen": -1.9294850826263428,
      "logps/rejected": -2.2716217041015625,
      "loss": 2.1792,
      "nll_loss": 2.1268293857574463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19294852018356323,
      "rewards/margins": 0.03421364724636078,
      "rewards/rejected": -0.227162167429924,
      "step": 2
    },
    {
      "epoch": 0.004728132387706856,
      "grad_norm": 0.8640011548995972,
      "learning_rate": 6.299212598425197e-08,
      "log_odds_chosen": 0.5660353899002075,
      "log_odds_ratio": -0.4610859751701355,
      "logits/chosen": -0.965283989906311,
      "logits/rejected": -0.0006121229380369186,
      "logps/chosen": -2.0610620975494385,
      "logps/rejected": -2.57210373878479,
      "loss": 2.3037,
      "nll_loss": 2.25754714012146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20610620081424713,
      "rewards/margins": 0.051104187965393066,
      "rewards/rejected": -0.2572104036808014,
      "step": 3
    },
    {
      "epoch": 0.006304176516942475,
      "grad_norm": 0.7450115084648132,
      "learning_rate": 9.448818897637795e-08,
      "log_odds_chosen": 0.5706238746643066,
      "log_odds_ratio": -0.464206337928772,
      "logits/chosen": -0.7551829218864441,
      "logits/rejected": 0.1403137892484665,
      "logps/chosen": -2.06683087348938,
      "logps/rejected": -2.584524631500244,
      "loss": 2.3069,
      "nll_loss": 2.260457754135132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20668308436870575,
      "rewards/margins": 0.05176939070224762,
      "rewards/rejected": -0.25845247507095337,
      "step": 4
    },
    {
      "epoch": 0.007880220646178092,
      "grad_norm": 0.8498793244361877,
      "learning_rate": 1.2598425196850394e-07,
      "log_odds_chosen": 0.7315797805786133,
      "log_odds_ratio": -0.41388875246047974,
      "logits/chosen": -0.8775473833084106,
      "logits/rejected": -0.0007386752404272556,
      "logps/chosen": -1.9901928901672363,
      "logps/rejected": -2.644153594970703,
      "loss": 2.2234,
      "nll_loss": 2.182061195373535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19901929795742035,
      "rewards/margins": 0.06539607048034668,
      "rewards/rejected": -0.26441535353660583,
      "step": 5
    },
    {
      "epoch": 0.009456264775413711,
      "grad_norm": 0.7722200751304626,
      "learning_rate": 1.5748031496062992e-07,
      "log_odds_chosen": 0.4112725853919983,
      "log_odds_ratio": -0.5124160647392273,
      "logits/chosen": -0.7046896815299988,
      "logits/rejected": 0.17564038932323456,
      "logps/chosen": -1.9167629480361938,
      "logps/rejected": -2.2782955169677734,
      "loss": 2.1976,
      "nll_loss": 2.1463887691497803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1916763037443161,
      "rewards/margins": 0.036153268069028854,
      "rewards/rejected": -0.22782957553863525,
      "step": 6
    },
    {
      "epoch": 0.01103230890464933,
      "grad_norm": 0.7802104949951172,
      "learning_rate": 1.889763779527559e-07,
      "log_odds_chosen": 0.5916500091552734,
      "log_odds_ratio": -0.45217183232307434,
      "logits/chosen": -0.7392858266830444,
      "logits/rejected": 0.03766726329922676,
      "logps/chosen": -1.985681176185608,
      "logps/rejected": -2.5173680782318115,
      "loss": 2.2284,
      "nll_loss": 2.1831605434417725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19856813549995422,
      "rewards/margins": 0.05316868796944618,
      "rewards/rejected": -0.2517368197441101,
      "step": 7
    },
    {
      "epoch": 0.01260835303388495,
      "grad_norm": 0.9769444465637207,
      "learning_rate": 2.2047244094488187e-07,
      "log_odds_chosen": 0.8452985882759094,
      "log_odds_ratio": -0.36809873580932617,
      "logits/chosen": -0.8406262397766113,
      "logits/rejected": -0.015037253499031067,
      "logps/chosen": -2.1048223972320557,
      "logps/rejected": -2.8729093074798584,
      "loss": 2.347,
      "nll_loss": 2.3101491928100586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21048225462436676,
      "rewards/margins": 0.07680864632129669,
      "rewards/rejected": -0.28729093074798584,
      "step": 8
    },
    {
      "epoch": 0.014184397163120567,
      "grad_norm": 0.7307494878768921,
      "learning_rate": 2.519685039370079e-07,
      "log_odds_chosen": 0.4397449791431427,
      "log_odds_ratio": -0.5038886070251465,
      "logits/chosen": -0.7611249089241028,
      "logits/rejected": 0.1530451774597168,
      "logps/chosen": -1.9963405132293701,
      "logps/rejected": -2.3867950439453125,
      "loss": 2.2222,
      "nll_loss": 2.1718318462371826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19963406026363373,
      "rewards/margins": 0.03904542699456215,
      "rewards/rejected": -0.23867949843406677,
      "step": 9
    },
    {
      "epoch": 0.015760441292356184,
      "grad_norm": 0.746634304523468,
      "learning_rate": 2.8346456692913386e-07,
      "log_odds_chosen": 0.42592883110046387,
      "log_odds_ratio": -0.5083432793617249,
      "logits/chosen": -0.794756293296814,
      "logits/rejected": -0.07558537274599075,
      "logps/chosen": -2.0108375549316406,
      "logps/rejected": -2.390498161315918,
      "loss": 2.2518,
      "nll_loss": 2.2009661197662354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20108374953269958,
      "rewards/margins": 0.03796607255935669,
      "rewards/rejected": -0.23904980719089508,
      "step": 10
    },
    {
      "epoch": 0.017336485421591805,
      "grad_norm": 0.8190391063690186,
      "learning_rate": 3.1496062992125984e-07,
      "log_odds_chosen": 0.4864140450954437,
      "log_odds_ratio": -0.48338061571121216,
      "logits/chosen": -0.7586538791656494,
      "logits/rejected": 0.0021292148157954216,
      "logps/chosen": -2.1001176834106445,
      "logps/rejected": -2.536212921142578,
      "loss": 2.3376,
      "nll_loss": 2.2892978191375732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2100117951631546,
      "rewards/margins": 0.04360952228307724,
      "rewards/rejected": -0.25362130999565125,
      "step": 11
    },
    {
      "epoch": 0.018912529550827423,
      "grad_norm": 0.9358005523681641,
      "learning_rate": 3.464566929133858e-07,
      "log_odds_chosen": 0.5531883835792542,
      "log_odds_ratio": -0.45853984355926514,
      "logits/chosen": -0.8949778079986572,
      "logits/rejected": 0.16316448152065277,
      "logps/chosen": -2.152214288711548,
      "logps/rejected": -2.653200387954712,
      "loss": 2.3963,
      "nll_loss": 2.350400924682617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21522141993045807,
      "rewards/margins": 0.050098612904548645,
      "rewards/rejected": -0.2653200626373291,
      "step": 12
    },
    {
      "epoch": 0.02048857368006304,
      "grad_norm": 0.9224507212638855,
      "learning_rate": 3.779527559055118e-07,
      "log_odds_chosen": 0.5066958069801331,
      "log_odds_ratio": -0.4827364981174469,
      "logits/chosen": -0.9888880252838135,
      "logits/rejected": 0.2006295621395111,
      "logps/chosen": -2.050516128540039,
      "logps/rejected": -2.5026915073394775,
      "loss": 2.2934,
      "nll_loss": 2.2450859546661377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20505160093307495,
      "rewards/margins": 0.04521755874156952,
      "rewards/rejected": -0.25026917457580566,
      "step": 13
    },
    {
      "epoch": 0.02206461780929866,
      "grad_norm": 0.7721648216247559,
      "learning_rate": 4.0944881889763777e-07,
      "log_odds_chosen": 0.3940409719944,
      "log_odds_ratio": -0.5303590297698975,
      "logits/chosen": -0.7946900725364685,
      "logits/rejected": -0.17097121477127075,
      "logps/chosen": -2.034684181213379,
      "logps/rejected": -2.387862205505371,
      "loss": 2.275,
      "nll_loss": 2.2220115661621094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2034684270620346,
      "rewards/margins": 0.035317789763212204,
      "rewards/rejected": -0.23878619074821472,
      "step": 14
    },
    {
      "epoch": 0.02364066193853428,
      "grad_norm": 0.8610419631004333,
      "learning_rate": 4.4094488188976375e-07,
      "log_odds_chosen": 0.4232204556465149,
      "log_odds_ratio": -0.5071219205856323,
      "logits/chosen": -0.8932058811187744,
      "logits/rejected": 0.05950244888663292,
      "logps/chosen": -2.0276124477386475,
      "logps/rejected": -2.4058046340942383,
      "loss": 2.2822,
      "nll_loss": 2.23144268989563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20276124775409698,
      "rewards/margins": 0.037819236516952515,
      "rewards/rejected": -0.2405804693698883,
      "step": 15
    },
    {
      "epoch": 0.0252167060677699,
      "grad_norm": 0.8575419783592224,
      "learning_rate": 4.7244094488188973e-07,
      "log_odds_chosen": 0.43113377690315247,
      "log_odds_ratio": -0.5063679814338684,
      "logits/chosen": -0.8862703442573547,
      "logits/rejected": 0.034714244306087494,
      "logps/chosen": -2.03637433052063,
      "logps/rejected": -2.4213576316833496,
      "loss": 2.2825,
      "nll_loss": 2.231827974319458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20363745093345642,
      "rewards/margins": 0.03849831968545914,
      "rewards/rejected": -0.24213576316833496,
      "step": 16
    },
    {
      "epoch": 0.026792750197005517,
      "grad_norm": 0.7937728762626648,
      "learning_rate": 5.039370078740158e-07,
      "log_odds_chosen": 0.642987072467804,
      "log_odds_ratio": -0.4362831711769104,
      "logits/chosen": -0.7214955687522888,
      "logits/rejected": -0.03120867908000946,
      "logps/chosen": -1.9728035926818848,
      "logps/rejected": -2.5497608184814453,
      "loss": 2.2244,
      "nll_loss": 2.1807427406311035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19728036224842072,
      "rewards/margins": 0.057695720344781876,
      "rewards/rejected": -0.2549760937690735,
      "step": 17
    },
    {
      "epoch": 0.028368794326241134,
      "grad_norm": 0.8444339036941528,
      "learning_rate": 5.354330708661418e-07,
      "log_odds_chosen": 0.3567027449607849,
      "log_odds_ratio": -0.5442062020301819,
      "logits/chosen": -0.7939973473548889,
      "logits/rejected": 0.3661736845970154,
      "logps/chosen": -2.0135817527770996,
      "logps/rejected": -2.328972578048706,
      "loss": 2.2728,
      "nll_loss": 2.2184267044067383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20135818421840668,
      "rewards/margins": 0.03153908997774124,
      "rewards/rejected": -0.23289725184440613,
      "step": 18
    },
    {
      "epoch": 0.029944838455476755,
      "grad_norm": 0.7747983336448669,
      "learning_rate": 5.669291338582677e-07,
      "log_odds_chosen": 0.6433313488960266,
      "log_odds_ratio": -0.4306049048900604,
      "logits/chosen": -0.9112986326217651,
      "logits/rejected": -0.013520637527108192,
      "logps/chosen": -1.900106430053711,
      "logps/rejected": -2.468703269958496,
      "loss": 2.1641,
      "nll_loss": 2.1210854053497314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1900106519460678,
      "rewards/margins": 0.056859664618968964,
      "rewards/rejected": -0.24687032401561737,
      "step": 19
    },
    {
      "epoch": 0.03152088258471237,
      "grad_norm": 0.8091546297073364,
      "learning_rate": 5.984251968503937e-07,
      "log_odds_chosen": 0.5987537503242493,
      "log_odds_ratio": -0.4577232897281647,
      "logits/chosen": -0.7413605451583862,
      "logits/rejected": 0.027445267885923386,
      "logps/chosen": -2.060060977935791,
      "logps/rejected": -2.60052490234375,
      "loss": 2.316,
      "nll_loss": 2.2701833248138428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20600609481334686,
      "rewards/margins": 0.0540463849902153,
      "rewards/rejected": -0.26005247235298157,
      "step": 20
    },
    {
      "epoch": 0.03309692671394799,
      "grad_norm": 0.8816735744476318,
      "learning_rate": 6.299212598425197e-07,
      "log_odds_chosen": 0.3807871341705322,
      "log_odds_ratio": -0.5288621187210083,
      "logits/chosen": -0.8584674000740051,
      "logits/rejected": 0.11941666901111603,
      "logps/chosen": -2.117626428604126,
      "logps/rejected": -2.4617509841918945,
      "loss": 2.3647,
      "nll_loss": 2.3118205070495605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21176263689994812,
      "rewards/margins": 0.0344124510884285,
      "rewards/rejected": -0.24617509543895721,
      "step": 21
    },
    {
      "epoch": 0.03467297084318361,
      "grad_norm": 0.8328697681427002,
      "learning_rate": 6.614173228346457e-07,
      "log_odds_chosen": 0.3781239092350006,
      "log_odds_ratio": -0.5352512001991272,
      "logits/chosen": -0.8822906017303467,
      "logits/rejected": 0.2483917474746704,
      "logps/chosen": -2.0555434226989746,
      "logps/rejected": -2.394667625427246,
      "loss": 2.311,
      "nll_loss": 2.257483959197998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20555433630943298,
      "rewards/margins": 0.03391243517398834,
      "rewards/rejected": -0.23946677148342133,
      "step": 22
    },
    {
      "epoch": 0.036249014972419225,
      "grad_norm": 0.8317603468894958,
      "learning_rate": 6.929133858267716e-07,
      "log_odds_chosen": 0.4104730486869812,
      "log_odds_ratio": -0.5118555426597595,
      "logits/chosen": -0.8775284886360168,
      "logits/rejected": 0.08889060467481613,
      "logps/chosen": -2.0935451984405518,
      "logps/rejected": -2.462343454360962,
      "loss": 2.3226,
      "nll_loss": 2.271423578262329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20935453474521637,
      "rewards/margins": 0.036879826337099075,
      "rewards/rejected": -0.24623435735702515,
      "step": 23
    },
    {
      "epoch": 0.037825059101654845,
      "grad_norm": 0.7823754549026489,
      "learning_rate": 7.244094488188977e-07,
      "log_odds_chosen": 0.44625383615493774,
      "log_odds_ratio": -0.5046113133430481,
      "logits/chosen": -0.5757933259010315,
      "logits/rejected": 0.12505289912223816,
      "logps/chosen": -2.035513162612915,
      "logps/rejected": -2.434288263320923,
      "loss": 2.2836,
      "nll_loss": 2.2331085205078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20355132222175598,
      "rewards/margins": 0.039877522736787796,
      "rewards/rejected": -0.24342885613441467,
      "step": 24
    },
    {
      "epoch": 0.039401103230890466,
      "grad_norm": 0.9246788024902344,
      "learning_rate": 7.559055118110236e-07,
      "log_odds_chosen": 0.375238299369812,
      "log_odds_ratio": -0.5334768891334534,
      "logits/chosen": -0.9162179231643677,
      "logits/rejected": -0.03538450971245766,
      "logps/chosen": -2.0804378986358643,
      "logps/rejected": -2.421753168106079,
      "loss": 2.3239,
      "nll_loss": 2.2705116271972656,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20804381370544434,
      "rewards/margins": 0.03413153439760208,
      "rewards/rejected": -0.24217534065246582,
      "step": 25
    },
    {
      "epoch": 0.04097714736012608,
      "grad_norm": 0.8534817695617676,
      "learning_rate": 7.874015748031496e-07,
      "log_odds_chosen": 0.4714248776435852,
      "log_odds_ratio": -0.49334338307380676,
      "logits/chosen": -1.0039349794387817,
      "logits/rejected": -0.07227999716997147,
      "logps/chosen": -2.0296528339385986,
      "logps/rejected": -2.4532673358917236,
      "loss": 2.2775,
      "nll_loss": 2.228158473968506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20296527445316315,
      "rewards/margins": 0.04236144199967384,
      "rewards/rejected": -0.24532672762870789,
      "step": 26
    },
    {
      "epoch": 0.0425531914893617,
      "grad_norm": 0.8221213221549988,
      "learning_rate": 8.188976377952755e-07,
      "log_odds_chosen": 0.7778643369674683,
      "log_odds_ratio": -0.396297425031662,
      "logits/chosen": -0.8032987713813782,
      "logits/rejected": 0.048319678753614426,
      "logps/chosen": -2.0571279525756836,
      "logps/rejected": -2.763603687286377,
      "loss": 2.306,
      "nll_loss": 2.2663841247558594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20571278035640717,
      "rewards/margins": 0.07064756751060486,
      "rewards/rejected": -0.2763603925704956,
      "step": 27
    },
    {
      "epoch": 0.04412923561859732,
      "grad_norm": 0.8112096786499023,
      "learning_rate": 8.503937007874016e-07,
      "log_odds_chosen": 0.3961740732192993,
      "log_odds_ratio": -0.5245606899261475,
      "logits/chosen": -0.841758668422699,
      "logits/rejected": 0.1445307582616806,
      "logps/chosen": -2.0094542503356934,
      "logps/rejected": -2.3631062507629395,
      "loss": 2.2755,
      "nll_loss": 2.223043918609619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2009454220533371,
      "rewards/margins": 0.03536517173051834,
      "rewards/rejected": -0.23631060123443604,
      "step": 28
    },
    {
      "epoch": 0.045705279747832936,
      "grad_norm": 0.7597041726112366,
      "learning_rate": 8.818897637795275e-07,
      "log_odds_chosen": 0.38566550612449646,
      "log_odds_ratio": -0.5244230031967163,
      "logits/chosen": -0.6991312503814697,
      "logits/rejected": 0.12269733846187592,
      "logps/chosen": -2.1073381900787354,
      "logps/rejected": -2.4572627544403076,
      "loss": 2.3533,
      "nll_loss": 2.3008675575256348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2107338309288025,
      "rewards/margins": 0.034992463886737823,
      "rewards/rejected": -0.24572627246379852,
      "step": 29
    },
    {
      "epoch": 0.04728132387706856,
      "grad_norm": 0.9624651074409485,
      "learning_rate": 9.133858267716535e-07,
      "log_odds_chosen": 0.6133925318717957,
      "log_odds_ratio": -0.4450993537902832,
      "logits/chosen": -0.8754658102989197,
      "logits/rejected": -0.06702499091625214,
      "logps/chosen": -2.0876407623291016,
      "logps/rejected": -2.642871141433716,
      "loss": 2.3645,
      "nll_loss": 2.320007801055908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20876409113407135,
      "rewards/margins": 0.05552302300930023,
      "rewards/rejected": -0.2642871141433716,
      "step": 30
    },
    {
      "epoch": 0.04885736800630418,
      "grad_norm": 0.8790305256843567,
      "learning_rate": 9.448818897637795e-07,
      "log_odds_chosen": 0.661622166633606,
      "log_odds_ratio": -0.4319807291030884,
      "logits/chosen": -0.8294932246208191,
      "logits/rejected": 0.24975837767124176,
      "logps/chosen": -2.129549264907837,
      "logps/rejected": -2.7335667610168457,
      "loss": 2.3622,
      "nll_loss": 2.3190011978149414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21295495331287384,
      "rewards/margins": 0.06040172278881073,
      "rewards/rejected": -0.27335667610168457,
      "step": 31
    },
    {
      "epoch": 0.0504334121355398,
      "grad_norm": 0.7277955412864685,
      "learning_rate": 9.763779527559055e-07,
      "log_odds_chosen": 0.7255255579948425,
      "log_odds_ratio": -0.41055694222450256,
      "logits/chosen": -0.7205650806427002,
      "logits/rejected": -0.08150428533554077,
      "logps/chosen": -1.98775053024292,
      "logps/rejected": -2.6428921222686768,
      "loss": 2.2284,
      "nll_loss": 2.1873605251312256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19877506792545319,
      "rewards/margins": 0.06551413238048553,
      "rewards/rejected": -0.2642892003059387,
      "step": 32
    },
    {
      "epoch": 0.05200945626477541,
      "grad_norm": 0.7199560403823853,
      "learning_rate": 1.0078740157480315e-06,
      "log_odds_chosen": 0.5971649289131165,
      "log_odds_ratio": -0.44201111793518066,
      "logits/chosen": -0.6816457509994507,
      "logits/rejected": 0.10904838144779205,
      "logps/chosen": -2.0162274837493896,
      "logps/rejected": -2.551992893218994,
      "loss": 2.2488,
      "nll_loss": 2.20456862449646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20162276923656464,
      "rewards/margins": 0.0535765141248703,
      "rewards/rejected": -0.25519925355911255,
      "step": 33
    },
    {
      "epoch": 0.05358550039401103,
      "grad_norm": 0.7956497073173523,
      "learning_rate": 1.0393700787401573e-06,
      "log_odds_chosen": 0.3375328779220581,
      "log_odds_ratio": -0.5497519373893738,
      "logits/chosen": -0.6955797672271729,
      "logits/rejected": 0.017697542905807495,
      "logps/chosen": -1.9696956872940063,
      "logps/rejected": -2.263640880584717,
      "loss": 2.2282,
      "nll_loss": 2.173248767852783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19696956872940063,
      "rewards/margins": 0.029394524171948433,
      "rewards/rejected": -0.2263641059398651,
      "step": 34
    },
    {
      "epoch": 0.055161544523246654,
      "grad_norm": 1.0039724111557007,
      "learning_rate": 1.0708661417322836e-06,
      "log_odds_chosen": 0.4184088408946991,
      "log_odds_ratio": -0.5078526735305786,
      "logits/chosen": -0.8881021738052368,
      "logits/rejected": 0.3383480906486511,
      "logps/chosen": -2.167163133621216,
      "logps/rejected": -2.5441415309906006,
      "loss": 2.4142,
      "nll_loss": 2.3633856773376465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21671633422374725,
      "rewards/margins": 0.03769781440496445,
      "rewards/rejected": -0.2544141411781311,
      "step": 35
    },
    {
      "epoch": 0.05673758865248227,
      "grad_norm": 0.8594557642936707,
      "learning_rate": 1.1023622047244094e-06,
      "log_odds_chosen": 0.5742133259773254,
      "log_odds_ratio": -0.4490886628627777,
      "logits/chosen": -0.991584300994873,
      "logits/rejected": 0.18718503415584564,
      "logps/chosen": -1.9992127418518066,
      "logps/rejected": -2.51247239112854,
      "loss": 2.2435,
      "nll_loss": 2.1985414028167725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19992126524448395,
      "rewards/margins": 0.05132598429918289,
      "rewards/rejected": -0.25124722719192505,
      "step": 36
    },
    {
      "epoch": 0.05831363278171789,
      "grad_norm": 0.7524440288543701,
      "learning_rate": 1.1338582677165354e-06,
      "log_odds_chosen": 0.46497225761413574,
      "log_odds_ratio": -0.4982447028160095,
      "logits/chosen": -0.8054354786872864,
      "logits/rejected": 0.2145153284072876,
      "logps/chosen": -1.9772131443023682,
      "logps/rejected": -2.387143135070801,
      "loss": 2.238,
      "nll_loss": 2.188201665878296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19772133231163025,
      "rewards/margins": 0.04099300131201744,
      "rewards/rejected": -0.238714337348938,
      "step": 37
    },
    {
      "epoch": 0.05988967691095351,
      "grad_norm": 0.8709132075309753,
      "learning_rate": 1.1653543307086612e-06,
      "log_odds_chosen": 0.3852269649505615,
      "log_odds_ratio": -0.5274136662483215,
      "logits/chosen": -0.9475684762001038,
      "logits/rejected": -0.014422068372368813,
      "logps/chosen": -1.9959006309509277,
      "logps/rejected": -2.3345746994018555,
      "loss": 2.2597,
      "nll_loss": 2.206918716430664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1995900422334671,
      "rewards/margins": 0.033867403864860535,
      "rewards/rejected": -0.23345746099948883,
      "step": 38
    },
    {
      "epoch": 0.061465721040189124,
      "grad_norm": 0.9423657059669495,
      "learning_rate": 1.1968503937007875e-06,
      "log_odds_chosen": 0.21351470053195953,
      "log_odds_ratio": -0.6031904816627502,
      "logits/chosen": -0.8634312152862549,
      "logits/rejected": 0.2100798338651657,
      "logps/chosen": -2.113935708999634,
      "logps/rejected": -2.3075613975524902,
      "loss": 2.3813,
      "nll_loss": 2.3209362030029297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2113935798406601,
      "rewards/margins": 0.019362568855285645,
      "rewards/rejected": -0.23075613379478455,
      "step": 39
    },
    {
      "epoch": 0.06304176516942474,
      "grad_norm": 0.739978015422821,
      "learning_rate": 1.2283464566929133e-06,
      "log_odds_chosen": 0.39791059494018555,
      "log_odds_ratio": -0.520595371723175,
      "logits/chosen": -0.8454897403717041,
      "logits/rejected": 0.05343304201960564,
      "logps/chosen": -1.95891535282135,
      "logps/rejected": -2.309115409851074,
      "loss": 2.2211,
      "nll_loss": 2.169041633605957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19589152932167053,
      "rewards/margins": 0.03501999378204346,
      "rewards/rejected": -0.23091153800487518,
      "step": 40
    },
    {
      "epoch": 0.06461780929866036,
      "grad_norm": 0.7263447046279907,
      "learning_rate": 1.2598425196850393e-06,
      "log_odds_chosen": 0.2720071077346802,
      "log_odds_ratio": -0.5738168358802795,
      "logits/chosen": -0.7613966464996338,
      "logits/rejected": 0.1504044085741043,
      "logps/chosen": -2.0094892978668213,
      "logps/rejected": -2.2524259090423584,
      "loss": 2.2512,
      "nll_loss": 2.1937756538391113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20094892382621765,
      "rewards/margins": 0.024293670430779457,
      "rewards/rejected": -0.22524259984493256,
      "step": 41
    },
    {
      "epoch": 0.06619385342789598,
      "grad_norm": 0.8427611589431763,
      "learning_rate": 1.2913385826771652e-06,
      "log_odds_chosen": 0.30227378010749817,
      "log_odds_ratio": -0.5599700808525085,
      "logits/chosen": -0.8331806063652039,
      "logits/rejected": 0.10789404809474945,
      "logps/chosen": -2.01611590385437,
      "logps/rejected": -2.2841808795928955,
      "loss": 2.2696,
      "nll_loss": 2.213594913482666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20161160826683044,
      "rewards/margins": 0.026806501671671867,
      "rewards/rejected": -0.22841811180114746,
      "step": 42
    },
    {
      "epoch": 0.0677698975571316,
      "grad_norm": 0.7538146376609802,
      "learning_rate": 1.3228346456692914e-06,
      "log_odds_chosen": 0.1851217895746231,
      "log_odds_ratio": -0.6138940453529358,
      "logits/chosen": -0.7334487438201904,
      "logits/rejected": 0.06407423317432404,
      "logps/chosen": -2.0350100994110107,
      "logps/rejected": -2.1991522312164307,
      "loss": 2.2871,
      "nll_loss": 2.2257142066955566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20350100100040436,
      "rewards/margins": 0.016414228826761246,
      "rewards/rejected": -0.2199152410030365,
      "step": 43
    },
    {
      "epoch": 0.06934594168636722,
      "grad_norm": 0.8424365520477295,
      "learning_rate": 1.3543307086614172e-06,
      "log_odds_chosen": 0.5302658677101135,
      "log_odds_ratio": -0.4734542965888977,
      "logits/chosen": -0.8889545202255249,
      "logits/rejected": 0.2788478136062622,
      "logps/chosen": -2.0052120685577393,
      "logps/rejected": -2.481436252593994,
      "loss": 2.2513,
      "nll_loss": 2.203930377960205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20052120089530945,
      "rewards/margins": 0.0476224310696125,
      "rewards/rejected": -0.24814364314079285,
      "step": 44
    },
    {
      "epoch": 0.07092198581560284,
      "grad_norm": 0.7189680933952332,
      "learning_rate": 1.3858267716535433e-06,
      "log_odds_chosen": 0.6273162961006165,
      "log_odds_ratio": -0.44312310218811035,
      "logits/chosen": -0.6784701347351074,
      "logits/rejected": 0.18980826437473297,
      "logps/chosen": -1.957032322883606,
      "logps/rejected": -2.5182816982269287,
      "loss": 2.2283,
      "nll_loss": 2.1839797496795654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19570323824882507,
      "rewards/margins": 0.056124940514564514,
      "rewards/rejected": -0.2518281638622284,
      "step": 45
    },
    {
      "epoch": 0.07249802994483845,
      "grad_norm": 0.9589075446128845,
      "learning_rate": 1.417322834645669e-06,
      "log_odds_chosen": 0.608472466468811,
      "log_odds_ratio": -0.4546305239200592,
      "logits/chosen": -0.9549905061721802,
      "logits/rejected": -0.06440718472003937,
      "logps/chosen": -2.0081634521484375,
      "logps/rejected": -2.555389165878296,
      "loss": 2.2656,
      "nll_loss": 2.220146894454956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.200816348195076,
      "rewards/margins": 0.054722562432289124,
      "rewards/rejected": -0.2555389106273651,
      "step": 46
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 0.796640157699585,
      "learning_rate": 1.4488188976377953e-06,
      "log_odds_chosen": 0.28869107365608215,
      "log_odds_ratio": -0.5726405382156372,
      "logits/chosen": -0.956196129322052,
      "logits/rejected": 0.0430324450135231,
      "logps/chosen": -2.036902904510498,
      "logps/rejected": -2.2967493534088135,
      "loss": 2.286,
      "nll_loss": 2.22874116897583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2036902904510498,
      "rewards/margins": 0.025984659790992737,
      "rewards/rejected": -0.22967493534088135,
      "step": 47
    },
    {
      "epoch": 0.07565011820330969,
      "grad_norm": 0.9021222591400146,
      "learning_rate": 1.4803149606299211e-06,
      "log_odds_chosen": 0.4264254570007324,
      "log_odds_ratio": -0.5128542184829712,
      "logits/chosen": -0.7777482867240906,
      "logits/rejected": 0.05279793590307236,
      "logps/chosen": -2.0905792713165283,
      "logps/rejected": -2.4763331413269043,
      "loss": 2.3535,
      "nll_loss": 2.302234411239624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20905792713165283,
      "rewards/margins": 0.038575392216444016,
      "rewards/rejected": -0.24763330817222595,
      "step": 48
    },
    {
      "epoch": 0.07722616233254531,
      "grad_norm": 0.721595823764801,
      "learning_rate": 1.5118110236220472e-06,
      "log_odds_chosen": 0.5007736682891846,
      "log_odds_ratio": -0.4825616478919983,
      "logits/chosen": -0.7792184948921204,
      "logits/rejected": 0.3143281936645508,
      "logps/chosen": -1.9676520824432373,
      "logps/rejected": -2.4107041358947754,
      "loss": 2.1893,
      "nll_loss": 2.1410772800445557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19676518440246582,
      "rewards/margins": 0.044305235147476196,
      "rewards/rejected": -0.2410704493522644,
      "step": 49
    },
    {
      "epoch": 0.07880220646178093,
      "grad_norm": 0.7847599387168884,
      "learning_rate": 1.543307086614173e-06,
      "log_odds_chosen": 0.29993894696235657,
      "log_odds_ratio": -0.5612264275550842,
      "logits/chosen": -0.8290573954582214,
      "logits/rejected": 0.06256364285945892,
      "logps/chosen": -2.046567916870117,
      "logps/rejected": -2.3153884410858154,
      "loss": 2.3029,
      "nll_loss": 2.2467591762542725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20465680956840515,
      "rewards/margins": 0.026882071048021317,
      "rewards/rejected": -0.23153886198997498,
      "step": 50
    },
    {
      "epoch": 0.08037825059101655,
      "grad_norm": 0.7472882270812988,
      "learning_rate": 1.5748031496062992e-06,
      "log_odds_chosen": 0.6475747227668762,
      "log_odds_ratio": -0.4279291033744812,
      "logits/chosen": -0.6958625316619873,
      "logits/rejected": 0.19621017575263977,
      "logps/chosen": -1.9286586046218872,
      "logps/rejected": -2.503767728805542,
      "loss": 2.171,
      "nll_loss": 2.1281983852386475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19286584854125977,
      "rewards/margins": 0.05751091241836548,
      "rewards/rejected": -0.25037679076194763,
      "step": 51
    },
    {
      "epoch": 0.08195429472025216,
      "grad_norm": 0.7411985993385315,
      "learning_rate": 1.6062992125984253e-06,
      "log_odds_chosen": 0.5698750615119934,
      "log_odds_ratio": -0.4549402594566345,
      "logits/chosen": -0.6439858078956604,
      "logits/rejected": 0.2690843939781189,
      "logps/chosen": -2.082113027572632,
      "logps/rejected": -2.5969245433807373,
      "loss": 2.3161,
      "nll_loss": 2.2706432342529297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20821130275726318,
      "rewards/margins": 0.05148116499185562,
      "rewards/rejected": -0.2596924602985382,
      "step": 52
    },
    {
      "epoch": 0.08353033884948778,
      "grad_norm": 0.8199878334999084,
      "learning_rate": 1.637795275590551e-06,
      "log_odds_chosen": 0.3187071979045868,
      "log_odds_ratio": -0.5507593154907227,
      "logits/chosen": -0.7804884910583496,
      "logits/rejected": 0.1774444580078125,
      "logps/chosen": -2.027057409286499,
      "logps/rejected": -2.3086814880371094,
      "loss": 2.276,
      "nll_loss": 2.2209625244140625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2027057409286499,
      "rewards/margins": 0.0281623974442482,
      "rewards/rejected": -0.2308681458234787,
      "step": 53
    },
    {
      "epoch": 0.0851063829787234,
      "grad_norm": 0.7717034816741943,
      "learning_rate": 1.6692913385826771e-06,
      "log_odds_chosen": 0.3182855248451233,
      "log_odds_ratio": -0.5493793487548828,
      "logits/chosen": -0.8416004180908203,
      "logits/rejected": 0.0956750363111496,
      "logps/chosen": -1.9605865478515625,
      "logps/rejected": -2.240133047103882,
      "loss": 2.2188,
      "nll_loss": 2.163889169692993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19605866074562073,
      "rewards/margins": 0.027954675257205963,
      "rewards/rejected": -0.2240133434534073,
      "step": 54
    },
    {
      "epoch": 0.08668242710795902,
      "grad_norm": 0.8444288372993469,
      "learning_rate": 1.7007874015748031e-06,
      "log_odds_chosen": 0.4131320118904114,
      "log_odds_ratio": -0.5114343762397766,
      "logits/chosen": -0.7947390675544739,
      "logits/rejected": -0.015921171754598618,
      "logps/chosen": -2.057856559753418,
      "logps/rejected": -2.428257942199707,
      "loss": 2.3167,
      "nll_loss": 2.2655208110809326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20578564703464508,
      "rewards/margins": 0.03704017400741577,
      "rewards/rejected": -0.24282580614089966,
      "step": 55
    },
    {
      "epoch": 0.08825847123719464,
      "grad_norm": 0.9216170907020569,
      "learning_rate": 1.7322834645669292e-06,
      "log_odds_chosen": 0.3599957823753357,
      "log_odds_ratio": -0.5413734316825867,
      "logits/chosen": -0.7153440117835999,
      "logits/rejected": 0.25690826773643494,
      "logps/chosen": -2.1631920337677,
      "logps/rejected": -2.4897618293762207,
      "loss": 2.4079,
      "nll_loss": 2.3537416458129883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21631920337677002,
      "rewards/margins": 0.03265698626637459,
      "rewards/rejected": -0.2489762008190155,
      "step": 56
    },
    {
      "epoch": 0.08983451536643026,
      "grad_norm": 0.698296070098877,
      "learning_rate": 1.763779527559055e-06,
      "log_odds_chosen": 0.37394940853118896,
      "log_odds_ratio": -0.5269332528114319,
      "logits/chosen": -0.7442439198493958,
      "logits/rejected": 0.17622074484825134,
      "logps/chosen": -2.0186767578125,
      "logps/rejected": -2.3512208461761475,
      "loss": 2.2588,
      "nll_loss": 2.2060835361480713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20186766982078552,
      "rewards/margins": 0.033254414796829224,
      "rewards/rejected": -0.23512208461761475,
      "step": 57
    },
    {
      "epoch": 0.09141055949566587,
      "grad_norm": 0.7946895360946655,
      "learning_rate": 1.795275590551181e-06,
      "log_odds_chosen": 0.39631205797195435,
      "log_odds_ratio": -0.5242730975151062,
      "logits/chosen": -0.7613857984542847,
      "logits/rejected": -0.0997920036315918,
      "logps/chosen": -1.9954885244369507,
      "logps/rejected": -2.352520227432251,
      "loss": 2.2451,
      "nll_loss": 2.192626953125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1995488554239273,
      "rewards/margins": 0.035703182220458984,
      "rewards/rejected": -0.2352520227432251,
      "step": 58
    },
    {
      "epoch": 0.09298660362490149,
      "grad_norm": 0.8159598112106323,
      "learning_rate": 1.826771653543307e-06,
      "log_odds_chosen": 0.7107657194137573,
      "log_odds_ratio": -0.4071710407733917,
      "logits/chosen": -0.6927255988121033,
      "logits/rejected": 0.18439871072769165,
      "logps/chosen": -2.0940284729003906,
      "logps/rejected": -2.741525411605835,
      "loss": 2.3155,
      "nll_loss": 2.2747528553009033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20940285921096802,
      "rewards/margins": 0.06474968045949936,
      "rewards/rejected": -0.274152547121048,
      "step": 59
    },
    {
      "epoch": 0.09456264775413711,
      "grad_norm": 0.8400620222091675,
      "learning_rate": 1.858267716535433e-06,
      "log_odds_chosen": 0.515011191368103,
      "log_odds_ratio": -0.4777131676673889,
      "logits/chosen": -0.8022063970565796,
      "logits/rejected": -0.011907435022294521,
      "logps/chosen": -2.0360805988311768,
      "logps/rejected": -2.5011496543884277,
      "loss": 2.2828,
      "nll_loss": 2.2350497245788574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20360803604125977,
      "rewards/margins": 0.04650692641735077,
      "rewards/rejected": -0.25011497735977173,
      "step": 60
    },
    {
      "epoch": 0.09613869188337273,
      "grad_norm": 0.861944317817688,
      "learning_rate": 1.889763779527559e-06,
      "log_odds_chosen": 0.26094532012939453,
      "log_odds_ratio": -0.580884575843811,
      "logits/chosen": -0.7177518010139465,
      "logits/rejected": 0.004429949447512627,
      "logps/chosen": -2.1590168476104736,
      "logps/rejected": -2.3958046436309814,
      "loss": 2.396,
      "nll_loss": 2.337953805923462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2159016877412796,
      "rewards/margins": 0.023678777739405632,
      "rewards/rejected": -0.23958046734333038,
      "step": 61
    },
    {
      "epoch": 0.09771473601260836,
      "grad_norm": 0.9885354042053223,
      "learning_rate": 1.9212598425196847e-06,
      "log_odds_chosen": 0.319558709859848,
      "log_odds_ratio": -0.5569351315498352,
      "logits/chosen": -0.7895075082778931,
      "logits/rejected": 0.13341262936592102,
      "logps/chosen": -2.1451916694641113,
      "logps/rejected": -2.4343841075897217,
      "loss": 2.3904,
      "nll_loss": 2.3347549438476562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21451915800571442,
      "rewards/margins": 0.02891925349831581,
      "rewards/rejected": -0.24343842267990112,
      "step": 62
    },
    {
      "epoch": 0.09929078014184398,
      "grad_norm": 0.7753734588623047,
      "learning_rate": 1.952755905511811e-06,
      "log_odds_chosen": 0.6025581359863281,
      "log_odds_ratio": -0.4435453712940216,
      "logits/chosen": -0.8974717855453491,
      "logits/rejected": -0.006312880665063858,
      "logps/chosen": -1.8660368919372559,
      "logps/rejected": -2.391263961791992,
      "loss": 2.127,
      "nll_loss": 2.0826830863952637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18660369515419006,
      "rewards/margins": 0.05252271145582199,
      "rewards/rejected": -0.23912642896175385,
      "step": 63
    },
    {
      "epoch": 0.1008668242710796,
      "grad_norm": 0.7583571076393127,
      "learning_rate": 1.9842519685039368e-06,
      "log_odds_chosen": 0.7151730060577393,
      "log_odds_ratio": -0.41310709714889526,
      "logits/chosen": -0.8151243925094604,
      "logits/rejected": 0.1741599142551422,
      "logps/chosen": -1.888004183769226,
      "logps/rejected": -2.5221481323242188,
      "loss": 2.1448,
      "nll_loss": 2.1034936904907227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18880042433738708,
      "rewards/margins": 0.06341440230607986,
      "rewards/rejected": -0.25221481919288635,
      "step": 64
    },
    {
      "epoch": 0.1024428684003152,
      "grad_norm": 0.7605916261672974,
      "learning_rate": 2.015748031496063e-06,
      "log_odds_chosen": 0.6391699314117432,
      "log_odds_ratio": -0.4316767752170563,
      "logits/chosen": -0.8075143098831177,
      "logits/rejected": -0.08158548176288605,
      "logps/chosen": -1.988166093826294,
      "logps/rejected": -2.5615224838256836,
      "loss": 2.2364,
      "nll_loss": 2.193208932876587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19881662726402283,
      "rewards/margins": 0.05733564868569374,
      "rewards/rejected": -0.25615230202674866,
      "step": 65
    },
    {
      "epoch": 0.10401891252955082,
      "grad_norm": 0.7569624185562134,
      "learning_rate": 2.047244094488189e-06,
      "log_odds_chosen": 0.47779494524002075,
      "log_odds_ratio": -0.49085569381713867,
      "logits/chosen": -0.7982770204544067,
      "logits/rejected": 0.1348651647567749,
      "logps/chosen": -2.072582721710205,
      "logps/rejected": -2.500241279602051,
      "loss": 2.3299,
      "nll_loss": 2.2808148860931396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20725825428962708,
      "rewards/margins": 0.042765870690345764,
      "rewards/rejected": -0.25002413988113403,
      "step": 66
    },
    {
      "epoch": 0.10559495665878645,
      "grad_norm": 0.792701244354248,
      "learning_rate": 2.0787401574803147e-06,
      "log_odds_chosen": 0.5766542553901672,
      "log_odds_ratio": -0.4502636194229126,
      "logits/chosen": -0.7492872476577759,
      "logits/rejected": 0.21686780452728271,
      "logps/chosen": -2.0231807231903076,
      "logps/rejected": -2.539250135421753,
      "loss": 2.2501,
      "nll_loss": 2.2051100730895996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20231808722019196,
      "rewards/margins": 0.051606953144073486,
      "rewards/rejected": -0.25392502546310425,
      "step": 67
    },
    {
      "epoch": 0.10717100078802207,
      "grad_norm": 0.9331843256950378,
      "learning_rate": 2.1102362204724405e-06,
      "log_odds_chosen": 0.4267440140247345,
      "log_odds_ratio": -0.510626494884491,
      "logits/chosen": -0.8918898105621338,
      "logits/rejected": 0.09724461287260056,
      "logps/chosen": -2.0314688682556152,
      "logps/rejected": -2.413175344467163,
      "loss": 2.3161,
      "nll_loss": 2.2650256156921387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20314690470695496,
      "rewards/margins": 0.038170650601387024,
      "rewards/rejected": -0.24131755530834198,
      "step": 68
    },
    {
      "epoch": 0.10874704491725769,
      "grad_norm": 0.8956534266471863,
      "learning_rate": 2.141732283464567e-06,
      "log_odds_chosen": 0.252351313829422,
      "log_odds_ratio": -0.5821089148521423,
      "logits/chosen": -0.5750194787979126,
      "logits/rejected": 0.10774422436952591,
      "logps/chosen": -2.1035430431365967,
      "logps/rejected": -2.328942060470581,
      "loss": 2.3665,
      "nll_loss": 2.3082704544067383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21035431325435638,
      "rewards/margins": 0.02253989316523075,
      "rewards/rejected": -0.23289422690868378,
      "step": 69
    },
    {
      "epoch": 0.11032308904649331,
      "grad_norm": 0.7740955352783203,
      "learning_rate": 2.173228346456693e-06,
      "log_odds_chosen": 0.4262125790119171,
      "log_odds_ratio": -0.508803129196167,
      "logits/chosen": -0.6928648948669434,
      "logits/rejected": 0.15248773992061615,
      "logps/chosen": -2.0291588306427,
      "logps/rejected": -2.410210132598877,
      "loss": 2.2883,
      "nll_loss": 2.237379312515259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20291589200496674,
      "rewards/margins": 0.03810514509677887,
      "rewards/rejected": -0.2410210371017456,
      "step": 70
    },
    {
      "epoch": 0.11189913317572892,
      "grad_norm": 0.9311802983283997,
      "learning_rate": 2.204724409448819e-06,
      "log_odds_chosen": 0.5261321067810059,
      "log_odds_ratio": -0.4780764877796173,
      "logits/chosen": -0.9807255268096924,
      "logits/rejected": -0.06657220423221588,
      "logps/chosen": -1.992706298828125,
      "logps/rejected": -2.4644320011138916,
      "loss": 2.2402,
      "nll_loss": 2.1923532485961914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19927063584327698,
      "rewards/margins": 0.04717259109020233,
      "rewards/rejected": -0.24644321203231812,
      "step": 71
    },
    {
      "epoch": 0.11347517730496454,
      "grad_norm": 0.801863431930542,
      "learning_rate": 2.2362204724409446e-06,
      "log_odds_chosen": 0.32317402958869934,
      "log_odds_ratio": -0.5580880641937256,
      "logits/chosen": -0.7395951151847839,
      "logits/rejected": 0.14740285277366638,
      "logps/chosen": -2.002899646759033,
      "logps/rejected": -2.290651321411133,
      "loss": 2.2534,
      "nll_loss": 2.1975412368774414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2002899944782257,
      "rewards/margins": 0.028775153681635857,
      "rewards/rejected": -0.2290651500225067,
      "step": 72
    },
    {
      "epoch": 0.11505122143420016,
      "grad_norm": 0.749702513217926,
      "learning_rate": 2.267716535433071e-06,
      "log_odds_chosen": 0.3722783923149109,
      "log_odds_ratio": -0.5272582769393921,
      "logits/chosen": -0.6592772006988525,
      "logits/rejected": 0.012937603518366814,
      "logps/chosen": -2.034029483795166,
      "logps/rejected": -2.3667900562286377,
      "loss": 2.2879,
      "nll_loss": 2.2351927757263184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20340295135974884,
      "rewards/margins": 0.03327607735991478,
      "rewards/rejected": -0.23667901754379272,
      "step": 73
    },
    {
      "epoch": 0.11662726556343578,
      "grad_norm": 0.8620375394821167,
      "learning_rate": 2.2992125984251967e-06,
      "log_odds_chosen": 0.4163767695426941,
      "log_odds_ratio": -0.5100041627883911,
      "logits/chosen": -0.8191778659820557,
      "logits/rejected": 0.17023847997188568,
      "logps/chosen": -1.923947811126709,
      "logps/rejected": -2.286886692047119,
      "loss": 2.1966,
      "nll_loss": 2.14556622505188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19239479303359985,
      "rewards/margins": 0.036293875426054,
      "rewards/rejected": -0.22868867218494415,
      "step": 74
    },
    {
      "epoch": 0.1182033096926714,
      "grad_norm": 0.817757248878479,
      "learning_rate": 2.3307086614173225e-06,
      "log_odds_chosen": 0.5530056953430176,
      "log_odds_ratio": -0.4610140919685364,
      "logits/chosen": -0.7592418193817139,
      "logits/rejected": 0.04180709272623062,
      "logps/chosen": -1.9531089067459106,
      "logps/rejected": -2.4464869499206543,
      "loss": 2.1989,
      "nll_loss": 2.152804136276245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19531089067459106,
      "rewards/margins": 0.049337826669216156,
      "rewards/rejected": -0.24464870989322662,
      "step": 75
    },
    {
      "epoch": 0.11977935382190702,
      "grad_norm": 1.0609755516052246,
      "learning_rate": 2.3622047244094483e-06,
      "log_odds_chosen": 0.5459141135215759,
      "log_odds_ratio": -0.46591970324516296,
      "logits/chosen": -0.6974417567253113,
      "logits/rejected": 0.22949092090129852,
      "logps/chosen": -1.9926183223724365,
      "logps/rejected": -2.4816536903381348,
      "loss": 2.2244,
      "nll_loss": 2.177783250808716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1992618292570114,
      "rewards/margins": 0.04890353977680206,
      "rewards/rejected": -0.24816536903381348,
      "step": 76
    },
    {
      "epoch": 0.12135539795114263,
      "grad_norm": 0.8557410836219788,
      "learning_rate": 2.393700787401575e-06,
      "log_odds_chosen": 0.5039350986480713,
      "log_odds_ratio": -0.48473480343818665,
      "logits/chosen": -0.7636761665344238,
      "logits/rejected": 0.024586625397205353,
      "logps/chosen": -2.024625778198242,
      "logps/rejected": -2.476198673248291,
      "loss": 2.2801,
      "nll_loss": 2.2316083908081055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20246253907680511,
      "rewards/margins": 0.04515731334686279,
      "rewards/rejected": -0.2476198673248291,
      "step": 77
    },
    {
      "epoch": 0.12293144208037825,
      "grad_norm": 0.6954981684684753,
      "learning_rate": 2.425196850393701e-06,
      "log_odds_chosen": 0.40985774993896484,
      "log_odds_ratio": -0.5190044641494751,
      "logits/chosen": -0.5615072250366211,
      "logits/rejected": -0.14762349426746368,
      "logps/chosen": -2.077012538909912,
      "logps/rejected": -2.4454402923583984,
      "loss": 2.3028,
      "nll_loss": 2.2509350776672363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20770126581192017,
      "rewards/margins": 0.03684278577566147,
      "rewards/rejected": -0.24454402923583984,
      "step": 78
    },
    {
      "epoch": 0.12450748620961387,
      "grad_norm": 0.7768843770027161,
      "learning_rate": 2.4566929133858266e-06,
      "log_odds_chosen": 0.5477795600891113,
      "log_odds_ratio": -0.46383628249168396,
      "logits/chosen": -0.6494770646095276,
      "logits/rejected": -0.007308202795684338,
      "logps/chosen": -2.0030148029327393,
      "logps/rejected": -2.4925520420074463,
      "loss": 2.2402,
      "nll_loss": 2.1937737464904785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20030148327350616,
      "rewards/margins": 0.04895370081067085,
      "rewards/rejected": -0.24925518035888672,
      "step": 79
    },
    {
      "epoch": 0.12608353033884948,
      "grad_norm": 0.734984815120697,
      "learning_rate": 2.488188976377953e-06,
      "log_odds_chosen": 0.5732316970825195,
      "log_odds_ratio": -0.4531470239162445,
      "logits/chosen": -0.726293683052063,
      "logits/rejected": 0.02343374863266945,
      "logps/chosen": -1.9289979934692383,
      "logps/rejected": -2.4362032413482666,
      "loss": 2.168,
      "nll_loss": 2.1227097511291504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19289979338645935,
      "rewards/margins": 0.05072051286697388,
      "rewards/rejected": -0.24362030625343323,
      "step": 80
    },
    {
      "epoch": 0.1276595744680851,
      "grad_norm": 0.6569381356239319,
      "learning_rate": 2.5196850393700787e-06,
      "log_odds_chosen": 0.42326849699020386,
      "log_odds_ratio": -0.5137047171592712,
      "logits/chosen": -0.5939977765083313,
      "logits/rejected": 0.033093176782131195,
      "logps/chosen": -1.9879000186920166,
      "logps/rejected": -2.3671982288360596,
      "loss": 2.2174,
      "nll_loss": 2.166071891784668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19878999888896942,
      "rewards/margins": 0.037929829210042953,
      "rewards/rejected": -0.23671981692314148,
      "step": 81
    },
    {
      "epoch": 0.12923561859732072,
      "grad_norm": 0.7049165964126587,
      "learning_rate": 2.5511811023622045e-06,
      "log_odds_chosen": 0.573046088218689,
      "log_odds_ratio": -0.46415337920188904,
      "logits/chosen": -0.6457680463790894,
      "logits/rejected": 0.08115581423044205,
      "logps/chosen": -1.9417915344238281,
      "logps/rejected": -2.458441734313965,
      "loss": 2.1829,
      "nll_loss": 2.136464834213257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19417917728424072,
      "rewards/margins": 0.05166499689221382,
      "rewards/rejected": -0.24584418535232544,
      "step": 82
    },
    {
      "epoch": 0.13081166272655634,
      "grad_norm": 0.7007700800895691,
      "learning_rate": 2.5826771653543303e-06,
      "log_odds_chosen": 0.3587024211883545,
      "log_odds_ratio": -0.5409030914306641,
      "logits/chosen": -0.5476737022399902,
      "logits/rejected": 0.08743837475776672,
      "logps/chosen": -1.971648931503296,
      "logps/rejected": -2.2917253971099854,
      "loss": 2.2082,
      "nll_loss": 2.154064416885376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1971648931503296,
      "rewards/margins": 0.03200764209032059,
      "rewards/rejected": -0.22917252779006958,
      "step": 83
    },
    {
      "epoch": 0.13238770685579196,
      "grad_norm": 0.8176639676094055,
      "learning_rate": 2.6141732283464566e-06,
      "log_odds_chosen": 0.5798341631889343,
      "log_odds_ratio": -0.4628051221370697,
      "logits/chosen": -0.7357879281044006,
      "logits/rejected": 0.003350723534822464,
      "logps/chosen": -1.933185338973999,
      "logps/rejected": -2.4499199390411377,
      "loss": 2.2023,
      "nll_loss": 2.1560540199279785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19331854581832886,
      "rewards/margins": 0.05167344957590103,
      "rewards/rejected": -0.2449919879436493,
      "step": 84
    },
    {
      "epoch": 0.13396375098502758,
      "grad_norm": 0.7031208872795105,
      "learning_rate": 2.645669291338583e-06,
      "log_odds_chosen": 0.5226806998252869,
      "log_odds_ratio": -0.46645060181617737,
      "logits/chosen": -0.5390961170196533,
      "logits/rejected": 0.1369251161813736,
      "logps/chosen": -1.9962283372879028,
      "logps/rejected": -2.4617295265197754,
      "loss": 2.2245,
      "nll_loss": 2.1779024600982666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19962282478809357,
      "rewards/margins": 0.046550117433071136,
      "rewards/rejected": -0.2461729496717453,
      "step": 85
    },
    {
      "epoch": 0.1355397951142632,
      "grad_norm": 0.7993008494377136,
      "learning_rate": 2.6771653543307086e-06,
      "log_odds_chosen": 0.5069440603256226,
      "log_odds_ratio": -0.476991206407547,
      "logits/chosen": -0.791284441947937,
      "logits/rejected": 0.08737476170063019,
      "logps/chosen": -1.9844413995742798,
      "logps/rejected": -2.4346706867218018,
      "loss": 2.2255,
      "nll_loss": 2.1777687072753906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1984441578388214,
      "rewards/margins": 0.04502289742231369,
      "rewards/rejected": -0.2434670478105545,
      "step": 86
    },
    {
      "epoch": 0.13711583924349882,
      "grad_norm": 0.6878076791763306,
      "learning_rate": 2.7086614173228344e-06,
      "log_odds_chosen": 0.3729919195175171,
      "log_odds_ratio": -0.5269061326980591,
      "logits/chosen": -0.5348572134971619,
      "logits/rejected": -0.09278356283903122,
      "logps/chosen": -1.9824546575546265,
      "logps/rejected": -2.310926914215088,
      "loss": 2.2139,
      "nll_loss": 2.1612019538879395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19824546575546265,
      "rewards/margins": 0.03284721076488495,
      "rewards/rejected": -0.2310926914215088,
      "step": 87
    },
    {
      "epoch": 0.13869188337273444,
      "grad_norm": 0.8332554697990417,
      "learning_rate": 2.7401574803149607e-06,
      "log_odds_chosen": 0.5939523577690125,
      "log_odds_ratio": -0.4592173099517822,
      "logits/chosen": -0.8398531079292297,
      "logits/rejected": 0.05735013633966446,
      "logps/chosen": -1.9121273756027222,
      "logps/rejected": -2.4417524337768555,
      "loss": 2.1817,
      "nll_loss": 2.135769844055176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1912127286195755,
      "rewards/margins": 0.05296248942613602,
      "rewards/rejected": -0.24417521059513092,
      "step": 88
    },
    {
      "epoch": 0.14026792750197006,
      "grad_norm": 0.8030682802200317,
      "learning_rate": 2.7716535433070865e-06,
      "log_odds_chosen": 0.5148739218711853,
      "log_odds_ratio": -0.4697887897491455,
      "logits/chosen": -0.6359347701072693,
      "logits/rejected": 0.13956201076507568,
      "logps/chosen": -2.0257599353790283,
      "logps/rejected": -2.486156463623047,
      "loss": 2.2714,
      "nll_loss": 2.224398374557495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20257601141929626,
      "rewards/margins": 0.0460396483540535,
      "rewards/rejected": -0.24861565232276917,
      "step": 89
    },
    {
      "epoch": 0.14184397163120568,
      "grad_norm": 0.7527587413787842,
      "learning_rate": 2.8031496062992123e-06,
      "log_odds_chosen": 0.43857112526893616,
      "log_odds_ratio": -0.5034822225570679,
      "logits/chosen": -0.5429673790931702,
      "logits/rejected": 0.09310950338840485,
      "logps/chosen": -2.072077989578247,
      "logps/rejected": -2.4671554565429688,
      "loss": 2.2851,
      "nll_loss": 2.2347841262817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2072077989578247,
      "rewards/margins": 0.03950772434473038,
      "rewards/rejected": -0.24671553075313568,
      "step": 90
    },
    {
      "epoch": 0.1434200157604413,
      "grad_norm": 0.6531558036804199,
      "learning_rate": 2.834645669291338e-06,
      "log_odds_chosen": 0.47114643454551697,
      "log_odds_ratio": -0.4919435381889343,
      "logits/chosen": -0.511391818523407,
      "logits/rejected": -0.023931853473186493,
      "logps/chosen": -1.860572338104248,
      "logps/rejected": -2.27347731590271,
      "loss": 2.0935,
      "nll_loss": 2.0442581176757812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1860572248697281,
      "rewards/margins": 0.0412905178964138,
      "rewards/rejected": -0.22734776139259338,
      "step": 91
    },
    {
      "epoch": 0.1449960598896769,
      "grad_norm": 0.7162215709686279,
      "learning_rate": 2.8661417322834644e-06,
      "log_odds_chosen": 0.5217415690422058,
      "log_odds_ratio": -0.47228407859802246,
      "logits/chosen": -0.6261119246482849,
      "logits/rejected": 0.10750691592693329,
      "logps/chosen": -1.8966529369354248,
      "logps/rejected": -2.357200860977173,
      "loss": 2.1443,
      "nll_loss": 2.0970218181610107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.189665287733078,
      "rewards/margins": 0.04605480283498764,
      "rewards/rejected": -0.23572009801864624,
      "step": 92
    },
    {
      "epoch": 0.14657210401891252,
      "grad_norm": 0.7588204145431519,
      "learning_rate": 2.8976377952755906e-06,
      "log_odds_chosen": 0.46790874004364014,
      "log_odds_ratio": -0.5142867565155029,
      "logits/chosen": -0.572394073009491,
      "logits/rejected": 0.08447833359241486,
      "logps/chosen": -1.9991455078125,
      "logps/rejected": -2.422028064727783,
      "loss": 2.2352,
      "nll_loss": 2.183756113052368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19991454482078552,
      "rewards/margins": 0.04228827357292175,
      "rewards/rejected": -0.24220280349254608,
      "step": 93
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 0.698870062828064,
      "learning_rate": 2.9291338582677165e-06,
      "log_odds_chosen": 0.4843592345714569,
      "log_odds_ratio": -0.4909193813800812,
      "logits/chosen": -0.5559593439102173,
      "logits/rejected": -0.03297539800405502,
      "logps/chosen": -1.9764482975006104,
      "logps/rejected": -2.404412269592285,
      "loss": 2.2021,
      "nll_loss": 2.1530561447143555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19764482975006104,
      "rewards/margins": 0.042796410620212555,
      "rewards/rejected": -0.240441232919693,
      "step": 94
    },
    {
      "epoch": 0.14972419227738376,
      "grad_norm": 0.7901385426521301,
      "learning_rate": 2.9606299212598423e-06,
      "log_odds_chosen": 0.568733811378479,
      "log_odds_ratio": -0.459706574678421,
      "logits/chosen": -0.6126303672790527,
      "logits/rejected": 0.1119111031293869,
      "logps/chosen": -1.9837087392807007,
      "logps/rejected": -2.4926552772521973,
      "loss": 2.2244,
      "nll_loss": 2.17838978767395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19837087392807007,
      "rewards/margins": 0.050894659012556076,
      "rewards/rejected": -0.24926555156707764,
      "step": 95
    },
    {
      "epoch": 0.15130023640661938,
      "grad_norm": 0.7014930844306946,
      "learning_rate": 2.9921259842519685e-06,
      "log_odds_chosen": 0.5473185777664185,
      "log_odds_ratio": -0.46086204051971436,
      "logits/chosen": -0.6081118583679199,
      "logits/rejected": -0.03256102278828621,
      "logps/chosen": -1.9777699708938599,
      "logps/rejected": -2.4649531841278076,
      "loss": 2.1907,
      "nll_loss": 2.1445720195770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19777700304985046,
      "rewards/margins": 0.048718295991420746,
      "rewards/rejected": -0.246495321393013,
      "step": 96
    },
    {
      "epoch": 0.152876280535855,
      "grad_norm": 0.6733537316322327,
      "learning_rate": 3.0236220472440943e-06,
      "log_odds_chosen": 0.36231333017349243,
      "log_odds_ratio": -0.529702365398407,
      "logits/chosen": -0.6357975006103516,
      "logits/rejected": 0.13661673665046692,
      "logps/chosen": -1.9484719038009644,
      "logps/rejected": -2.264392137527466,
      "loss": 2.1665,
      "nll_loss": 2.113529682159424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19484718143939972,
      "rewards/margins": 0.03159204497933388,
      "rewards/rejected": -0.2264392226934433,
      "step": 97
    },
    {
      "epoch": 0.15445232466509062,
      "grad_norm": 0.6935497522354126,
      "learning_rate": 3.05511811023622e-06,
      "log_odds_chosen": 0.4252420663833618,
      "log_odds_ratio": -0.5098926424980164,
      "logits/chosen": -0.4698202908039093,
      "logits/rejected": -0.029691245406866074,
      "logps/chosen": -2.0581109523773193,
      "logps/rejected": -2.439418077468872,
      "loss": 2.2929,
      "nll_loss": 2.24191951751709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20581108331680298,
      "rewards/margins": 0.038130730390548706,
      "rewards/rejected": -0.24394181370735168,
      "step": 98
    },
    {
      "epoch": 0.15602836879432624,
      "grad_norm": 0.753554105758667,
      "learning_rate": 3.086614173228346e-06,
      "log_odds_chosen": 0.2644384205341339,
      "log_odds_ratio": -0.5719925165176392,
      "logits/chosen": -0.7020218372344971,
      "logits/rejected": -0.07810735702514648,
      "logps/chosen": -1.9831633567810059,
      "logps/rejected": -2.2148847579956055,
      "loss": 2.2295,
      "nll_loss": 2.172307252883911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19831633567810059,
      "rewards/margins": 0.023172136396169662,
      "rewards/rejected": -0.22148849070072174,
      "step": 99
    },
    {
      "epoch": 0.15760441292356187,
      "grad_norm": 0.6946485042572021,
      "learning_rate": 3.1181102362204722e-06,
      "log_odds_chosen": 0.3836790919303894,
      "log_odds_ratio": -0.5257760286331177,
      "logits/chosen": -0.6223403215408325,
      "logits/rejected": -0.06376251578330994,
      "logps/chosen": -1.8918278217315674,
      "logps/rejected": -2.230099678039551,
      "loss": 2.1396,
      "nll_loss": 2.0870351791381836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1891828179359436,
      "rewards/margins": 0.03382713720202446,
      "rewards/rejected": -0.22300994396209717,
      "step": 100
    },
    {
      "epoch": 0.15918045705279749,
      "grad_norm": 0.6816413402557373,
      "learning_rate": 3.1496062992125985e-06,
      "log_odds_chosen": 0.44349855184555054,
      "log_odds_ratio": -0.5018309354782104,
      "logits/chosen": -0.5561724901199341,
      "logits/rejected": 0.10826245695352554,
      "logps/chosen": -1.8893253803253174,
      "logps/rejected": -2.280471086502075,
      "loss": 2.1421,
      "nll_loss": 2.0918774604797363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18893250823020935,
      "rewards/margins": 0.03911459445953369,
      "rewards/rejected": -0.22804711759090424,
      "step": 101
    },
    {
      "epoch": 0.1607565011820331,
      "grad_norm": 0.6148979067802429,
      "learning_rate": 3.1811023622047243e-06,
      "log_odds_chosen": 0.4480132758617401,
      "log_odds_ratio": -0.508931577205658,
      "logits/chosen": -0.5319602489471436,
      "logits/rejected": -0.08936317265033722,
      "logps/chosen": -1.8357093334197998,
      "logps/rejected": -2.2293267250061035,
      "loss": 2.0816,
      "nll_loss": 2.0307018756866455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18357093632221222,
      "rewards/margins": 0.03936174511909485,
      "rewards/rejected": -0.22293266654014587,
      "step": 102
    },
    {
      "epoch": 0.16233254531126873,
      "grad_norm": 0.6112706661224365,
      "learning_rate": 3.2125984251968505e-06,
      "log_odds_chosen": 0.5046467185020447,
      "log_odds_ratio": -0.4848136007785797,
      "logits/chosen": -0.4200552999973297,
      "logits/rejected": 0.25700271129608154,
      "logps/chosen": -1.818860411643982,
      "logps/rejected": -2.2571394443511963,
      "loss": 2.0709,
      "nll_loss": 2.022395372390747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18188604712486267,
      "rewards/margins": 0.04382789134979248,
      "rewards/rejected": -0.22571393847465515,
      "step": 103
    },
    {
      "epoch": 0.16390858944050432,
      "grad_norm": 0.704250156879425,
      "learning_rate": 3.2440944881889763e-06,
      "log_odds_chosen": 0.46751153469085693,
      "log_odds_ratio": -0.49732449650764465,
      "logits/chosen": -0.38590556383132935,
      "logits/rejected": 0.3286210894584656,
      "logps/chosen": -1.9726375341415405,
      "logps/rejected": -2.3892126083374023,
      "loss": 2.2131,
      "nll_loss": 2.163343667984009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19726374745368958,
      "rewards/margins": 0.04165750741958618,
      "rewards/rejected": -0.23892125487327576,
      "step": 104
    },
    {
      "epoch": 0.16548463356973994,
      "grad_norm": 0.5806333422660828,
      "learning_rate": 3.275590551181102e-06,
      "log_odds_chosen": 0.5418988466262817,
      "log_odds_ratio": -0.46480318903923035,
      "logits/chosen": -0.5418411493301392,
      "logits/rejected": 0.17040874063968658,
      "logps/chosen": -1.8416911363601685,
      "logps/rejected": -2.3162972927093506,
      "loss": 2.0965,
      "nll_loss": 2.0500617027282715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18416911363601685,
      "rewards/margins": 0.047460608184337616,
      "rewards/rejected": -0.23162971436977386,
      "step": 105
    },
    {
      "epoch": 0.16706067769897556,
      "grad_norm": 0.6116374135017395,
      "learning_rate": 3.307086614173228e-06,
      "log_odds_chosen": 0.47532159090042114,
      "log_odds_ratio": -0.4957353472709656,
      "logits/chosen": -0.5306176543235779,
      "logits/rejected": 0.20767346024513245,
      "logps/chosen": -1.8645740747451782,
      "logps/rejected": -2.2723007202148438,
      "loss": 2.1024,
      "nll_loss": 2.0528042316436768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18645742535591125,
      "rewards/margins": 0.04077265411615372,
      "rewards/rejected": -0.22723007202148438,
      "step": 106
    },
    {
      "epoch": 0.16863672182821118,
      "grad_norm": 0.6184962391853333,
      "learning_rate": 3.3385826771653542e-06,
      "log_odds_chosen": 0.4586809277534485,
      "log_odds_ratio": -0.5017204284667969,
      "logits/chosen": -0.4147133529186249,
      "logits/rejected": 0.059773970395326614,
      "logps/chosen": -1.9274760484695435,
      "logps/rejected": -2.333932399749756,
      "loss": 2.1614,
      "nll_loss": 2.1112310886383057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19274760782718658,
      "rewards/margins": 0.04064565151929855,
      "rewards/rejected": -0.23339325189590454,
      "step": 107
    },
    {
      "epoch": 0.1702127659574468,
      "grad_norm": 0.5825785398483276,
      "learning_rate": 3.37007874015748e-06,
      "log_odds_chosen": 0.35450294613838196,
      "log_odds_ratio": -0.5409185886383057,
      "logits/chosen": -0.39133238792419434,
      "logits/rejected": 0.04965784400701523,
      "logps/chosen": -1.9083962440490723,
      "logps/rejected": -2.217801809310913,
      "loss": 2.1444,
      "nll_loss": 2.090298652648926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19083963334560394,
      "rewards/margins": 0.030940547585487366,
      "rewards/rejected": -0.22178016602993011,
      "step": 108
    },
    {
      "epoch": 0.17178881008668243,
      "grad_norm": 0.6102772951126099,
      "learning_rate": 3.4015748031496063e-06,
      "log_odds_chosen": 0.48623126745224,
      "log_odds_ratio": -0.4813230633735657,
      "logits/chosen": -0.31981658935546875,
      "logits/rejected": 0.002862483263015747,
      "logps/chosen": -1.9655230045318604,
      "logps/rejected": -2.394603729248047,
      "loss": 2.1966,
      "nll_loss": 2.1484334468841553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1965523064136505,
      "rewards/margins": 0.042908087372779846,
      "rewards/rejected": -0.23946037888526917,
      "step": 109
    },
    {
      "epoch": 0.17336485421591805,
      "grad_norm": 0.595453679561615,
      "learning_rate": 3.433070866141732e-06,
      "log_odds_chosen": 0.5263383388519287,
      "log_odds_ratio": -0.47105205059051514,
      "logits/chosen": -0.42303267121315,
      "logits/rejected": 0.10894276201725006,
      "logps/chosen": -1.8528122901916504,
      "logps/rejected": -2.307560920715332,
      "loss": 2.0631,
      "nll_loss": 2.016003131866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18528123199939728,
      "rewards/margins": 0.045474857091903687,
      "rewards/rejected": -0.23075608909130096,
      "step": 110
    },
    {
      "epoch": 0.17494089834515367,
      "grad_norm": 0.6606065630912781,
      "learning_rate": 3.4645669291338583e-06,
      "log_odds_chosen": 0.5373308062553406,
      "log_odds_ratio": -0.47022971510887146,
      "logits/chosen": -0.39981910586357117,
      "logits/rejected": 0.004297524690628052,
      "logps/chosen": -1.961772084236145,
      "logps/rejected": -2.4356696605682373,
      "loss": 2.1978,
      "nll_loss": 2.150740385055542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19617722928524017,
      "rewards/margins": 0.047389768064022064,
      "rewards/rejected": -0.24356698989868164,
      "step": 111
    },
    {
      "epoch": 0.1765169424743893,
      "grad_norm": 0.603425920009613,
      "learning_rate": 3.496062992125984e-06,
      "log_odds_chosen": 0.3936331868171692,
      "log_odds_ratio": -0.5209735035896301,
      "logits/chosen": -0.33380576968193054,
      "logits/rejected": 0.18296881020069122,
      "logps/chosen": -1.9979010820388794,
      "logps/rejected": -2.344928026199341,
      "loss": 2.2248,
      "nll_loss": 2.1727278232574463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1997901350259781,
      "rewards/margins": 0.034702688455581665,
      "rewards/rejected": -0.23449280858039856,
      "step": 112
    },
    {
      "epoch": 0.1780929866036249,
      "grad_norm": 0.5860929489135742,
      "learning_rate": 3.52755905511811e-06,
      "log_odds_chosen": 0.4042174816131592,
      "log_odds_ratio": -0.5187764763832092,
      "logits/chosen": -0.4759305715560913,
      "logits/rejected": 0.05041307210922241,
      "logps/chosen": -1.8414995670318604,
      "logps/rejected": -2.19472599029541,
      "loss": 2.0856,
      "nll_loss": 2.0337438583374023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18414995074272156,
      "rewards/margins": 0.035322654992341995,
      "rewards/rejected": -0.21947260200977325,
      "step": 113
    },
    {
      "epoch": 0.17966903073286053,
      "grad_norm": 0.5595982074737549,
      "learning_rate": 3.559055118110236e-06,
      "log_odds_chosen": 0.32051122188568115,
      "log_odds_ratio": -0.562548041343689,
      "logits/chosen": -0.22491343319416046,
      "logits/rejected": 0.09666822850704193,
      "logps/chosen": -1.968389630317688,
      "logps/rejected": -2.2551064491271973,
      "loss": 2.2079,
      "nll_loss": 2.151672124862671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19683893024921417,
      "rewards/margins": 0.028671709820628166,
      "rewards/rejected": -0.22551067173480988,
      "step": 114
    },
    {
      "epoch": 0.18124507486209615,
      "grad_norm": 0.5502125024795532,
      "learning_rate": 3.590551181102362e-06,
      "log_odds_chosen": 0.49229851365089417,
      "log_odds_ratio": -0.48761364817619324,
      "logits/chosen": -0.34304749965667725,
      "logits/rejected": -0.03732027858495712,
      "logps/chosen": -1.8795965909957886,
      "logps/rejected": -2.3121144771575928,
      "loss": 2.1061,
      "nll_loss": 2.0573036670684814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18795965611934662,
      "rewards/margins": 0.043251797556877136,
      "rewards/rejected": -0.23121145367622375,
      "step": 115
    },
    {
      "epoch": 0.18282111899133174,
      "grad_norm": 0.5441986918449402,
      "learning_rate": 3.622047244094488e-06,
      "log_odds_chosen": 0.4177536368370056,
      "log_odds_ratio": -0.5228228569030762,
      "logits/chosen": -0.32923153042793274,
      "logits/rejected": 0.12902945280075073,
      "logps/chosen": -1.9323790073394775,
      "logps/rejected": -2.3019943237304688,
      "loss": 2.1496,
      "nll_loss": 2.097325563430786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19323790073394775,
      "rewards/margins": 0.03696154057979584,
      "rewards/rejected": -0.2301994264125824,
      "step": 116
    },
    {
      "epoch": 0.18439716312056736,
      "grad_norm": 0.5771306753158569,
      "learning_rate": 3.653543307086614e-06,
      "log_odds_chosen": 0.5892273783683777,
      "log_odds_ratio": -0.45790567994117737,
      "logits/chosen": -0.42785924673080444,
      "logits/rejected": -0.1212269514799118,
      "logps/chosen": -1.8676490783691406,
      "logps/rejected": -2.3864071369171143,
      "loss": 2.106,
      "nll_loss": 2.060208559036255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1867649108171463,
      "rewards/margins": 0.05187581479549408,
      "rewards/rejected": -0.23864072561264038,
      "step": 117
    },
    {
      "epoch": 0.18597320724980299,
      "grad_norm": 0.5304055213928223,
      "learning_rate": 3.68503937007874e-06,
      "log_odds_chosen": 0.3501071035861969,
      "log_odds_ratio": -0.5444656610488892,
      "logits/chosen": -0.19212201237678528,
      "logits/rejected": 0.03166138380765915,
      "logps/chosen": -1.8767951726913452,
      "logps/rejected": -2.1823959350585938,
      "loss": 2.1161,
      "nll_loss": 2.0616323947906494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18767952919006348,
      "rewards/margins": 0.030560053884983063,
      "rewards/rejected": -0.21823956072330475,
      "step": 118
    },
    {
      "epoch": 0.1875492513790386,
      "grad_norm": 0.5213944315910339,
      "learning_rate": 3.716535433070866e-06,
      "log_odds_chosen": 0.3437122404575348,
      "log_odds_ratio": -0.5484852194786072,
      "logits/chosen": -0.2524073123931885,
      "logits/rejected": 0.11574040353298187,
      "logps/chosen": -1.8521205186843872,
      "logps/rejected": -2.152954578399658,
      "loss": 2.1082,
      "nll_loss": 2.0533039569854736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18521204590797424,
      "rewards/margins": 0.03008340485394001,
      "rewards/rejected": -0.2152954488992691,
      "step": 119
    },
    {
      "epoch": 0.18912529550827423,
      "grad_norm": 0.4969903528690338,
      "learning_rate": 3.748031496062992e-06,
      "log_odds_chosen": 0.6459031701087952,
      "log_odds_ratio": -0.4308077096939087,
      "logits/chosen": -0.2889327108860016,
      "logits/rejected": 0.09839168190956116,
      "logps/chosen": -1.7536612749099731,
      "logps/rejected": -2.3134381771087646,
      "loss": 1.9798,
      "nll_loss": 1.9366917610168457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1753661185503006,
      "rewards/margins": 0.05597769469022751,
      "rewards/rejected": -0.2313438206911087,
      "step": 120
    },
    {
      "epoch": 0.19070133963750985,
      "grad_norm": 0.47284096479415894,
      "learning_rate": 3.779527559055118e-06,
      "log_odds_chosen": 0.6493716239929199,
      "log_odds_ratio": -0.4239138066768646,
      "logits/chosen": -0.33733659982681274,
      "logits/rejected": -0.07449808716773987,
      "logps/chosen": -1.8633861541748047,
      "logps/rejected": -2.43552303314209,
      "loss": 2.0796,
      "nll_loss": 2.0372438430786133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1863386034965515,
      "rewards/margins": 0.0572136826813221,
      "rewards/rejected": -0.2435522973537445,
      "step": 121
    },
    {
      "epoch": 0.19227738376674547,
      "grad_norm": 0.5358994603157043,
      "learning_rate": 3.8110236220472436e-06,
      "log_odds_chosen": 0.552227795124054,
      "log_odds_ratio": -0.4722144603729248,
      "logits/chosen": -0.4011126756668091,
      "logits/rejected": 0.0521632581949234,
      "logps/chosen": -1.7913762331008911,
      "logps/rejected": -2.2755613327026367,
      "loss": 2.0092,
      "nll_loss": 1.961932897567749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17913760244846344,
      "rewards/margins": 0.04841852933168411,
      "rewards/rejected": -0.22755613923072815,
      "step": 122
    },
    {
      "epoch": 0.1938534278959811,
      "grad_norm": 0.5210543870925903,
      "learning_rate": 3.8425196850393695e-06,
      "log_odds_chosen": 0.42144662141799927,
      "log_odds_ratio": -0.5061027407646179,
      "logits/chosen": -0.13411694765090942,
      "logits/rejected": 0.00012157857418060303,
      "logps/chosen": -1.8857041597366333,
      "logps/rejected": -2.2534890174865723,
      "loss": 2.1206,
      "nll_loss": 2.0700302124023438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18857041001319885,
      "rewards/margins": 0.036778487265110016,
      "rewards/rejected": -0.22534890472888947,
      "step": 123
    },
    {
      "epoch": 0.1954294720252167,
      "grad_norm": 0.4755237400531769,
      "learning_rate": 3.874015748031496e-06,
      "log_odds_chosen": 0.4865044355392456,
      "log_odds_ratio": -0.49245476722717285,
      "logits/chosen": -0.34655338525772095,
      "logits/rejected": 0.11143852770328522,
      "logps/chosen": -1.7666759490966797,
      "logps/rejected": -2.1775081157684326,
      "loss": 1.9857,
      "nll_loss": 1.9364441633224487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17666758596897125,
      "rewards/margins": 0.04108322411775589,
      "rewards/rejected": -0.21775081753730774,
      "step": 124
    },
    {
      "epoch": 0.19700551615445233,
      "grad_norm": 0.4905242323875427,
      "learning_rate": 3.905511811023622e-06,
      "log_odds_chosen": 0.46543189883232117,
      "log_odds_ratio": -0.49489825963974,
      "logits/chosen": -0.2513599693775177,
      "logits/rejected": -0.10763716697692871,
      "logps/chosen": -1.8007551431655884,
      "logps/rejected": -2.2026453018188477,
      "loss": 2.0412,
      "nll_loss": 1.9916858673095703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1800755262374878,
      "rewards/margins": 0.04018900915980339,
      "rewards/rejected": -0.2202645242214203,
      "step": 125
    },
    {
      "epoch": 0.19858156028368795,
      "grad_norm": 0.5292844772338867,
      "learning_rate": 3.937007874015748e-06,
      "log_odds_chosen": 0.6494875550270081,
      "log_odds_ratio": -0.4264853000640869,
      "logits/chosen": -0.21110114455223083,
      "logits/rejected": 0.15301132202148438,
      "logps/chosen": -1.8184471130371094,
      "logps/rejected": -2.390329599380493,
      "loss": 2.038,
      "nll_loss": 1.995354413986206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18184472620487213,
      "rewards/margins": 0.0571882426738739,
      "rewards/rejected": -0.23903295397758484,
      "step": 126
    },
    {
      "epoch": 0.20015760441292357,
      "grad_norm": 0.5565134882926941,
      "learning_rate": 3.9685039370078736e-06,
      "log_odds_chosen": 0.5321060419082642,
      "log_odds_ratio": -0.47239407896995544,
      "logits/chosen": -0.21071690320968628,
      "logits/rejected": 0.12574365735054016,
      "logps/chosen": -1.849616289138794,
      "logps/rejected": -2.318277597427368,
      "loss": 2.1051,
      "nll_loss": 2.0578956604003906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18496163189411163,
      "rewards/margins": 0.04686610400676727,
      "rewards/rejected": -0.2318277359008789,
      "step": 127
    },
    {
      "epoch": 0.2017336485421592,
      "grad_norm": 0.5013044476509094,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.3478800356388092,
      "log_odds_ratio": -0.5399872064590454,
      "logits/chosen": -0.08074170351028442,
      "logits/rejected": 0.3015192151069641,
      "logps/chosen": -1.9230244159698486,
      "logps/rejected": -2.2274985313415527,
      "loss": 2.1378,
      "nll_loss": 2.083803653717041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19230243563652039,
      "rewards/margins": 0.030447421595454216,
      "rewards/rejected": -0.22274987399578094,
      "step": 128
    },
    {
      "epoch": 0.2033096926713948,
      "grad_norm": 0.4872080385684967,
      "learning_rate": 3.999992445477635e-06,
      "log_odds_chosen": 0.4767901301383972,
      "log_odds_ratio": -0.49153631925582886,
      "logits/chosen": -0.129595085978508,
      "logits/rejected": 0.0004668300971388817,
      "logps/chosen": -1.9134724140167236,
      "logps/rejected": -2.3344011306762695,
      "loss": 2.1148,
      "nll_loss": 2.065617799758911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19134724140167236,
      "rewards/margins": 0.04209288954734802,
      "rewards/rejected": -0.2334401160478592,
      "step": 129
    },
    {
      "epoch": 0.2048857368006304,
      "grad_norm": 0.5020763278007507,
      "learning_rate": 3.999969781967615e-06,
      "log_odds_chosen": 0.30211132764816284,
      "log_odds_ratio": -0.5571832060813904,
      "logits/chosen": -0.12993694841861725,
      "logits/rejected": 0.16511544585227966,
      "logps/chosen": -2.020594358444214,
      "logps/rejected": -2.288883686065674,
      "loss": 2.2185,
      "nll_loss": 2.162761688232422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20205941796302795,
      "rewards/margins": 0.026828955858945847,
      "rewards/rejected": -0.2288883775472641,
      "step": 130
    },
    {
      "epoch": 0.20646178092986603,
      "grad_norm": 0.45242053270339966,
      "learning_rate": 3.99993200964115e-06,
      "log_odds_chosen": 0.3914204239845276,
      "log_odds_ratio": -0.5182069540023804,
      "logits/chosen": -0.19754824042320251,
      "logits/rejected": -0.0378432460129261,
      "logps/chosen": -1.8395533561706543,
      "logps/rejected": -2.1777098178863525,
      "loss": 2.0554,
      "nll_loss": 2.00355863571167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1839553415775299,
      "rewards/margins": 0.03381563723087311,
      "rewards/rejected": -0.21777096390724182,
      "step": 131
    },
    {
      "epoch": 0.20803782505910165,
      "grad_norm": 0.5595487952232361,
      "learning_rate": 3.99987912878359e-06,
      "log_odds_chosen": 0.18296487629413605,
      "log_odds_ratio": -0.6110657453536987,
      "logits/chosen": -0.14159467816352844,
      "logits/rejected": 0.2608286738395691,
      "logps/chosen": -1.8113925457000732,
      "logps/rejected": -1.9622738361358643,
      "loss": 2.0762,
      "nll_loss": 2.0150678157806396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1811392605304718,
      "rewards/margins": 0.015088127925992012,
      "rewards/rejected": -0.19622740149497986,
      "step": 132
    },
    {
      "epoch": 0.20961386918833727,
      "grad_norm": 0.4690943956375122,
      "learning_rate": 3.999811139794429e-06,
      "log_odds_chosen": 0.41473308205604553,
      "log_odds_ratio": -0.5109004378318787,
      "logits/chosen": -0.10903345048427582,
      "logits/rejected": 0.08273053169250488,
      "logps/chosen": -1.8365156650543213,
      "logps/rejected": -2.195512056350708,
      "loss": 2.0443,
      "nll_loss": 1.9931950569152832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18365156650543213,
      "rewards/margins": 0.03589965030550957,
      "rewards/rejected": -0.2195512056350708,
      "step": 133
    },
    {
      "epoch": 0.2111899133175729,
      "grad_norm": 0.49801507592201233,
      "learning_rate": 3.999728043187288e-06,
      "log_odds_chosen": 0.4041176736354828,
      "log_odds_ratio": -0.5218353867530823,
      "logits/chosen": -0.08679277449846268,
      "logits/rejected": 0.06324957311153412,
      "logps/chosen": -1.8879791498184204,
      "logps/rejected": -2.2436025142669678,
      "loss": 2.1203,
      "nll_loss": 2.068152904510498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18879792094230652,
      "rewards/margins": 0.03556232899427414,
      "rewards/rejected": -0.22436025738716125,
      "step": 134
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 0.4822520315647125,
      "learning_rate": 3.999629839589922e-06,
      "log_odds_chosen": 0.25128376483917236,
      "log_odds_ratio": -0.5789471864700317,
      "logits/chosen": -0.040255192667245865,
      "logits/rejected": -0.0998617559671402,
      "logps/chosen": -1.9691245555877686,
      "logps/rejected": -2.189525842666626,
      "loss": 2.1988,
      "nll_loss": 2.140864372253418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1969124674797058,
      "rewards/margins": 0.022040119394659996,
      "rewards/rejected": -0.21895259618759155,
      "step": 135
    },
    {
      "epoch": 0.21434200157604413,
      "grad_norm": 0.46929681301116943,
      "learning_rate": 3.999516529744215e-06,
      "log_odds_chosen": 0.356317937374115,
      "log_odds_ratio": -0.5354443192481995,
      "logits/chosen": 0.03701853007078171,
      "logits/rejected": -0.025361815467476845,
      "logps/chosen": -1.8457211256027222,
      "logps/rejected": -2.154231548309326,
      "loss": 2.0676,
      "nll_loss": 2.0140366554260254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18457213044166565,
      "rewards/margins": 0.03085101954638958,
      "rewards/rejected": -0.215423122048378,
      "step": 136
    },
    {
      "epoch": 0.21591804570527975,
      "grad_norm": 0.45633479952812195,
      "learning_rate": 3.999388114506166e-06,
      "log_odds_chosen": 0.44111165404319763,
      "log_odds_ratio": -0.5032743811607361,
      "logits/chosen": -0.10049010813236237,
      "logits/rejected": -0.09671763330698013,
      "logps/chosen": -1.7441117763519287,
      "logps/rejected": -2.1240317821502686,
      "loss": 1.958,
      "nll_loss": 1.9076509475708008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17441117763519287,
      "rewards/margins": 0.03799201920628548,
      "rewards/rejected": -0.21240317821502686,
      "step": 137
    },
    {
      "epoch": 0.21749408983451538,
      "grad_norm": 0.44847702980041504,
      "learning_rate": 3.999244594845892e-06,
      "log_odds_chosen": 0.30588337779045105,
      "log_odds_ratio": -0.5579502582550049,
      "logits/chosen": -0.06541359424591064,
      "logits/rejected": -0.00460642296820879,
      "logps/chosen": -1.7904387712478638,
      "logps/rejected": -2.0546517372131348,
      "loss": 2.0168,
      "nll_loss": 1.9609698057174683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17904390394687653,
      "rewards/margins": 0.026421282440423965,
      "rewards/rejected": -0.2054651826620102,
      "step": 138
    },
    {
      "epoch": 0.219070133963751,
      "grad_norm": 0.45796382427215576,
      "learning_rate": 3.999085971847616e-06,
      "log_odds_chosen": 0.26914283633232117,
      "log_odds_ratio": -0.5760324597358704,
      "logits/chosen": -0.05198541656136513,
      "logits/rejected": -0.22539588809013367,
      "logps/chosen": -1.850884199142456,
      "logps/rejected": -2.085418462753296,
      "loss": 2.0685,
      "nll_loss": 2.010895252227783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18508842587471008,
      "rewards/margins": 0.02345343679189682,
      "rewards/rejected": -0.2085418552160263,
      "step": 139
    },
    {
      "epoch": 0.22064617809298662,
      "grad_norm": 0.44731172919273376,
      "learning_rate": 3.998912246709658e-06,
      "log_odds_chosen": 0.41634848713874817,
      "log_odds_ratio": -0.5201424360275269,
      "logits/chosen": -0.1276709884405136,
      "logits/rejected": -0.13816407322883606,
      "logps/chosen": -1.74984610080719,
      "logps/rejected": -2.1083877086639404,
      "loss": 1.9674,
      "nll_loss": 1.915358304977417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1749846190214157,
      "rewards/margins": 0.03585416078567505,
      "rewards/rejected": -0.21083875000476837,
      "step": 140
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 0.47976112365722656,
      "learning_rate": 3.9987234207444295e-06,
      "log_odds_chosen": 0.36653023958206177,
      "log_odds_ratio": -0.5325444340705872,
      "logits/chosen": -0.09957035630941391,
      "logits/rejected": 0.037235602736473083,
      "logps/chosen": -1.85134756565094,
      "logps/rejected": -2.1700351238250732,
      "loss": 2.0714,
      "nll_loss": 2.018110752105713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18513476848602295,
      "rewards/margins": 0.031868740916252136,
      "rewards/rejected": -0.21700350940227509,
      "step": 141
    },
    {
      "epoch": 0.22379826635145783,
      "grad_norm": 0.47072869539260864,
      "learning_rate": 3.998519495378419e-06,
      "log_odds_chosen": 0.38448622822761536,
      "log_odds_ratio": -0.524972677230835,
      "logits/chosen": 0.012963086366653442,
      "logits/rejected": 0.13648656010627747,
      "logps/chosen": -1.9236266613006592,
      "logps/rejected": -2.262057304382324,
      "loss": 2.1162,
      "nll_loss": 2.063678503036499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19236266613006592,
      "rewards/margins": 0.03384307771921158,
      "rewards/rejected": -0.2262057363986969,
      "step": 142
    },
    {
      "epoch": 0.22537431048069345,
      "grad_norm": 0.4084656536579132,
      "learning_rate": 3.998300472152187e-06,
      "log_odds_chosen": 0.3563390374183655,
      "log_odds_ratio": -0.539592981338501,
      "logits/chosen": -0.11657491326332092,
      "logits/rejected": -0.21950428187847137,
      "logps/chosen": -1.7084014415740967,
      "logps/rejected": -2.0116360187530518,
      "loss": 1.9309,
      "nll_loss": 1.8769128322601318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17084015905857086,
      "rewards/margins": 0.030323447659611702,
      "rewards/rejected": -0.20116360485553741,
      "step": 143
    },
    {
      "epoch": 0.22695035460992907,
      "grad_norm": 0.44106900691986084,
      "learning_rate": 3.998066352720347e-06,
      "log_odds_chosen": 0.45243191719055176,
      "log_odds_ratio": -0.49694910645484924,
      "logits/chosen": 0.07859814167022705,
      "logits/rejected": 0.016231544315814972,
      "logps/chosen": -1.8335413932800293,
      "logps/rejected": -2.2273075580596924,
      "loss": 2.0408,
      "nll_loss": 1.9911000728607178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18335412442684174,
      "rewards/margins": 0.0393766388297081,
      "rewards/rejected": -0.22273077070713043,
      "step": 144
    },
    {
      "epoch": 0.2285263987391647,
      "grad_norm": 0.41912582516670227,
      "learning_rate": 3.997817138851562e-06,
      "log_odds_chosen": 0.33265724778175354,
      "log_odds_ratio": -0.5518072843551636,
      "logits/chosen": 0.032988160848617554,
      "logits/rejected": 0.03446170315146446,
      "logps/chosen": -1.9223456382751465,
      "logps/rejected": -2.215817928314209,
      "loss": 2.1297,
      "nll_loss": 2.0745091438293457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1922345608472824,
      "rewards/margins": 0.029347218573093414,
      "rewards/rejected": -0.22158177196979523,
      "step": 145
    },
    {
      "epoch": 0.23010244286840031,
      "grad_norm": 0.40147900581359863,
      "learning_rate": 3.997552832428522e-06,
      "log_odds_chosen": 0.42633017897605896,
      "log_odds_ratio": -0.5056131482124329,
      "logits/chosen": -0.0994889885187149,
      "logits/rejected": 0.03103024512529373,
      "logps/chosen": -1.7114800214767456,
      "logps/rejected": -2.072697639465332,
      "loss": 1.9484,
      "nll_loss": 1.8978841304779053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17114798724651337,
      "rewards/margins": 0.03612176328897476,
      "rewards/rejected": -0.20726974308490753,
      "step": 146
    },
    {
      "epoch": 0.23167848699763594,
      "grad_norm": 0.39530226588249207,
      "learning_rate": 3.9972734354479366e-06,
      "log_odds_chosen": 0.3678266108036041,
      "log_odds_ratio": -0.5362446308135986,
      "logits/chosen": 0.022561486810445786,
      "logits/rejected": -0.250899076461792,
      "logps/chosen": -1.7041471004486084,
      "logps/rejected": -2.0142199993133545,
      "loss": 1.9268,
      "nll_loss": 1.8731917142868042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17041468620300293,
      "rewards/margins": 0.0310073085129261,
      "rewards/rejected": -0.20142200589179993,
      "step": 147
    },
    {
      "epoch": 0.23325453112687156,
      "grad_norm": 0.43584540486335754,
      "learning_rate": 3.996978950020517e-06,
      "log_odds_chosen": 0.23817729949951172,
      "log_odds_ratio": -0.5939024090766907,
      "logits/chosen": 0.11393069475889206,
      "logits/rejected": 0.006526513025164604,
      "logps/chosen": -1.8926572799682617,
      "logps/rejected": -2.095795154571533,
      "loss": 2.1141,
      "nll_loss": 2.0546836853027344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18926572799682617,
      "rewards/margins": 0.020313819870352745,
      "rewards/rejected": -0.20957954227924347,
      "step": 148
    },
    {
      "epoch": 0.23483057525610718,
      "grad_norm": 0.37672045826911926,
      "learning_rate": 3.996669378370959e-06,
      "log_odds_chosen": 0.5706081986427307,
      "log_odds_ratio": -0.4550899863243103,
      "logits/chosen": -0.02084711194038391,
      "logits/rejected": -0.013911699876189232,
      "logps/chosen": -1.6749799251556396,
      "logps/rejected": -2.16387939453125,
      "loss": 1.8911,
      "nll_loss": 1.8456203937530518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16749800741672516,
      "rewards/margins": 0.04888994246721268,
      "rewards/rejected": -0.21638792753219604,
      "step": 149
    },
    {
      "epoch": 0.2364066193853428,
      "grad_norm": 0.5808645486831665,
      "learning_rate": 3.996344722837929e-06,
      "log_odds_chosen": 0.5190019011497498,
      "log_odds_ratio": -0.4721868932247162,
      "logits/chosen": -0.027724824845790863,
      "logits/rejected": -0.011851985938847065,
      "logps/chosen": -1.842341661453247,
      "logps/rejected": -2.2962372303009033,
      "loss": 2.0389,
      "nll_loss": 1.9916990995407104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18423417210578918,
      "rewards/margins": 0.045389577746391296,
      "rewards/rejected": -0.2296237349510193,
      "step": 150
    },
    {
      "epoch": 0.23798266351457842,
      "grad_norm": 0.35176411271095276,
      "learning_rate": 3.996004985874043e-06,
      "log_odds_chosen": 0.2666694223880768,
      "log_odds_ratio": -0.5715771913528442,
      "logits/chosen": 0.09955260902643204,
      "logits/rejected": -0.007417397573590279,
      "logps/chosen": -1.7254502773284912,
      "logps/rejected": -1.9494432210922241,
      "loss": 1.9513,
      "nll_loss": 1.8941357135772705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17254504561424255,
      "rewards/margins": 0.02239927649497986,
      "rewards/rejected": -0.1949443221092224,
      "step": 151
    },
    {
      "epoch": 0.23955870764381404,
      "grad_norm": 0.39265239238739014,
      "learning_rate": 3.995650170045855e-06,
      "log_odds_chosen": 0.31694257259368896,
      "log_odds_ratio": -0.5533034801483154,
      "logits/chosen": 0.014175724238157272,
      "logits/rejected": -0.0333397313952446,
      "logps/chosen": -1.7931270599365234,
      "logps/rejected": -2.063905954360962,
      "loss": 2.0047,
      "nll_loss": 1.9493213891983032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17931270599365234,
      "rewards/margins": 0.027077894657850266,
      "rewards/rejected": -0.2063905894756317,
      "step": 152
    },
    {
      "epoch": 0.24113475177304963,
      "grad_norm": 0.40053820610046387,
      "learning_rate": 3.995280278033825e-06,
      "log_odds_chosen": 0.23943039774894714,
      "log_odds_ratio": -0.589293360710144,
      "logits/chosen": 0.09469472616910934,
      "logits/rejected": 0.06681188195943832,
      "logps/chosen": -1.8935778141021729,
      "logps/rejected": -2.0991814136505127,
      "loss": 2.1063,
      "nll_loss": 2.047382354736328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18935777246952057,
      "rewards/margins": 0.020560378208756447,
      "rewards/rejected": -0.20991815626621246,
      "step": 153
    },
    {
      "epoch": 0.24271079590228525,
      "grad_norm": 0.3706309199333191,
      "learning_rate": 3.994895312632314e-06,
      "log_odds_chosen": 0.5549086928367615,
      "log_odds_ratio": -0.459675669670105,
      "logits/chosen": 0.16506488621234894,
      "logits/rejected": -0.45457974076271057,
      "logps/chosen": -1.7671860456466675,
      "logps/rejected": -2.246805429458618,
      "loss": 1.9762,
      "nll_loss": 1.9302539825439453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17671862244606018,
      "rewards/margins": 0.04796193540096283,
      "rewards/rejected": -0.224680557847023,
      "step": 154
    },
    {
      "epoch": 0.24428684003152087,
      "grad_norm": 0.3580048680305481,
      "learning_rate": 3.994495276749549e-06,
      "log_odds_chosen": 0.505012035369873,
      "log_odds_ratio": -0.4793119430541992,
      "logits/chosen": 0.06876569986343384,
      "logits/rejected": -0.03154408931732178,
      "logps/chosen": -1.786881446838379,
      "logps/rejected": -2.225484609603882,
      "loss": 1.9825,
      "nll_loss": 1.934592843055725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1786881536245346,
      "rewards/margins": 0.04386031627655029,
      "rewards/rejected": -0.2225484549999237,
      "step": 155
    },
    {
      "epoch": 0.2458628841607565,
      "grad_norm": 0.35918235778808594,
      "learning_rate": 3.994080173407612e-06,
      "log_odds_chosen": 0.2776962220668793,
      "log_odds_ratio": -0.5682939291000366,
      "logits/chosen": 0.12636655569076538,
      "logits/rejected": -0.06872869282960892,
      "logps/chosen": -1.819372296333313,
      "logps/rejected": -2.0557971000671387,
      "loss": 2.0238,
      "nll_loss": 1.967012882232666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18193721771240234,
      "rewards/margins": 0.023642467334866524,
      "rewards/rejected": -0.20557968318462372,
      "step": 156
    },
    {
      "epoch": 0.24743892828999212,
      "grad_norm": 0.3515869081020355,
      "learning_rate": 3.993650005742409e-06,
      "log_odds_chosen": 0.345825731754303,
      "log_odds_ratio": -0.5401284694671631,
      "logits/chosen": 0.05361388623714447,
      "logits/rejected": -0.005025242455303669,
      "logps/chosen": -1.7907469272613525,
      "logps/rejected": -2.0871944427490234,
      "loss": 1.9961,
      "nll_loss": 1.9421340227127075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17907467484474182,
      "rewards/margins": 0.029644787311553955,
      "rewards/rejected": -0.20871947705745697,
      "step": 157
    },
    {
      "epoch": 0.24901497241922774,
      "grad_norm": 0.3238402009010315,
      "learning_rate": 3.993204777003652e-06,
      "log_odds_chosen": 0.36766839027404785,
      "log_odds_ratio": -0.531983494758606,
      "logits/chosen": 0.0647420734167099,
      "logits/rejected": -0.17432400584220886,
      "logps/chosen": -1.720491647720337,
      "logps/rejected": -2.0343194007873535,
      "loss": 1.9217,
      "nll_loss": 1.8685407638549805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17204917967319489,
      "rewards/margins": 0.031382784247398376,
      "rewards/rejected": -0.20343194901943207,
      "step": 158
    },
    {
      "epoch": 0.25059101654846333,
      "grad_norm": 0.3346635699272156,
      "learning_rate": 3.992744490554832e-06,
      "log_odds_chosen": 0.35473060607910156,
      "log_odds_ratio": -0.537135899066925,
      "logits/chosen": 0.2205273061990738,
      "logits/rejected": -0.0024103261530399323,
      "logps/chosen": -1.7656519412994385,
      "logps/rejected": -2.068941116333008,
      "loss": 1.9832,
      "nll_loss": 1.9294898509979248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17656521499156952,
      "rewards/margins": 0.030328912660479546,
      "rewards/rejected": -0.20689409971237183,
      "step": 159
    },
    {
      "epoch": 0.25216706067769895,
      "grad_norm": 0.3138229250907898,
      "learning_rate": 3.992269149873192e-06,
      "log_odds_chosen": 0.34384748339653015,
      "log_odds_ratio": -0.5416197776794434,
      "logits/chosen": 0.021261127665638924,
      "logits/rejected": -0.09686248004436493,
      "logps/chosen": -1.7365537881851196,
      "logps/rejected": -2.0252068042755127,
      "loss": 1.9407,
      "nll_loss": 1.8865716457366943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17365536093711853,
      "rewards/margins": 0.028865298256278038,
      "rewards/rejected": -0.20252066850662231,
      "step": 160
    },
    {
      "epoch": 0.25374310480693457,
      "grad_norm": 0.3120760917663574,
      "learning_rate": 3.991778758549705e-06,
      "log_odds_chosen": 0.21335069835186005,
      "log_odds_ratio": -0.5993782877922058,
      "logits/chosen": 0.2368992120027542,
      "logits/rejected": 0.06489282846450806,
      "logps/chosen": -1.8601230382919312,
      "logps/rejected": -2.0463929176330566,
      "loss": 2.0466,
      "nll_loss": 1.9866821765899658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18601229786872864,
      "rewards/margins": 0.0186269860714674,
      "rewards/rejected": -0.2046392858028412,
      "step": 161
    },
    {
      "epoch": 0.2553191489361702,
      "grad_norm": 0.3198974132537842,
      "learning_rate": 3.9912733202890415e-06,
      "log_odds_chosen": 0.3527311384677887,
      "log_odds_ratio": -0.53630530834198,
      "logits/chosen": 0.10290198773145676,
      "logits/rejected": -0.1413993537425995,
      "logps/chosen": -1.7850815057754517,
      "logps/rejected": -2.086127519607544,
      "loss": 1.9862,
      "nll_loss": 1.9325822591781616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17850814759731293,
      "rewards/margins": 0.03010460175573826,
      "rewards/rejected": -0.20861275494098663,
      "step": 162
    },
    {
      "epoch": 0.2568951930654058,
      "grad_norm": 0.3027884066104889,
      "learning_rate": 3.990752838909548e-06,
      "log_odds_chosen": 0.21511955559253693,
      "log_odds_ratio": -0.5959004759788513,
      "logits/chosen": 0.2015780210494995,
      "logits/rejected": -0.022823316976428032,
      "logps/chosen": -1.8808540105819702,
      "logps/rejected": -2.0652594566345215,
      "loss": 2.072,
      "nll_loss": 2.012399911880493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1880854070186615,
      "rewards/margins": 0.01844053901731968,
      "rewards/rejected": -0.20652592182159424,
      "step": 163
    },
    {
      "epoch": 0.25847123719464143,
      "grad_norm": 0.3271465599536896,
      "learning_rate": 3.990217318343213e-06,
      "log_odds_chosen": 0.5507936477661133,
      "log_odds_ratio": -0.45798903703689575,
      "logits/chosen": 0.1652592420578003,
      "logits/rejected": -0.24176833033561707,
      "logps/chosen": -1.9504748582839966,
      "logps/rejected": -2.439309597015381,
      "loss": 2.1296,
      "nll_loss": 2.083810567855835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1950474977493286,
      "rewards/margins": 0.04888347536325455,
      "rewards/rejected": -0.24393096566200256,
      "step": 164
    },
    {
      "epoch": 0.26004728132387706,
      "grad_norm": 0.31373828649520874,
      "learning_rate": 3.989666762635637e-06,
      "log_odds_chosen": 0.3172238767147064,
      "log_odds_ratio": -0.5503073334693909,
      "logits/chosen": 0.15342219173908234,
      "logits/rejected": -0.018054338172078133,
      "logps/chosen": -1.7960243225097656,
      "logps/rejected": -2.067051887512207,
      "loss": 2.0099,
      "nll_loss": 1.9548213481903076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17960244417190552,
      "rewards/margins": 0.027102762833237648,
      "rewards/rejected": -0.20670519769191742,
      "step": 165
    },
    {
      "epoch": 0.2616233254531127,
      "grad_norm": 0.30167585611343384,
      "learning_rate": 3.9891011759460056e-06,
      "log_odds_chosen": 0.3845152258872986,
      "log_odds_ratio": -0.5250376462936401,
      "logits/chosen": 0.14441440999507904,
      "logits/rejected": -0.09336265176534653,
      "logps/chosen": -1.7806988954544067,
      "logps/rejected": -2.10469913482666,
      "loss": 1.9846,
      "nll_loss": 1.9321050643920898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17806988954544067,
      "rewards/margins": 0.03240001201629639,
      "rewards/rejected": -0.21046990156173706,
      "step": 166
    },
    {
      "epoch": 0.2631993695823483,
      "grad_norm": 0.2798842787742615,
      "learning_rate": 3.988520562547057e-06,
      "log_odds_chosen": 0.4200190603733063,
      "log_odds_ratio": -0.5160478949546814,
      "logits/chosen": 0.15987613797187805,
      "logits/rejected": -0.1669200360774994,
      "logps/chosen": -1.7405807971954346,
      "logps/rejected": -2.096102714538574,
      "loss": 1.9268,
      "nll_loss": 1.875168800354004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17405809462070465,
      "rewards/margins": 0.03555220365524292,
      "rewards/rejected": -0.20961029827594757,
      "step": 167
    },
    {
      "epoch": 0.2647754137115839,
      "grad_norm": 0.30115410685539246,
      "learning_rate": 3.987924926825047e-06,
      "log_odds_chosen": 0.44322502613067627,
      "log_odds_ratio": -0.5085245370864868,
      "logits/chosen": 0.1368703693151474,
      "logits/rejected": -0.08649970591068268,
      "logps/chosen": -1.8128654956817627,
      "logps/rejected": -2.1969082355499268,
      "loss": 2.0227,
      "nll_loss": 1.971897006034851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18128655850887299,
      "rewards/margins": 0.03840426355600357,
      "rewards/rejected": -0.21969082951545715,
      "step": 168
    },
    {
      "epoch": 0.26635145784081954,
      "grad_norm": 0.29241764545440674,
      "learning_rate": 3.98731427327972e-06,
      "log_odds_chosen": 0.4207753539085388,
      "log_odds_ratio": -0.5057156085968018,
      "logits/chosen": 0.1068802922964096,
      "logits/rejected": -0.30606210231781006,
      "logps/chosen": -1.747009515762329,
      "logps/rejected": -2.1049232482910156,
      "loss": 1.9335,
      "nll_loss": 1.8828821182250977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17470093071460724,
      "rewards/margins": 0.035791389644145966,
      "rewards/rejected": -0.2104923129081726,
      "step": 169
    },
    {
      "epoch": 0.26792750197005516,
      "grad_norm": 0.28724029660224915,
      "learning_rate": 3.986688606524273e-06,
      "log_odds_chosen": 0.35461530089378357,
      "log_odds_ratio": -0.5425211787223816,
      "logits/chosen": 0.2177436798810959,
      "logits/rejected": -0.5054653882980347,
      "logps/chosen": -1.7733808755874634,
      "logps/rejected": -2.080212354660034,
      "loss": 1.9646,
      "nll_loss": 1.9103646278381348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17733806371688843,
      "rewards/margins": 0.030683180317282677,
      "rewards/rejected": -0.20802125334739685,
      "step": 170
    },
    {
      "epoch": 0.2695035460992908,
      "grad_norm": 0.2893814444541931,
      "learning_rate": 3.986047931285315e-06,
      "log_odds_chosen": 0.5224654078483582,
      "log_odds_ratio": -0.4700799584388733,
      "logits/chosen": 0.10108586400747299,
      "logits/rejected": -0.44279980659484863,
      "logps/chosen": -1.6865884065628052,
      "logps/rejected": -2.128582239151001,
      "loss": 1.8737,
      "nll_loss": 1.8267263174057007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16865883767604828,
      "rewards/margins": 0.0441993847489357,
      "rewards/rejected": -0.21285822987556458,
      "step": 171
    },
    {
      "epoch": 0.2710795902285264,
      "grad_norm": 0.2736685574054718,
      "learning_rate": 3.985392252402847e-06,
      "log_odds_chosen": 0.25993895530700684,
      "log_odds_ratio": -0.5742053389549255,
      "logits/chosen": 0.0892399251461029,
      "logits/rejected": -0.24680867791175842,
      "logps/chosen": -1.6766496896743774,
      "logps/rejected": -1.8926042318344116,
      "loss": 1.8747,
      "nll_loss": 1.8173226118087769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16766498982906342,
      "rewards/margins": 0.02159545198082924,
      "rewards/rejected": -0.18926043808460236,
      "step": 172
    },
    {
      "epoch": 0.272655634357762,
      "grad_norm": 0.289073646068573,
      "learning_rate": 3.984721574830206e-06,
      "log_odds_chosen": 0.30355799198150635,
      "log_odds_ratio": -0.5568473935127258,
      "logits/chosen": 0.1579122394323349,
      "logits/rejected": -0.27320054173469543,
      "logps/chosen": -1.78755521774292,
      "logps/rejected": -2.046647548675537,
      "loss": 1.9764,
      "nll_loss": 1.9206898212432861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1787555068731308,
      "rewards/margins": 0.02590925060212612,
      "rewards/rejected": -0.20466476678848267,
      "step": 173
    },
    {
      "epoch": 0.27423167848699764,
      "grad_norm": 0.2663167715072632,
      "learning_rate": 3.984035903634041e-06,
      "log_odds_chosen": 0.30917906761169434,
      "log_odds_ratio": -0.5573617815971375,
      "logits/chosen": 0.2104271948337555,
      "logits/rejected": -0.3466285467147827,
      "logps/chosen": -1.7617706060409546,
      "logps/rejected": -2.023843288421631,
      "loss": 1.9372,
      "nll_loss": 1.8814386129379272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17617705464363098,
      "rewards/margins": 0.026207279413938522,
      "rewards/rejected": -0.2023843377828598,
      "step": 174
    },
    {
      "epoch": 0.27580772261623326,
      "grad_norm": 0.2792463004589081,
      "learning_rate": 3.983335243994273e-06,
      "log_odds_chosen": 0.4087638258934021,
      "log_odds_ratio": -0.5167474746704102,
      "logits/chosen": 0.21963684260845184,
      "logits/rejected": -0.2987426817417145,
      "logps/chosen": -1.7534160614013672,
      "logps/rejected": -2.1018359661102295,
      "loss": 1.9423,
      "nll_loss": 1.8906673192977905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17534160614013672,
      "rewards/margins": 0.03484198823571205,
      "rewards/rejected": -0.21018359065055847,
      "step": 175
    },
    {
      "epoch": 0.2773837667454689,
      "grad_norm": 0.30124711990356445,
      "learning_rate": 3.982619601204049e-06,
      "log_odds_chosen": 0.3245881497859955,
      "log_odds_ratio": -0.5491042137145996,
      "logits/chosen": 0.1435730904340744,
      "logits/rejected": -0.3272641599178314,
      "logps/chosen": -1.8128660917282104,
      "logps/rejected": -2.0914556980133057,
      "loss": 2.001,
      "nll_loss": 1.9461297988891602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18128660321235657,
      "rewards/margins": 0.027858952060341835,
      "rewards/rejected": -0.20914556086063385,
      "step": 176
    },
    {
      "epoch": 0.2789598108747045,
      "grad_norm": 0.2849816083908081,
      "learning_rate": 3.9818889806697085e-06,
      "log_odds_chosen": 0.3977530002593994,
      "log_odds_ratio": -0.5233108401298523,
      "logits/chosen": 0.2636592984199524,
      "logits/rejected": -0.3068069815635681,
      "logps/chosen": -1.8528499603271484,
      "logps/rejected": -2.1990413665771484,
      "loss": 2.0173,
      "nll_loss": 1.964940071105957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18528500199317932,
      "rewards/margins": 0.03461914509534836,
      "rewards/rejected": -0.2199041247367859,
      "step": 177
    },
    {
      "epoch": 0.2805358550039401,
      "grad_norm": 0.27957502007484436,
      "learning_rate": 3.98114338791074e-06,
      "log_odds_chosen": 0.49340713024139404,
      "log_odds_ratio": -0.4914700388908386,
      "logits/chosen": 0.13450200855731964,
      "logits/rejected": -0.340486079454422,
      "logps/chosen": -1.7227312326431274,
      "logps/rejected": -2.1473944187164307,
      "loss": 1.915,
      "nll_loss": 1.865891933441162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17227312922477722,
      "rewards/margins": 0.04246632009744644,
      "rewards/rejected": -0.21473942697048187,
      "step": 178
    },
    {
      "epoch": 0.28211189913317575,
      "grad_norm": 0.2686019837856293,
      "learning_rate": 3.980382828559742e-06,
      "log_odds_chosen": 0.39518705010414124,
      "log_odds_ratio": -0.5202585458755493,
      "logits/chosen": 0.14406242966651917,
      "logits/rejected": -0.3577483594417572,
      "logps/chosen": -1.7618299722671509,
      "logps/rejected": -2.098731279373169,
      "loss": 1.9315,
      "nll_loss": 1.8795194625854492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17618300020694733,
      "rewards/margins": 0.033690109848976135,
      "rewards/rejected": -0.20987312495708466,
      "step": 179
    },
    {
      "epoch": 0.28368794326241137,
      "grad_norm": 0.28153085708618164,
      "learning_rate": 3.9796073083623774e-06,
      "log_odds_chosen": 0.3710815906524658,
      "log_odds_ratio": -0.5394359230995178,
      "logits/chosen": 0.16021382808685303,
      "logits/rejected": -0.4299715459346771,
      "logps/chosen": -1.836319088935852,
      "logps/rejected": -2.1562142372131348,
      "loss": 2.0036,
      "nll_loss": 1.9496216773986816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18363192677497864,
      "rewards/margins": 0.031989507377147675,
      "rewards/rejected": -0.21562142670154572,
      "step": 180
    },
    {
      "epoch": 0.285263987391647,
      "grad_norm": 0.2878672182559967,
      "learning_rate": 3.978816833177329e-06,
      "log_odds_chosen": 0.3781207203865051,
      "log_odds_ratio": -0.5273905992507935,
      "logits/chosen": 0.2487175166606903,
      "logits/rejected": -0.232444629073143,
      "logps/chosen": -1.759475588798523,
      "logps/rejected": -2.082449436187744,
      "loss": 1.9353,
      "nll_loss": 1.8825562000274658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17594757676124573,
      "rewards/margins": 0.03229738026857376,
      "rewards/rejected": -0.2082449495792389,
      "step": 181
    },
    {
      "epoch": 0.2868400315208826,
      "grad_norm": 0.28296828269958496,
      "learning_rate": 3.978011408976261e-06,
      "log_odds_chosen": 0.24777647852897644,
      "log_odds_ratio": -0.5824975371360779,
      "logits/chosen": 0.20205409824848175,
      "logits/rejected": -0.42155444622039795,
      "logps/chosen": -1.77714204788208,
      "logps/rejected": -1.985145092010498,
      "loss": 1.9669,
      "nll_loss": 1.9086382389068604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17771419882774353,
      "rewards/margins": 0.020800314843654633,
      "rewards/rejected": -0.19851452112197876,
      "step": 182
    },
    {
      "epoch": 0.28841607565011823,
      "grad_norm": 0.29371410608291626,
      "learning_rate": 3.9771910418437674e-06,
      "log_odds_chosen": 0.32709380984306335,
      "log_odds_ratio": -0.544011116027832,
      "logits/chosen": 0.20650595426559448,
      "logits/rejected": -0.11976965516805649,
      "logps/chosen": -1.8121384382247925,
      "logps/rejected": -2.093899726867676,
      "loss": 1.9856,
      "nll_loss": 1.9311554431915283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1812138557434082,
      "rewards/margins": 0.028176117688417435,
      "rewards/rejected": -0.20938995480537415,
      "step": 183
    },
    {
      "epoch": 0.2899921197793538,
      "grad_norm": 0.28328511118888855,
      "learning_rate": 3.976355737977332e-06,
      "log_odds_chosen": 0.39620110392570496,
      "log_odds_ratio": -0.5227944254875183,
      "logits/chosen": 0.21991243958473206,
      "logits/rejected": -0.3309701681137085,
      "logps/chosen": -1.7577619552612305,
      "logps/rejected": -2.0953731536865234,
      "loss": 1.939,
      "nll_loss": 1.8866724967956543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1757761836051941,
      "rewards/margins": 0.03376111388206482,
      "rewards/rejected": -0.2095372974872589,
      "step": 184
    },
    {
      "epoch": 0.2915681639085894,
      "grad_norm": 0.2596893608570099,
      "learning_rate": 3.975505503687274e-06,
      "log_odds_chosen": 0.3184034824371338,
      "log_odds_ratio": -0.5498912930488586,
      "logits/chosen": 0.26362285017967224,
      "logits/rejected": -0.1583450883626938,
      "logps/chosen": -1.7432103157043457,
      "logps/rejected": -2.0135068893432617,
      "loss": 1.9195,
      "nll_loss": 1.864485263824463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1743210256099701,
      "rewards/margins": 0.027029668912291527,
      "rewards/rejected": -0.20135070383548737,
      "step": 185
    },
    {
      "epoch": 0.29314420803782504,
      "grad_norm": 0.25459468364715576,
      "learning_rate": 3.974640345396708e-06,
      "log_odds_chosen": 0.4846782386302948,
      "log_odds_ratio": -0.4870211184024811,
      "logits/chosen": 0.1184130385518074,
      "logits/rejected": -0.6584069132804871,
      "logps/chosen": -1.7136808633804321,
      "logps/rejected": -2.1256215572357178,
      "loss": 1.8877,
      "nll_loss": 1.8389503955841064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1713680922985077,
      "rewards/margins": 0.041194066405296326,
      "rewards/rejected": -0.21256215870380402,
      "step": 186
    },
    {
      "epoch": 0.29472025216706066,
      "grad_norm": 0.2585580050945282,
      "learning_rate": 3.9737602696414925e-06,
      "log_odds_chosen": 0.17746226489543915,
      "log_odds_ratio": -0.6186371445655823,
      "logits/chosen": 0.2338634729385376,
      "logits/rejected": -0.4608267545700073,
      "logps/chosen": -1.804581880569458,
      "logps/rejected": -1.954529881477356,
      "loss": 1.9595,
      "nll_loss": 1.897607445716858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1804581880569458,
      "rewards/margins": 0.014994792640209198,
      "rewards/rejected": -0.1954529881477356,
      "step": 187
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 0.2511560618877411,
      "learning_rate": 3.972865283070179e-06,
      "log_odds_chosen": 0.40657275915145874,
      "log_odds_ratio": -0.5161192417144775,
      "logits/chosen": 0.18840941786766052,
      "logits/rejected": -0.6009777188301086,
      "logps/chosen": -1.7036168575286865,
      "logps/rejected": -2.0506670475006104,
      "loss": 1.8859,
      "nll_loss": 1.8342941999435425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1703616976737976,
      "rewards/margins": 0.0347050204873085,
      "rewards/rejected": -0.2050667107105255,
      "step": 188
    },
    {
      "epoch": 0.2978723404255319,
      "grad_norm": 0.26080071926116943,
      "learning_rate": 3.971955392443965e-06,
      "log_odds_chosen": 0.3762245774269104,
      "log_odds_ratio": -0.528282642364502,
      "logits/chosen": 0.2339557558298111,
      "logits/rejected": -0.35206758975982666,
      "logps/chosen": -1.735405683517456,
      "logps/rejected": -2.0585732460021973,
      "loss": 1.905,
      "nll_loss": 1.852203607559204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17354057729244232,
      "rewards/margins": 0.03231675922870636,
      "rewards/rejected": -0.2058573216199875,
      "step": 189
    },
    {
      "epoch": 0.2994483845547675,
      "grad_norm": 0.26392868161201477,
      "learning_rate": 3.971030604636637e-06,
      "log_odds_chosen": 0.2363440990447998,
      "log_odds_ratio": -0.5862751007080078,
      "logits/chosen": 0.2692638039588928,
      "logits/rejected": -0.36195865273475647,
      "logps/chosen": -1.7536004781723022,
      "logps/rejected": -1.9518376588821411,
      "loss": 1.9225,
      "nll_loss": 1.8638556003570557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1753600686788559,
      "rewards/margins": 0.019823705777525902,
      "rewards/rejected": -0.19518375396728516,
      "step": 190
    },
    {
      "epoch": 0.30102442868400314,
      "grad_norm": 0.2797010540962219,
      "learning_rate": 3.970090926634526e-06,
      "log_odds_chosen": 0.31876862049102783,
      "log_odds_ratio": -0.5496628284454346,
      "logits/chosen": 0.33125385642051697,
      "logits/rejected": -0.578338623046875,
      "logps/chosen": -1.8113794326782227,
      "logps/rejected": -2.0837459564208984,
      "loss": 1.9864,
      "nll_loss": 1.931410789489746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18113794922828674,
      "rewards/margins": 0.027236640453338623,
      "rewards/rejected": -0.20837458968162537,
      "step": 191
    },
    {
      "epoch": 0.30260047281323876,
      "grad_norm": 0.2825038731098175,
      "learning_rate": 3.9691363655364526e-06,
      "log_odds_chosen": 0.3615967631340027,
      "log_odds_ratio": -0.5366310477256775,
      "logits/chosen": 0.23022601008415222,
      "logits/rejected": -0.39873024821281433,
      "logps/chosen": -1.787469744682312,
      "logps/rejected": -2.097710609436035,
      "loss": 1.9624,
      "nll_loss": 1.9087271690368652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1787469983100891,
      "rewards/margins": 0.031024089083075523,
      "rewards/rejected": -0.209771066904068,
      "step": 192
    },
    {
      "epoch": 0.3041765169424744,
      "grad_norm": 0.26274222135543823,
      "learning_rate": 3.968166928553666e-06,
      "log_odds_chosen": 0.3207397162914276,
      "log_odds_ratio": -0.5503286123275757,
      "logits/chosen": 0.2751014530658722,
      "logits/rejected": -0.5721142888069153,
      "logps/chosen": -1.731804370880127,
      "logps/rejected": -2.001689910888672,
      "loss": 1.9222,
      "nll_loss": 1.8671228885650635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1731804460287094,
      "rewards/margins": 0.026988547295331955,
      "rewards/rejected": -0.20016898214817047,
      "step": 193
    },
    {
      "epoch": 0.30575256107171,
      "grad_norm": 0.26265692710876465,
      "learning_rate": 3.967182623009804e-06,
      "log_odds_chosen": 0.4414345324039459,
      "log_odds_ratio": -0.5021454691886902,
      "logits/chosen": 0.22120928764343262,
      "logits/rejected": -0.43845516443252563,
      "logps/chosen": -1.760418176651001,
      "logps/rejected": -2.14136004447937,
      "loss": 1.9192,
      "nll_loss": 1.8690216541290283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.176041841506958,
      "rewards/margins": 0.0380941778421402,
      "rewards/rejected": -0.2141360193490982,
      "step": 194
    },
    {
      "epoch": 0.3073286052009456,
      "grad_norm": 0.30240899324417114,
      "learning_rate": 3.966183456340821e-06,
      "log_odds_chosen": 0.21254514157772064,
      "log_odds_ratio": -0.5945923328399658,
      "logits/chosen": 0.27597901225090027,
      "logits/rejected": -0.4168132543563843,
      "logps/chosen": -1.7859089374542236,
      "logps/rejected": -1.9644159078598022,
      "loss": 1.96,
      "nll_loss": 1.9005576372146606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17859089374542236,
      "rewards/margins": 0.017850710079073906,
      "rewards/rejected": -0.19644160568714142,
      "step": 195
    },
    {
      "epoch": 0.30890464933018125,
      "grad_norm": 0.28170743584632874,
      "learning_rate": 3.965169436094947e-06,
      "log_odds_chosen": 0.38190627098083496,
      "log_odds_ratio": -0.5301341414451599,
      "logits/chosen": 0.2661678194999695,
      "logits/rejected": -0.5325687527656555,
      "logps/chosen": -1.6825157403945923,
      "logps/rejected": -2.00687313079834,
      "loss": 1.8654,
      "nll_loss": 1.8123762607574463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16825160384178162,
      "rewards/margins": 0.03243573009967804,
      "rewards/rejected": -0.20068733394145966,
      "step": 196
    },
    {
      "epoch": 0.31048069345941687,
      "grad_norm": 0.2710944414138794,
      "learning_rate": 3.964140569932618e-06,
      "log_odds_chosen": 0.3068499267101288,
      "log_odds_ratio": -0.5730952620506287,
      "logits/chosen": 0.2248048484325409,
      "logits/rejected": -0.7354637980461121,
      "logps/chosen": -1.806632161140442,
      "logps/rejected": -2.067098379135132,
      "loss": 1.9701,
      "nll_loss": 1.912825345993042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18066321313381195,
      "rewards/margins": 0.026046616956591606,
      "rewards/rejected": -0.2067098468542099,
      "step": 197
    },
    {
      "epoch": 0.3120567375886525,
      "grad_norm": 0.31186628341674805,
      "learning_rate": 3.9630968656264285e-06,
      "log_odds_chosen": 0.44008272886276245,
      "log_odds_ratio": -0.503641664981842,
      "logits/chosen": 0.24553100764751434,
      "logits/rejected": -0.38101547956466675,
      "logps/chosen": -1.7257674932479858,
      "logps/rejected": -2.101093292236328,
      "loss": 1.8853,
      "nll_loss": 1.8349802494049072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17257672548294067,
      "rewards/margins": 0.03753257542848587,
      "rewards/rejected": -0.21010930836200714,
      "step": 198
    },
    {
      "epoch": 0.3136327817178881,
      "grad_norm": 0.2788161337375641,
      "learning_rate": 3.962038331061065e-06,
      "log_odds_chosen": 0.17320865392684937,
      "log_odds_ratio": -0.622861385345459,
      "logits/chosen": 0.321949303150177,
      "logits/rejected": -0.5195744037628174,
      "logps/chosen": -1.7807538509368896,
      "logps/rejected": -1.9327521324157715,
      "loss": 1.9626,
      "nll_loss": 1.900266408920288,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1780754029750824,
      "rewards/margins": 0.015199808403849602,
      "rewards/rejected": -0.19327519834041595,
      "step": 199
    },
    {
      "epoch": 0.31520882584712373,
      "grad_norm": 0.27439412474632263,
      "learning_rate": 3.96096497423325e-06,
      "log_odds_chosen": 0.2597573399543762,
      "log_odds_ratio": -0.5759460926055908,
      "logits/chosen": 0.36237332224845886,
      "logits/rejected": -0.16011810302734375,
      "logps/chosen": -1.6894915103912354,
      "logps/rejected": -1.9068094491958618,
      "loss": 1.8906,
      "nll_loss": 1.8330268859863281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.168949156999588,
      "rewards/margins": 0.021731778979301453,
      "rewards/rejected": -0.19068092107772827,
      "step": 200
    },
    {
      "epoch": 0.31678486997635935,
      "grad_norm": 0.25059738755226135,
      "learning_rate": 3.959876803251684e-06,
      "log_odds_chosen": 0.39733871817588806,
      "log_odds_ratio": -0.5207474231719971,
      "logits/chosen": 0.21183565258979797,
      "logits/rejected": -0.4376421570777893,
      "logps/chosen": -1.7023674249649048,
      "logps/rejected": -2.038724184036255,
      "loss": 1.8566,
      "nll_loss": 1.8045035600662231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1702367514371872,
      "rewards/margins": 0.033635687083005905,
      "rewards/rejected": -0.2038724273443222,
      "step": 201
    },
    {
      "epoch": 0.31836091410559497,
      "grad_norm": 0.25566282868385315,
      "learning_rate": 3.958773826336977e-06,
      "log_odds_chosen": 0.2807323634624481,
      "log_odds_ratio": -0.564405083656311,
      "logits/chosen": 0.31915491819381714,
      "logits/rejected": -0.293272465467453,
      "logps/chosen": -1.7425774335861206,
      "logps/rejected": -1.9793710708618164,
      "loss": 1.9173,
      "nll_loss": 1.8608673810958862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1742577701807022,
      "rewards/margins": 0.0236793402582407,
      "rewards/rejected": -0.19793710112571716,
      "step": 202
    },
    {
      "epoch": 0.3199369582348306,
      "grad_norm": 0.26755860447883606,
      "learning_rate": 3.957656051821592e-06,
      "log_odds_chosen": 0.2489960789680481,
      "log_odds_ratio": -0.5815601348876953,
      "logits/chosen": 0.3447924852371216,
      "logits/rejected": -0.6091679930686951,
      "logps/chosen": -1.7780265808105469,
      "logps/rejected": -1.9914788007736206,
      "loss": 1.9267,
      "nll_loss": 1.868532419204712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1778026670217514,
      "rewards/margins": 0.02134522795677185,
      "rewards/rejected": -0.19914788007736206,
      "step": 203
    },
    {
      "epoch": 0.3215130023640662,
      "grad_norm": 0.27641505002975464,
      "learning_rate": 3.956523488149783e-06,
      "log_odds_chosen": 0.31930261850357056,
      "log_odds_ratio": -0.5506942272186279,
      "logits/chosen": 0.3657802641391754,
      "logits/rejected": -0.375068724155426,
      "logps/chosen": -1.8819518089294434,
      "logps/rejected": -2.158445358276367,
      "loss": 2.0406,
      "nll_loss": 1.9855563640594482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18819519877433777,
      "rewards/margins": 0.02764933556318283,
      "rewards/rejected": -0.2158445119857788,
      "step": 204
    },
    {
      "epoch": 0.32308904649330183,
      "grad_norm": 0.27057579159736633,
      "learning_rate": 3.9553761438775285e-06,
      "log_odds_chosen": 0.5523542165756226,
      "log_odds_ratio": -0.4604877531528473,
      "logits/chosen": 0.22423981130123138,
      "logits/rejected": -0.40424689650535583,
      "logps/chosen": -1.6142833232879639,
      "logps/rejected": -2.081634521484375,
      "loss": 1.7752,
      "nll_loss": 1.7291845083236694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1614283323287964,
      "rewards/margins": 0.04673512279987335,
      "rewards/rejected": -0.20816347002983093,
      "step": 205
    },
    {
      "epoch": 0.32466509062253746,
      "grad_norm": 0.24270053207874298,
      "learning_rate": 3.954214027672465e-06,
      "log_odds_chosen": 0.49918606877326965,
      "log_odds_ratio": -0.48198771476745605,
      "logits/chosen": 0.22353754937648773,
      "logits/rejected": -0.8031458854675293,
      "logps/chosen": -1.7024301290512085,
      "logps/rejected": -2.131436347961426,
      "loss": 1.8545,
      "nll_loss": 1.8062725067138672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1702430099248886,
      "rewards/margins": 0.04290061816573143,
      "rewards/rejected": -0.21314363181591034,
      "step": 206
    },
    {
      "epoch": 0.3262411347517731,
      "grad_norm": 0.2744021713733673,
      "learning_rate": 3.953037148313825e-06,
      "log_odds_chosen": 0.2930293679237366,
      "log_odds_ratio": -0.5630696415901184,
      "logits/chosen": 0.3285022974014282,
      "logits/rejected": -0.46074235439300537,
      "logps/chosen": -1.6958521604537964,
      "logps/rejected": -1.9453150033950806,
      "loss": 1.8815,
      "nll_loss": 1.8252310752868652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1695852279663086,
      "rewards/margins": 0.024946292862296104,
      "rewards/rejected": -0.19453151524066925,
      "step": 207
    },
    {
      "epoch": 0.32781717888100864,
      "grad_norm": 0.28513532876968384,
      "learning_rate": 3.951845514692371e-06,
      "log_odds_chosen": 0.3969423770904541,
      "log_odds_ratio": -0.5238114595413208,
      "logits/chosen": 0.25184959173202515,
      "logits/rejected": -0.3663902282714844,
      "logps/chosen": -1.7550077438354492,
      "logps/rejected": -2.095698833465576,
      "loss": 1.9278,
      "nll_loss": 1.875378131866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1755007952451706,
      "rewards/margins": 0.03406910225749016,
      "rewards/rejected": -0.20956988632678986,
      "step": 208
    },
    {
      "epoch": 0.32939322301024426,
      "grad_norm": 0.2841278910636902,
      "learning_rate": 3.950639135810325e-06,
      "log_odds_chosen": 0.32432445883750916,
      "log_odds_ratio": -0.5494241118431091,
      "logits/chosen": 0.3648757338523865,
      "logits/rejected": -0.3396746516227722,
      "logps/chosen": -1.87786066532135,
      "logps/rejected": -2.157219171524048,
      "loss": 2.0224,
      "nll_loss": 1.967415452003479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1877860575914383,
      "rewards/margins": 0.027935855090618134,
      "rewards/rejected": -0.21572190523147583,
      "step": 209
    },
    {
      "epoch": 0.3309692671394799,
      "grad_norm": 0.2702895998954773,
      "learning_rate": 3.9494180207813044e-06,
      "log_odds_chosen": 0.25352904200553894,
      "log_odds_ratio": -0.5853363275527954,
      "logits/chosen": 0.3584989011287689,
      "logits/rejected": -0.6828628182411194,
      "logps/chosen": -1.7445666790008545,
      "logps/rejected": -1.9578887224197388,
      "loss": 1.9155,
      "nll_loss": 1.8569811582565308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17445670068264008,
      "rewards/margins": 0.02133220247924328,
      "rewards/rejected": -0.1957888901233673,
      "step": 210
    },
    {
      "epoch": 0.3325453112687155,
      "grad_norm": 0.2695715129375458,
      "learning_rate": 3.948182178830249e-06,
      "log_odds_chosen": 0.3178305923938751,
      "log_odds_ratio": -0.5667225122451782,
      "logits/chosen": 0.3227792978286743,
      "logits/rejected": -0.824029266834259,
      "logps/chosen": -1.7716305255889893,
      "logps/rejected": -2.047945737838745,
      "loss": 1.9257,
      "nll_loss": 1.869060754776001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17716306447982788,
      "rewards/margins": 0.027631523087620735,
      "rewards/rejected": -0.20479457080364227,
      "step": 211
    },
    {
      "epoch": 0.3341213553979511,
      "grad_norm": 0.29492443799972534,
      "learning_rate": 3.9469316192933545e-06,
      "log_odds_chosen": 0.5440890192985535,
      "log_odds_ratio": -0.4644208550453186,
      "logits/chosen": 0.2088673710823059,
      "logits/rejected": -0.4241965413093567,
      "logps/chosen": -1.629172444343567,
      "logps/rejected": -2.0890142917633057,
      "loss": 1.798,
      "nll_loss": 1.751549482345581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16291725635528564,
      "rewards/margins": 0.0459841787815094,
      "rewards/rejected": -0.20890143513679504,
      "step": 212
    },
    {
      "epoch": 0.33569739952718675,
      "grad_norm": 0.26417070627212524,
      "learning_rate": 3.945666351618001e-06,
      "log_odds_chosen": 0.28261005878448486,
      "log_odds_ratio": -0.5651870369911194,
      "logits/chosen": 0.3082667589187622,
      "logits/rejected": -0.4787600636482239,
      "logps/chosen": -1.6829228401184082,
      "logps/rejected": -1.9201278686523438,
      "loss": 1.8496,
      "nll_loss": 1.7930916547775269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16829228401184082,
      "rewards/margins": 0.02372048795223236,
      "rewards/rejected": -0.19201278686523438,
      "step": 213
    },
    {
      "epoch": 0.33727344365642237,
      "grad_norm": 0.2696978449821472,
      "learning_rate": 3.9443863853626825e-06,
      "log_odds_chosen": 0.3259792923927307,
      "log_odds_ratio": -0.5532336831092834,
      "logits/chosen": 0.34843507409095764,
      "logits/rejected": -0.5844205021858215,
      "logps/chosen": -1.6465122699737549,
      "logps/rejected": -1.9122709035873413,
      "loss": 1.8357,
      "nll_loss": 1.7804213762283325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16465122997760773,
      "rewards/margins": 0.02657586894929409,
      "rewards/rejected": -0.19122707843780518,
      "step": 214
    },
    {
      "epoch": 0.338849487785658,
      "grad_norm": 0.2630347013473511,
      "learning_rate": 3.943091730196931e-06,
      "log_odds_chosen": 0.2849031388759613,
      "log_odds_ratio": -0.5690271258354187,
      "logits/chosen": 0.39357054233551025,
      "logits/rejected": -0.36922866106033325,
      "logps/chosen": -1.6450810432434082,
      "logps/rejected": -1.8856078386306763,
      "loss": 1.8206,
      "nll_loss": 1.7637048959732056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16450810432434082,
      "rewards/margins": 0.024052688851952553,
      "rewards/rejected": -0.18856079876422882,
      "step": 215
    },
    {
      "epoch": 0.3404255319148936,
      "grad_norm": 0.2966042160987854,
      "learning_rate": 3.941782395901249e-06,
      "log_odds_chosen": 0.35482144355773926,
      "log_odds_ratio": -0.5378621220588684,
      "logits/chosen": 0.27690693736076355,
      "logits/rejected": -0.5179506540298462,
      "logps/chosen": -1.7256001234054565,
      "logps/rejected": -2.026144027709961,
      "loss": 1.9001,
      "nll_loss": 1.846313714981079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17256002128124237,
      "rewards/margins": 0.030054409056901932,
      "rewards/rejected": -0.2026144117116928,
      "step": 216
    },
    {
      "epoch": 0.34200157604412923,
      "grad_norm": 0.271468847990036,
      "learning_rate": 3.940458392367032e-06,
      "log_odds_chosen": 0.32546529173851013,
      "log_odds_ratio": -0.5480217933654785,
      "logits/chosen": 0.30692675709724426,
      "logits/rejected": -0.7998979091644287,
      "logps/chosen": -1.775266170501709,
      "logps/rejected": -2.052006959915161,
      "loss": 1.9171,
      "nll_loss": 1.862282633781433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17752662301063538,
      "rewards/margins": 0.02767406962811947,
      "rewards/rejected": -0.2052006870508194,
      "step": 217
    },
    {
      "epoch": 0.34357762017336485,
      "grad_norm": 0.27401667833328247,
      "learning_rate": 3.939119729596493e-06,
      "log_odds_chosen": 0.3677333891391754,
      "log_odds_ratio": -0.5295292735099792,
      "logits/chosen": 0.36077094078063965,
      "logits/rejected": -0.6209971904754639,
      "logps/chosen": -1.7778162956237793,
      "logps/rejected": -2.092672109603882,
      "loss": 1.9304,
      "nll_loss": 1.8774876594543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1777816265821457,
      "rewards/margins": 0.031485576182603836,
      "rewards/rejected": -0.20926719903945923,
      "step": 218
    },
    {
      "epoch": 0.34515366430260047,
      "grad_norm": 0.26384812593460083,
      "learning_rate": 3.93776641770259e-06,
      "log_odds_chosen": 0.2971005141735077,
      "log_odds_ratio": -0.562300980091095,
      "logits/chosen": 0.3340204954147339,
      "logits/rejected": -0.6690645217895508,
      "logps/chosen": -1.745128870010376,
      "logps/rejected": -1.998238205909729,
      "loss": 1.8973,
      "nll_loss": 1.841081142425537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17451287806034088,
      "rewards/margins": 0.02531094290316105,
      "rewards/rejected": -0.19982382655143738,
      "step": 219
    },
    {
      "epoch": 0.3467297084318361,
      "grad_norm": 0.2645641565322876,
      "learning_rate": 3.93639846690895e-06,
      "log_odds_chosen": 0.43852394819259644,
      "log_odds_ratio": -0.5020437240600586,
      "logits/chosen": 0.3577096164226532,
      "logits/rejected": -0.8902232646942139,
      "logps/chosen": -1.7646898031234741,
      "logps/rejected": -2.1421992778778076,
      "loss": 1.9106,
      "nll_loss": 1.8603509664535522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17646898329257965,
      "rewards/margins": 0.0377509668469429,
      "rewards/rejected": -0.21421995759010315,
      "step": 220
    },
    {
      "epoch": 0.3483057525610717,
      "grad_norm": 0.26245489716529846,
      "learning_rate": 3.9350158875497855e-06,
      "log_odds_chosen": 0.5075678825378418,
      "log_odds_ratio": -0.47529348731040955,
      "logits/chosen": 0.36207839846611023,
      "logits/rejected": -0.7290895581245422,
      "logps/chosen": -1.6745859384536743,
      "logps/rejected": -2.103607654571533,
      "loss": 1.8345,
      "nll_loss": 1.7869353294372559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16745859384536743,
      "rewards/margins": 0.04290216416120529,
      "rewards/rejected": -0.21036075055599213,
      "step": 221
    },
    {
      "epoch": 0.34988179669030733,
      "grad_norm": 0.25845250487327576,
      "learning_rate": 3.933618690069824e-06,
      "log_odds_chosen": 0.46901246905326843,
      "log_odds_ratio": -0.4901207983493805,
      "logits/chosen": 0.22216065227985382,
      "logits/rejected": -0.667863667011261,
      "logps/chosen": -1.6221312284469604,
      "logps/rejected": -2.011317491531372,
      "loss": 1.7945,
      "nll_loss": 1.7455064058303833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16221313178539276,
      "rewards/margins": 0.0389186255633831,
      "rewards/rejected": -0.20113176107406616,
      "step": 222
    },
    {
      "epoch": 0.35145784081954295,
      "grad_norm": 0.26442599296569824,
      "learning_rate": 3.932206885024226e-06,
      "log_odds_chosen": 0.2337113320827484,
      "log_odds_ratio": -0.5857836604118347,
      "logits/chosen": 0.27757957577705383,
      "logits/rejected": -0.6246755123138428,
      "logps/chosen": -1.7342665195465088,
      "logps/rejected": -1.9312242269515991,
      "loss": 1.8784,
      "nll_loss": 1.8198186159133911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17342665791511536,
      "rewards/margins": 0.019695764407515526,
      "rewards/rejected": -0.19312243163585663,
      "step": 223
    },
    {
      "epoch": 0.3530338849487786,
      "grad_norm": 0.24192552268505096,
      "learning_rate": 3.930780483078502e-06,
      "log_odds_chosen": 0.3615175485610962,
      "log_odds_ratio": -0.5337653160095215,
      "logits/chosen": 0.24594762921333313,
      "logits/rejected": -0.6662352085113525,
      "logps/chosen": -1.6547470092773438,
      "logps/rejected": -1.9560520648956299,
      "loss": 1.8248,
      "nll_loss": 1.7714567184448242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16547471284866333,
      "rewards/margins": 0.03013051114976406,
      "rewards/rejected": -0.19560521841049194,
      "step": 224
    },
    {
      "epoch": 0.3546099290780142,
      "grad_norm": 0.2694976329803467,
      "learning_rate": 3.92933949500844e-06,
      "log_odds_chosen": 0.3964681923389435,
      "log_odds_ratio": -0.5309619307518005,
      "logits/chosen": 0.34370604157447815,
      "logits/rejected": -0.6380555033683777,
      "logps/chosen": -1.6309007406234741,
      "logps/rejected": -1.9560682773590088,
      "loss": 1.8016,
      "nll_loss": 1.7484827041625977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1630900502204895,
      "rewards/margins": 0.03251676633954048,
      "rewards/rejected": -0.19560682773590088,
      "step": 225
    },
    {
      "epoch": 0.3561859732072498,
      "grad_norm": 0.256059855222702,
      "learning_rate": 3.9278839317000155e-06,
      "log_odds_chosen": 0.4125687777996063,
      "log_odds_ratio": -0.513237714767456,
      "logits/chosen": 0.3524473309516907,
      "logits/rejected": -0.6497681736946106,
      "logps/chosen": -1.6583887338638306,
      "logps/rejected": -2.0051815509796143,
      "loss": 1.8159,
      "nll_loss": 1.7646095752716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16583886742591858,
      "rewards/margins": 0.03467928618192673,
      "rewards/rejected": -0.2005181461572647,
      "step": 226
    },
    {
      "epoch": 0.35776201733648544,
      "grad_norm": 0.2596084177494049,
      "learning_rate": 3.926413804149314e-06,
      "log_odds_chosen": 0.1289360225200653,
      "log_odds_ratio": -0.6390258073806763,
      "logits/chosen": 0.33115288615226746,
      "logits/rejected": -0.6374678611755371,
      "logps/chosen": -1.7433453798294067,
      "logps/rejected": -1.851163625717163,
      "loss": 1.899,
      "nll_loss": 1.8351025581359863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1743345409631729,
      "rewards/margins": 0.010781830176711082,
      "rewards/rejected": -0.18511638045310974,
      "step": 227
    },
    {
      "epoch": 0.35933806146572106,
      "grad_norm": 0.26425355672836304,
      "learning_rate": 3.924929123462447e-06,
      "log_odds_chosen": 0.2934200167655945,
      "log_odds_ratio": -0.5636212229728699,
      "logits/chosen": 0.387963205575943,
      "logits/rejected": -0.7065924406051636,
      "logps/chosen": -1.7703977823257446,
      "logps/rejected": -2.0198216438293457,
      "loss": 1.9186,
      "nll_loss": 1.8622859716415405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17703978717327118,
      "rewards/margins": 0.024942385032773018,
      "rewards/rejected": -0.20198215544223785,
      "step": 228
    },
    {
      "epoch": 0.3609141055949567,
      "grad_norm": 0.27313873171806335,
      "learning_rate": 3.923429900855468e-06,
      "log_odds_chosen": 0.21642562747001648,
      "log_odds_ratio": -0.5981402397155762,
      "logits/chosen": 0.3475606143474579,
      "logits/rejected": -0.4283681809902191,
      "logps/chosen": -1.8448374271392822,
      "logps/rejected": -2.0277938842773438,
      "loss": 1.9955,
      "nll_loss": 1.9356396198272705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18448375165462494,
      "rewards/margins": 0.01829562336206436,
      "rewards/rejected": -0.2027793526649475,
      "step": 229
    },
    {
      "epoch": 0.3624901497241923,
      "grad_norm": 0.24974492192268372,
      "learning_rate": 3.921916147654287e-06,
      "log_odds_chosen": 0.4641319811344147,
      "log_odds_ratio": -0.4937167465686798,
      "logits/chosen": 0.2954743504524231,
      "logits/rejected": -0.7521487474441528,
      "logps/chosen": -1.6764249801635742,
      "logps/rejected": -2.066699743270874,
      "loss": 1.8146,
      "nll_loss": 1.7652264833450317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1676425188779831,
      "rewards/margins": 0.03902747482061386,
      "rewards/rejected": -0.20666998624801636,
      "step": 230
    },
    {
      "epoch": 0.3640661938534279,
      "grad_norm": 0.2573936879634857,
      "learning_rate": 3.920387875294588e-06,
      "log_odds_chosen": 0.30775612592697144,
      "log_odds_ratio": -0.5542665719985962,
      "logits/chosen": 0.32833412289619446,
      "logits/rejected": -0.5938704609870911,
      "logps/chosen": -1.6527678966522217,
      "logps/rejected": -1.90794038772583,
      "loss": 1.8093,
      "nll_loss": 1.753854751586914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16527679562568665,
      "rewards/margins": 0.02551724575459957,
      "rewards/rejected": -0.19079405069351196,
      "step": 231
    },
    {
      "epoch": 0.3656422379826635,
      "grad_norm": 0.25298821926116943,
      "learning_rate": 3.918845095321737e-06,
      "log_odds_chosen": 0.28933507204055786,
      "log_odds_ratio": -0.5692440271377563,
      "logits/chosen": 0.2770174443721771,
      "logits/rejected": -0.8114198446273804,
      "logps/chosen": -1.6806954145431519,
      "logps/rejected": -1.9158856868743896,
      "loss": 1.829,
      "nll_loss": 1.7720324993133545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16806954145431519,
      "rewards/margins": 0.023519039154052734,
      "rewards/rejected": -0.19158856570720673,
      "step": 232
    },
    {
      "epoch": 0.3672182821118991,
      "grad_norm": 0.2757298946380615,
      "learning_rate": 3.9172878193907004e-06,
      "log_odds_chosen": 0.32267412543296814,
      "log_odds_ratio": -0.5543497204780579,
      "logits/chosen": 0.23460762202739716,
      "logits/rejected": -0.6888258457183838,
      "logps/chosen": -1.7551041841506958,
      "logps/rejected": -2.0310540199279785,
      "loss": 1.9168,
      "nll_loss": 1.8614091873168945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17551042139530182,
      "rewards/margins": 0.027594992890954018,
      "rewards/rejected": -0.20310541987419128,
      "step": 233
    },
    {
      "epoch": 0.36879432624113473,
      "grad_norm": 0.25875324010849,
      "learning_rate": 3.915716059265955e-06,
      "log_odds_chosen": 0.34329843521118164,
      "log_odds_ratio": -0.5443562269210815,
      "logits/chosen": 0.3021478056907654,
      "logits/rejected": -0.7619258761405945,
      "logps/chosen": -1.7081949710845947,
      "logps/rejected": -1.9994511604309082,
      "loss": 1.8579,
      "nll_loss": 1.8034164905548096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.170819491147995,
      "rewards/margins": 0.029125628992915154,
      "rewards/rejected": -0.1999451220035553,
      "step": 234
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 0.23391850292682648,
      "learning_rate": 3.9141298268213966e-06,
      "log_odds_chosen": 0.4332599639892578,
      "log_odds_ratio": -0.5040322542190552,
      "logits/chosen": 0.3331410884857178,
      "logits/rejected": -0.6711893677711487,
      "logps/chosen": -1.6272982358932495,
      "logps/rejected": -1.991321325302124,
      "loss": 1.7828,
      "nll_loss": 1.7323882579803467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16272982954978943,
      "rewards/margins": 0.03640230745077133,
      "rewards/rejected": -0.19913212954998016,
      "step": 235
    },
    {
      "epoch": 0.37194641449960597,
      "grad_norm": 0.28272461891174316,
      "learning_rate": 3.912529134040255e-06,
      "log_odds_chosen": 0.2862742841243744,
      "log_odds_ratio": -0.5635257959365845,
      "logits/chosen": 0.3295111358165741,
      "logits/rejected": -0.5666621327400208,
      "logps/chosen": -1.7123397588729858,
      "logps/rejected": -1.9539713859558105,
      "loss": 1.874,
      "nll_loss": 1.8176579475402832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17123398184776306,
      "rewards/margins": 0.02416316047310829,
      "rewards/rejected": -0.19539715349674225,
      "step": 236
    },
    {
      "epoch": 0.3735224586288416,
      "grad_norm": 0.27505528926849365,
      "learning_rate": 3.910913993014998e-06,
      "log_odds_chosen": 0.45064306259155273,
      "log_odds_ratio": -0.4959080219268799,
      "logits/chosen": 0.41635861992836,
      "logits/rejected": -0.7518233060836792,
      "logps/chosen": -1.7062500715255737,
      "logps/rejected": -2.0893027782440186,
      "loss": 1.854,
      "nll_loss": 1.8043806552886963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1706250011920929,
      "rewards/margins": 0.03830527514219284,
      "rewards/rejected": -0.20893028378486633,
      "step": 237
    },
    {
      "epoch": 0.3750985027580772,
      "grad_norm": 0.26204821467399597,
      "learning_rate": 3.909284415947246e-06,
      "log_odds_chosen": 0.42578941583633423,
      "log_odds_ratio": -0.5093416571617126,
      "logits/chosen": 0.37128570675849915,
      "logits/rejected": -0.8889274597167969,
      "logps/chosen": -1.7555906772613525,
      "logps/rejected": -2.1172666549682617,
      "loss": 1.8899,
      "nll_loss": 1.8389294147491455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17555907368659973,
      "rewards/margins": 0.036167606711387634,
      "rewards/rejected": -0.21172668039798737,
      "step": 238
    },
    {
      "epoch": 0.37667454688731283,
      "grad_norm": 0.2497677356004715,
      "learning_rate": 3.907640415147674e-06,
      "log_odds_chosen": 0.47326362133026123,
      "log_odds_ratio": -0.4884757101535797,
      "logits/chosen": 0.25646257400512695,
      "logits/rejected": -0.9685801863670349,
      "logps/chosen": -1.6502771377563477,
      "logps/rejected": -2.047640800476074,
      "loss": 1.8045,
      "nll_loss": 1.75564706325531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16502772271633148,
      "rewards/margins": 0.03973636403679848,
      "rewards/rejected": -0.20476408302783966,
      "step": 239
    },
    {
      "epoch": 0.37825059101654845,
      "grad_norm": 0.2736426293849945,
      "learning_rate": 3.905982003035924e-06,
      "log_odds_chosen": 0.38030922412872314,
      "log_odds_ratio": -0.5255998373031616,
      "logits/chosen": 0.28299060463905334,
      "logits/rejected": -0.8722066283226013,
      "logps/chosen": -1.7557415962219238,
      "logps/rejected": -2.0799028873443604,
      "loss": 1.893,
      "nll_loss": 1.8404154777526855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1755741536617279,
      "rewards/margins": 0.03241611272096634,
      "rewards/rejected": -0.20799028873443604,
      "step": 240
    },
    {
      "epoch": 0.3798266351457841,
      "grad_norm": 0.2504989504814148,
      "learning_rate": 3.904309192140506e-06,
      "log_odds_chosen": 0.2628810703754425,
      "log_odds_ratio": -0.5786678194999695,
      "logits/chosen": 0.300167053937912,
      "logits/rejected": -0.4999735355377197,
      "logps/chosen": -1.6737117767333984,
      "logps/rejected": -1.8940290212631226,
      "loss": 1.8283,
      "nll_loss": 1.7704728841781616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16737116873264313,
      "rewards/margins": 0.022031733766198158,
      "rewards/rejected": -0.18940290808677673,
      "step": 241
    },
    {
      "epoch": 0.3814026792750197,
      "grad_norm": 0.2762889564037323,
      "learning_rate": 3.90262199509871e-06,
      "log_odds_chosen": 0.2762555181980133,
      "log_odds_ratio": -0.5677258968353271,
      "logits/chosen": 0.2766878604888916,
      "logits/rejected": -0.591003954410553,
      "logps/chosen": -1.704395055770874,
      "logps/rejected": -1.93495512008667,
      "loss": 1.8625,
      "nll_loss": 1.80568265914917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17043951153755188,
      "rewards/margins": 0.02305598370730877,
      "rewards/rejected": -0.1934954971075058,
      "step": 242
    },
    {
      "epoch": 0.3829787234042553,
      "grad_norm": 0.25611984729766846,
      "learning_rate": 3.900920424656501e-06,
      "log_odds_chosen": 0.41975897550582886,
      "log_odds_ratio": -0.5121146440505981,
      "logits/chosen": 0.2905770540237427,
      "logits/rejected": -0.7793818712234497,
      "logps/chosen": -1.61533784866333,
      "logps/rejected": -1.9650779962539673,
      "loss": 1.7628,
      "nll_loss": 1.7115776538848877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16153380274772644,
      "rewards/margins": 0.034974001348018646,
      "rewards/rejected": -0.1965077966451645,
      "step": 243
    },
    {
      "epoch": 0.38455476753349094,
      "grad_norm": 0.273844838142395,
      "learning_rate": 3.899204493668432e-06,
      "log_odds_chosen": 0.3490802049636841,
      "log_odds_ratio": -0.5446145534515381,
      "logits/chosen": 0.3348943293094635,
      "logits/rejected": -0.6913176774978638,
      "logps/chosen": -1.7777903079986572,
      "logps/rejected": -2.080195903778076,
      "loss": 1.916,
      "nll_loss": 1.861492395401001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17777903378009796,
      "rewards/margins": 0.030240550637245178,
      "rewards/rejected": -0.20801956951618195,
      "step": 244
    },
    {
      "epoch": 0.38613081166272656,
      "grad_norm": 0.2658459544181824,
      "learning_rate": 3.897474215097542e-06,
      "log_odds_chosen": 0.34318360686302185,
      "log_odds_ratio": -0.5412995219230652,
      "logits/chosen": 0.3551873564720154,
      "logits/rejected": -0.9330862760543823,
      "logps/chosen": -1.7187716960906982,
      "logps/rejected": -2.0109448432922363,
      "loss": 1.8774,
      "nll_loss": 1.8233082294464111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1718771755695343,
      "rewards/margins": 0.029217295348644257,
      "rewards/rejected": -0.20109446346759796,
      "step": 245
    },
    {
      "epoch": 0.3877068557919622,
      "grad_norm": 0.24852316081523895,
      "learning_rate": 3.8957296020152596e-06,
      "log_odds_chosen": 0.45998305082321167,
      "log_odds_ratio": -0.4943844676017761,
      "logits/chosen": 0.3586815297603607,
      "logits/rejected": -0.8210923671722412,
      "logps/chosen": -1.6819372177124023,
      "logps/rejected": -2.0723345279693604,
      "loss": 1.8356,
      "nll_loss": 1.786157488822937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1681937277317047,
      "rewards/margins": 0.039039719849824905,
      "rewards/rejected": -0.20723344385623932,
      "step": 246
    },
    {
      "epoch": 0.3892828999211978,
      "grad_norm": 0.28672662377357483,
      "learning_rate": 3.893970667601303e-06,
      "log_odds_chosen": 0.2993201017379761,
      "log_odds_ratio": -0.5586628913879395,
      "logits/chosen": 0.24899174273014069,
      "logits/rejected": -1.121972680091858,
      "logps/chosen": -1.7663580179214478,
      "logps/rejected": -2.020313024520874,
      "loss": 1.9069,
      "nll_loss": 1.8509916067123413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17663580179214478,
      "rewards/margins": 0.025395508855581284,
      "rewards/rejected": -0.20203131437301636,
      "step": 247
    },
    {
      "epoch": 0.3908589440504334,
      "grad_norm": 0.26399871706962585,
      "learning_rate": 3.892197425143581e-06,
      "log_odds_chosen": 0.46287333965301514,
      "log_odds_ratio": -0.49190735816955566,
      "logits/chosen": 0.30056482553482056,
      "logits/rejected": -0.6871494650840759,
      "logps/chosen": -1.6935536861419678,
      "logps/rejected": -2.085163116455078,
      "loss": 1.8295,
      "nll_loss": 1.7803454399108887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1693553626537323,
      "rewards/margins": 0.03916092962026596,
      "rewards/rejected": -0.20851629972457886,
      "step": 248
    },
    {
      "epoch": 0.39243498817966904,
      "grad_norm": 0.2822303771972656,
      "learning_rate": 3.890409888038094e-06,
      "log_odds_chosen": 0.3473043441772461,
      "log_odds_ratio": -0.5403538942337036,
      "logits/chosen": 0.332716166973114,
      "logits/rejected": -0.7040198445320129,
      "logps/chosen": -1.7375996112823486,
      "logps/rejected": -2.0354490280151367,
      "loss": 1.8869,
      "nll_loss": 1.8328940868377686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17375998198986053,
      "rewards/margins": 0.02978491596877575,
      "rewards/rejected": -0.20354489982128143,
      "step": 249
    },
    {
      "epoch": 0.39401103230890466,
      "grad_norm": 0.2671235203742981,
      "learning_rate": 3.888608069788831e-06,
      "log_odds_chosen": 0.3373025357723236,
      "log_odds_ratio": -0.5513738393783569,
      "logits/chosen": 0.2397560477256775,
      "logits/rejected": -0.565992534160614,
      "logps/chosen": -1.6891679763793945,
      "logps/rejected": -1.9773098230361938,
      "loss": 1.8412,
      "nll_loss": 1.7861071825027466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16891679167747498,
      "rewards/margins": 0.028814200311899185,
      "rewards/rejected": -0.19773098826408386,
      "step": 250
    },
    {
      "epoch": 0.3955870764381403,
      "grad_norm": 0.24620778858661652,
      "learning_rate": 3.8867919840076685e-06,
      "log_odds_chosen": 0.4103567898273468,
      "log_odds_ratio": -0.5125883221626282,
      "logits/chosen": 0.26780739426612854,
      "logits/rejected": -0.8729146122932434,
      "logps/chosen": -1.618111252784729,
      "logps/rejected": -1.9599648714065552,
      "loss": 1.7577,
      "nll_loss": 1.7063930034637451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16181112825870514,
      "rewards/margins": 0.03418537974357605,
      "rewards/rejected": -0.1959965080022812,
      "step": 251
    },
    {
      "epoch": 0.3971631205673759,
      "grad_norm": 0.2638973593711853,
      "learning_rate": 3.884961644414267e-06,
      "log_odds_chosen": 0.33575767278671265,
      "log_odds_ratio": -0.5410860776901245,
      "logits/chosen": 0.2888558804988861,
      "logits/rejected": -0.4153730273246765,
      "logps/chosen": -1.7437500953674316,
      "logps/rejected": -2.0283775329589844,
      "loss": 1.865,
      "nll_loss": 1.810841679573059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1743750125169754,
      "rewards/margins": 0.028462760150432587,
      "rewards/rejected": -0.2028377652168274,
      "step": 252
    },
    {
      "epoch": 0.3987391646966115,
      "grad_norm": 0.2482423335313797,
      "learning_rate": 3.883117064835967e-06,
      "log_odds_chosen": 0.38616234064102173,
      "log_odds_ratio": -0.5299303531646729,
      "logits/chosen": 0.2521415948867798,
      "logits/rejected": -0.7721424102783203,
      "logps/chosen": -1.6814165115356445,
      "logps/rejected": -2.0103795528411865,
      "loss": 1.8339,
      "nll_loss": 1.7809207439422607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1681416630744934,
      "rewards/margins": 0.03289632126688957,
      "rewards/rejected": -0.20103797316551208,
      "step": 253
    },
    {
      "epoch": 0.40031520882584715,
      "grad_norm": 0.2557568848133087,
      "learning_rate": 3.881258259207688e-06,
      "log_odds_chosen": 0.32933154702186584,
      "log_odds_ratio": -0.5472507476806641,
      "logits/chosen": 0.34836918115615845,
      "logits/rejected": -0.8260299563407898,
      "logps/chosen": -1.6516460180282593,
      "logps/rejected": -1.9261865615844727,
      "loss": 1.7824,
      "nll_loss": 1.727644920349121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16516461968421936,
      "rewards/margins": 0.027454044669866562,
      "rewards/rejected": -0.19261866807937622,
      "step": 254
    },
    {
      "epoch": 0.40189125295508277,
      "grad_norm": 0.2454916387796402,
      "learning_rate": 3.8793852415718165e-06,
      "log_odds_chosen": 0.41223955154418945,
      "log_odds_ratio": -0.5096971988677979,
      "logits/chosen": 0.26183220744132996,
      "logits/rejected": -1.1104363203048706,
      "logps/chosen": -1.5731486082077026,
      "logps/rejected": -1.911667823791504,
      "loss": 1.7079,
      "nll_loss": 1.656911015510559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15731485188007355,
      "rewards/margins": 0.03385192155838013,
      "rewards/rejected": -0.19116678833961487,
      "step": 255
    },
    {
      "epoch": 0.4034672970843184,
      "grad_norm": 0.24788597226142883,
      "learning_rate": 3.877498026078107e-06,
      "log_odds_chosen": 0.2555736005306244,
      "log_odds_ratio": -0.5793567299842834,
      "logits/chosen": 0.2708052396774292,
      "logits/rejected": -0.9535562992095947,
      "logps/chosen": -1.718558430671692,
      "logps/rejected": -1.936058759689331,
      "loss": 1.8553,
      "nll_loss": 1.7973601818084717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1718558371067047,
      "rewards/margins": 0.021750036627054214,
      "rewards/rejected": -0.19360588490962982,
      "step": 256
    },
    {
      "epoch": 0.40504334121355395,
      "grad_norm": 0.2589392364025116,
      "learning_rate": 3.875596626983573e-06,
      "log_odds_chosen": 0.469348669052124,
      "log_odds_ratio": -0.4969593286514282,
      "logits/chosen": 0.29150527715682983,
      "logits/rejected": -0.6516475677490234,
      "logps/chosen": -1.6384902000427246,
      "logps/rejected": -2.030492067337036,
      "loss": 1.7747,
      "nll_loss": 1.7249572277069092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16384902596473694,
      "rewards/margins": 0.039200179278850555,
      "rewards/rejected": -0.2030492126941681,
      "step": 257
    },
    {
      "epoch": 0.4066193853427896,
      "grad_norm": 0.2652013599872589,
      "learning_rate": 3.873681058652374e-06,
      "log_odds_chosen": 0.2649344503879547,
      "log_odds_ratio": -0.5769127607345581,
      "logits/chosen": 0.26984673738479614,
      "logits/rejected": -0.7516025304794312,
      "logps/chosen": -1.7576013803482056,
      "logps/rejected": -1.9836013317108154,
      "loss": 1.8938,
      "nll_loss": 1.836103081703186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1757601499557495,
      "rewards/margins": 0.02259998768568039,
      "rewards/rejected": -0.1983601450920105,
      "step": 258
    },
    {
      "epoch": 0.4081954294720252,
      "grad_norm": 0.275820255279541,
      "learning_rate": 3.871751335555715e-06,
      "log_odds_chosen": 0.26289334893226624,
      "log_odds_ratio": -0.5722215175628662,
      "logits/chosen": 0.25413140654563904,
      "logits/rejected": -0.9038801193237305,
      "logps/chosen": -1.6938437223434448,
      "logps/rejected": -1.9135998487472534,
      "loss": 1.8231,
      "nll_loss": 1.7658631801605225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16938436031341553,
      "rewards/margins": 0.021975621581077576,
      "rewards/rejected": -0.1913599818944931,
      "step": 259
    },
    {
      "epoch": 0.4097714736012608,
      "grad_norm": 0.24297621846199036,
      "learning_rate": 3.869807472271731e-06,
      "log_odds_chosen": 0.33671316504478455,
      "log_odds_ratio": -0.5421686172485352,
      "logits/chosen": 0.2816615402698517,
      "logits/rejected": -0.782688558101654,
      "logps/chosen": -1.6605300903320312,
      "logps/rejected": -1.943679928779602,
      "loss": 1.8234,
      "nll_loss": 1.7691376209259033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16605301201343536,
      "rewards/margins": 0.02831498347222805,
      "rewards/rejected": -0.19436800479888916,
      "step": 260
    },
    {
      "epoch": 0.41134751773049644,
      "grad_norm": 0.24730239808559418,
      "learning_rate": 3.8678494834853826e-06,
      "log_odds_chosen": 0.3380075693130493,
      "log_odds_ratio": -0.5455616116523743,
      "logits/chosen": 0.2523096799850464,
      "logits/rejected": -0.7331975698471069,
      "logps/chosen": -1.6633615493774414,
      "logps/rejected": -1.9441428184509277,
      "loss": 1.7975,
      "nll_loss": 1.7429437637329102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16633614897727966,
      "rewards/margins": 0.02807813137769699,
      "rewards/rejected": -0.19441428780555725,
      "step": 261
    },
    {
      "epoch": 0.41292356185973206,
      "grad_norm": 0.27988502383232117,
      "learning_rate": 3.865877383988339e-06,
      "log_odds_chosen": 0.4063892066478729,
      "log_odds_ratio": -0.5170100331306458,
      "logits/chosen": 0.25469520688056946,
      "logits/rejected": -0.7429967522621155,
      "logps/chosen": -1.6950594186782837,
      "logps/rejected": -2.035081624984741,
      "loss": 1.8501,
      "nll_loss": 1.7983577251434326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16950593888759613,
      "rewards/margins": 0.03400222212076187,
      "rewards/rejected": -0.2035081684589386,
      "step": 262
    },
    {
      "epoch": 0.4144996059889677,
      "grad_norm": 0.25723162293434143,
      "learning_rate": 3.863891188678869e-06,
      "log_odds_chosen": 0.14911779761314392,
      "log_odds_ratio": -0.631981372833252,
      "logits/chosen": 0.2207110971212387,
      "logits/rejected": -0.883880615234375,
      "logps/chosen": -1.6087576150894165,
      "logps/rejected": -1.7343672513961792,
      "loss": 1.7568,
      "nll_loss": 1.6936231851577759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16087576746940613,
      "rewards/margins": 0.012560954317450523,
      "rewards/rejected": -0.1734367161989212,
      "step": 263
    },
    {
      "epoch": 0.4160756501182033,
      "grad_norm": 0.25149017572402954,
      "learning_rate": 3.8618909125617305e-06,
      "log_odds_chosen": 0.5187549591064453,
      "log_odds_ratio": -0.47057828307151794,
      "logits/chosen": 0.2168709635734558,
      "logits/rejected": -1.114577293395996,
      "logps/chosen": -1.5999822616577148,
      "logps/rejected": -2.0340919494628906,
      "loss": 1.7533,
      "nll_loss": 1.7062718868255615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15999823808670044,
      "rewards/margins": 0.04341099411249161,
      "rewards/rejected": -0.20340920984745026,
      "step": 264
    },
    {
      "epoch": 0.4176516942474389,
      "grad_norm": 0.2453727126121521,
      "learning_rate": 3.859876570748054e-06,
      "log_odds_chosen": 0.3410520851612091,
      "log_odds_ratio": -0.5392724871635437,
      "logits/chosen": 0.2857251465320587,
      "logits/rejected": -0.38999515771865845,
      "logps/chosen": -1.6569011211395264,
      "logps/rejected": -1.9416050910949707,
      "loss": 1.8086,
      "nll_loss": 1.7547087669372559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1656901240348816,
      "rewards/margins": 0.028470391407608986,
      "rewards/rejected": -0.19416050612926483,
      "step": 265
    },
    {
      "epoch": 0.41922773837667454,
      "grad_norm": 0.24315941333770752,
      "learning_rate": 3.857848178455231e-06,
      "log_odds_chosen": 0.10583052039146423,
      "log_odds_ratio": -0.6480942964553833,
      "logits/chosen": 0.2967842221260071,
      "logits/rejected": -1.1144427061080933,
      "logps/chosen": -1.6708816289901733,
      "logps/rejected": -1.7641648054122925,
      "loss": 1.8376,
      "nll_loss": 1.7727646827697754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16708815097808838,
      "rewards/margins": 0.009328329935669899,
      "rewards/rejected": -0.17641650140285492,
      "step": 266
    },
    {
      "epoch": 0.42080378250591016,
      "grad_norm": 0.27732643485069275,
      "learning_rate": 3.855805751006794e-06,
      "log_odds_chosen": 0.4604467749595642,
      "log_odds_ratio": -0.498542845249176,
      "logits/chosen": 0.2691567838191986,
      "logits/rejected": -1.1059002876281738,
      "logps/chosen": -1.6944466829299927,
      "logps/rejected": -2.0833187103271484,
      "loss": 1.8215,
      "nll_loss": 1.7716130018234253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16944465041160583,
      "rewards/margins": 0.03888721019029617,
      "rewards/rejected": -0.2083318680524826,
      "step": 267
    },
    {
      "epoch": 0.4223798266351458,
      "grad_norm": 0.23589597642421722,
      "learning_rate": 3.853749303832308e-06,
      "log_odds_chosen": 0.33512479066848755,
      "log_odds_ratio": -0.5445858836174011,
      "logits/chosen": 0.24153344333171844,
      "logits/rejected": -0.648094117641449,
      "logps/chosen": -1.6203473806381226,
      "logps/rejected": -1.897982120513916,
      "loss": 1.761,
      "nll_loss": 1.706520438194275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16203473508358002,
      "rewards/margins": 0.027763469144701958,
      "rewards/rejected": -0.18979820609092712,
      "step": 268
    },
    {
      "epoch": 0.4239558707643814,
      "grad_norm": 0.2540602385997772,
      "learning_rate": 3.8516788524672495e-06,
      "log_odds_chosen": 0.20173487067222595,
      "log_odds_ratio": -0.6037977933883667,
      "logits/chosen": 0.27147728204727173,
      "logits/rejected": -0.9890564680099487,
      "logps/chosen": -1.69952392578125,
      "logps/rejected": -1.8702551126480103,
      "loss": 1.8336,
      "nll_loss": 1.773256540298462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.169952392578125,
      "rewards/margins": 0.01707313396036625,
      "rewards/rejected": -0.1870255172252655,
      "step": 269
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 0.24478159844875336,
      "learning_rate": 3.849594412552889e-06,
      "log_odds_chosen": 0.33746102452278137,
      "log_odds_ratio": -0.5440058708190918,
      "logits/chosen": 0.285639226436615,
      "logits/rejected": -0.7275075912475586,
      "logps/chosen": -1.6040332317352295,
      "logps/rejected": -1.8771222829818726,
      "loss": 1.7444,
      "nll_loss": 1.689996361732483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1604033261537552,
      "rewards/margins": 0.027308890596032143,
      "rewards/rejected": -0.18771222233772278,
      "step": 270
    },
    {
      "epoch": 0.42710795902285265,
      "grad_norm": 0.25568917393684387,
      "learning_rate": 3.847495999836175e-06,
      "log_odds_chosen": 0.3471507430076599,
      "log_odds_ratio": -0.5386791229248047,
      "logits/chosen": 0.3089812994003296,
      "logits/rejected": -0.9828994870185852,
      "logps/chosen": -1.6793549060821533,
      "logps/rejected": -1.9712432622909546,
      "loss": 1.8059,
      "nll_loss": 1.7519892454147339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16793550550937653,
      "rewards/margins": 0.02918882854282856,
      "rewards/rejected": -0.19712434709072113,
      "step": 271
    },
    {
      "epoch": 0.42868400315208827,
      "grad_norm": 0.254783034324646,
      "learning_rate": 3.845383630169613e-06,
      "log_odds_chosen": 0.31106436252593994,
      "log_odds_ratio": -0.5561890602111816,
      "logits/chosen": 0.16303634643554688,
      "logits/rejected": -1.0483474731445312,
      "logps/chosen": -1.5915002822875977,
      "logps/rejected": -1.8480899333953857,
      "loss": 1.7205,
      "nll_loss": 1.6648414134979248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15915003418922424,
      "rewards/margins": 0.02565896324813366,
      "rewards/rejected": -0.18480899930000305,
      "step": 272
    },
    {
      "epoch": 0.4302600472813239,
      "grad_norm": 0.25443676114082336,
      "learning_rate": 3.843257319511147e-06,
      "log_odds_chosen": 0.54793381690979,
      "log_odds_ratio": -0.4633035957813263,
      "logits/chosen": 0.21883374452590942,
      "logits/rejected": -1.1158220767974854,
      "logps/chosen": -1.5903308391571045,
      "logps/rejected": -2.0446715354919434,
      "loss": 1.7182,
      "nll_loss": 1.6718711853027344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15903308987617493,
      "rewards/margins": 0.04543406888842583,
      "rewards/rejected": -0.20446714758872986,
      "step": 273
    },
    {
      "epoch": 0.4318360914105595,
      "grad_norm": 0.25186702609062195,
      "learning_rate": 3.841117083924039e-06,
      "log_odds_chosen": 0.3146308958530426,
      "log_odds_ratio": -0.5538010597229004,
      "logits/chosen": 0.20665690302848816,
      "logits/rejected": -1.150301218032837,
      "logps/chosen": -1.6276448965072632,
      "logps/rejected": -1.8898921012878418,
      "loss": 1.7612,
      "nll_loss": 1.705790400505066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16276448965072632,
      "rewards/margins": 0.026224708184599876,
      "rewards/rejected": -0.18898919224739075,
      "step": 274
    },
    {
      "epoch": 0.43341213553979513,
      "grad_norm": 0.2522598206996918,
      "learning_rate": 3.838962939576746e-06,
      "log_odds_chosen": 0.41350072622299194,
      "log_odds_ratio": -0.5153959393501282,
      "logits/chosen": 0.17553453147411346,
      "logits/rejected": -0.9353335499763489,
      "logps/chosen": -1.5784237384796143,
      "logps/rejected": -1.9216094017028809,
      "loss": 1.7319,
      "nll_loss": 1.6803406476974487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15784236788749695,
      "rewards/margins": 0.03431856259703636,
      "rewards/rejected": -0.1921609491109848,
      "step": 275
    },
    {
      "epoch": 0.43498817966903075,
      "grad_norm": 0.2677028477191925,
      "learning_rate": 3.8367949027427985e-06,
      "log_odds_chosen": 0.2886348366737366,
      "log_odds_ratio": -0.5637444257736206,
      "logits/chosen": 0.23681315779685974,
      "logits/rejected": -0.8251351714134216,
      "logps/chosen": -1.6966907978057861,
      "logps/rejected": -1.9407154321670532,
      "loss": 1.8551,
      "nll_loss": 1.798759937286377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16966909170150757,
      "rewards/margins": 0.024402447044849396,
      "rewards/rejected": -0.19407154619693756,
      "step": 276
    },
    {
      "epoch": 0.43656422379826637,
      "grad_norm": 0.27280256152153015,
      "learning_rate": 3.834612989800681e-06,
      "log_odds_chosen": 0.5323548316955566,
      "log_odds_ratio": -0.46610027551651,
      "logits/chosen": 0.25051310658454895,
      "logits/rejected": -1.3698272705078125,
      "logps/chosen": -1.680455207824707,
      "logps/rejected": -2.1312544345855713,
      "loss": 1.7988,
      "nll_loss": 1.7521436214447021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1680455207824707,
      "rewards/margins": 0.04507990926504135,
      "rewards/rejected": -0.21312543749809265,
      "step": 277
    },
    {
      "epoch": 0.438140267927502,
      "grad_norm": 0.2634829878807068,
      "learning_rate": 3.832417217233703e-06,
      "log_odds_chosen": 0.29642635583877563,
      "log_odds_ratio": -0.5586500763893127,
      "logits/chosen": 0.21414227783679962,
      "logits/rejected": -0.7982608675956726,
      "logps/chosen": -1.6701531410217285,
      "logps/rejected": -1.9189419746398926,
      "loss": 1.8078,
      "nll_loss": 1.751927375793457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16701531410217285,
      "rewards/margins": 0.024878891184926033,
      "rewards/rejected": -0.19189421832561493,
      "step": 278
    },
    {
      "epoch": 0.4397163120567376,
      "grad_norm": 0.2531651556491852,
      "learning_rate": 3.8302076016298775e-06,
      "log_odds_chosen": 0.4342229664325714,
      "log_odds_ratio": -0.5029668211936951,
      "logits/chosen": 0.1577448695898056,
      "logits/rejected": -1.0718286037445068,
      "logps/chosen": -1.6032099723815918,
      "logps/rejected": -1.9638495445251465,
      "loss": 1.7458,
      "nll_loss": 1.695469617843628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16032099723815918,
      "rewards/margins": 0.03606396168470383,
      "rewards/rejected": -0.1963849514722824,
      "step": 279
    },
    {
      "epoch": 0.44129235618597323,
      "grad_norm": 0.2585320472717285,
      "learning_rate": 3.827984159681796e-06,
      "log_odds_chosen": 0.33446642756462097,
      "log_odds_ratio": -0.5452247262001038,
      "logits/chosen": 0.14251501858234406,
      "logits/rejected": -0.8923399448394775,
      "logps/chosen": -1.654819130897522,
      "logps/rejected": -1.932909369468689,
      "loss": 1.7837,
      "nll_loss": 1.7291784286499023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16548191010951996,
      "rewards/margins": 0.027809027582406998,
      "rewards/rejected": -0.19329093396663666,
      "step": 280
    },
    {
      "epoch": 0.4428684003152088,
      "grad_norm": 0.2518828809261322,
      "learning_rate": 3.825746908186498e-06,
      "log_odds_chosen": 0.38528937101364136,
      "log_odds_ratio": -0.5269155502319336,
      "logits/chosen": 0.2786559760570526,
      "logits/rejected": -1.1318894624710083,
      "logps/chosen": -1.6995317935943604,
      "logps/rejected": -2.024996519088745,
      "loss": 1.8116,
      "nll_loss": 1.7589478492736816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16995318233966827,
      "rewards/margins": 0.03254646807909012,
      "rewards/rejected": -0.2024996429681778,
      "step": 281
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.26809749007225037,
      "learning_rate": 3.823495864045352e-06,
      "log_odds_chosen": 0.4371758699417114,
      "log_odds_ratio": -0.5007396936416626,
      "logits/chosen": 0.2518642842769623,
      "logits/rejected": -0.5491166114807129,
      "logps/chosen": -1.6472457647323608,
      "logps/rejected": -2.0112357139587402,
      "loss": 1.7796,
      "nll_loss": 1.7295327186584473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16472457349300385,
      "rewards/margins": 0.036398995667696,
      "rewards/rejected": -0.20112356543540955,
      "step": 282
    },
    {
      "epoch": 0.44602048857368004,
      "grad_norm": 0.2323625683784485,
      "learning_rate": 3.8212310442639205e-06,
      "log_odds_chosen": 0.32891276478767395,
      "log_odds_ratio": -0.5458555221557617,
      "logits/chosen": 0.24299903213977814,
      "logits/rejected": -0.7525103092193604,
      "logps/chosen": -1.592462182044983,
      "logps/rejected": -1.8642407655715942,
      "loss": 1.738,
      "nll_loss": 1.6833724975585938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15924622118473053,
      "rewards/margins": 0.027177870273590088,
      "rewards/rejected": -0.18642409145832062,
      "step": 283
    },
    {
      "epoch": 0.44759653270291566,
      "grad_norm": 0.2419874519109726,
      "learning_rate": 3.8189524659518355e-06,
      "log_odds_chosen": 0.39278745651245117,
      "log_odds_ratio": -0.5193477869033813,
      "logits/chosen": 0.31300458312034607,
      "logits/rejected": -0.9649338722229004,
      "logps/chosen": -1.664646029472351,
      "logps/rejected": -1.992173194885254,
      "loss": 1.7913,
      "nll_loss": 1.7394092082977295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16646461188793182,
      "rewards/margins": 0.03275272995233536,
      "rewards/rejected": -0.19921733438968658,
      "step": 284
    },
    {
      "epoch": 0.4491725768321513,
      "grad_norm": 0.2631385326385498,
      "learning_rate": 3.816660146322667e-06,
      "log_odds_chosen": 0.3710939884185791,
      "log_odds_ratio": -0.5282797813415527,
      "logits/chosen": 0.22600415349006653,
      "logits/rejected": -0.7380872964859009,
      "logps/chosen": -1.5571084022521973,
      "logps/rejected": -1.8607244491577148,
      "loss": 1.6912,
      "nll_loss": 1.6383720636367798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557108461856842,
      "rewards/margins": 0.030361609533429146,
      "rewards/rejected": -0.1860724538564682,
      "step": 285
    },
    {
      "epoch": 0.4507486209613869,
      "grad_norm": 0.2479519248008728,
      "learning_rate": 3.814354102693797e-06,
      "log_odds_chosen": 0.3791089355945587,
      "log_odds_ratio": -0.5239448547363281,
      "logits/chosen": 0.2559822201728821,
      "logits/rejected": -1.0194425582885742,
      "logps/chosen": -1.6287118196487427,
      "logps/rejected": -1.9415102005004883,
      "loss": 1.7601,
      "nll_loss": 1.7077348232269287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16287118196487427,
      "rewards/margins": 0.03127983957529068,
      "rewards/rejected": -0.19415102899074554,
      "step": 286
    },
    {
      "epoch": 0.4523246650906225,
      "grad_norm": 0.2415776550769806,
      "learning_rate": 3.8120343524862814e-06,
      "log_odds_chosen": 0.3282337784767151,
      "log_odds_ratio": -0.5441159009933472,
      "logits/chosen": 0.22718586027622223,
      "logits/rejected": -0.8667283058166504,
      "logps/chosen": -1.669337272644043,
      "logps/rejected": -1.9410953521728516,
      "loss": 1.7842,
      "nll_loss": 1.7298314571380615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16693371534347534,
      "rewards/margins": 0.027175823226571083,
      "rewards/rejected": -0.19410954415798187,
      "step": 287
    },
    {
      "epoch": 0.45390070921985815,
      "grad_norm": 0.23657608032226562,
      "learning_rate": 3.809700913224726e-06,
      "log_odds_chosen": 0.37087053060531616,
      "log_odds_ratio": -0.5310470461845398,
      "logits/chosen": 0.19090989232063293,
      "logits/rejected": -1.3478600978851318,
      "logps/chosen": -1.5922802686691284,
      "logps/rejected": -1.897598147392273,
      "loss": 1.7178,
      "nll_loss": 1.6647306680679321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15922804176807404,
      "rewards/margins": 0.030531786382198334,
      "rewards/rejected": -0.18975980579853058,
      "step": 288
    },
    {
      "epoch": 0.45547675334909377,
      "grad_norm": 0.24203184247016907,
      "learning_rate": 3.8073538025371494e-06,
      "log_odds_chosen": 0.46555230021476746,
      "log_odds_ratio": -0.49043652415275574,
      "logits/chosen": 0.30645501613616943,
      "logits/rejected": -1.1749683618545532,
      "logps/chosen": -1.568337321281433,
      "logps/rejected": -1.9523917436599731,
      "loss": 1.7066,
      "nll_loss": 1.6575335264205933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1568337380886078,
      "rewards/margins": 0.03840543329715729,
      "rewards/rejected": -0.19523918628692627,
      "step": 289
    },
    {
      "epoch": 0.4570527974783294,
      "grad_norm": 0.2672315537929535,
      "learning_rate": 3.804993038154852e-06,
      "log_odds_chosen": 0.34897854924201965,
      "log_odds_ratio": -0.541498601436615,
      "logits/chosen": 0.2300131767988205,
      "logits/rejected": -1.0130763053894043,
      "logps/chosen": -1.6003525257110596,
      "logps/rejected": -1.8910918235778809,
      "loss": 1.7494,
      "nll_loss": 1.6952693462371826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16003525257110596,
      "rewards/margins": 0.02907394804060459,
      "rewards/rejected": -0.189109206199646,
      "step": 290
    },
    {
      "epoch": 0.458628841607565,
      "grad_norm": 0.2524731755256653,
      "learning_rate": 3.8026186379122816e-06,
      "log_odds_chosen": 0.2566876709461212,
      "log_odds_ratio": -0.5760392546653748,
      "logits/chosen": 0.19924253225326538,
      "logits/rejected": -1.0389641523361206,
      "logps/chosen": -1.6778925657272339,
      "logps/rejected": -1.891493797302246,
      "loss": 1.8,
      "nll_loss": 1.7424429655075073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1677892506122589,
      "rewards/margins": 0.021360116079449654,
      "rewards/rejected": -0.1891493797302246,
      "step": 291
    },
    {
      "epoch": 0.46020488573680063,
      "grad_norm": 0.24778525531291962,
      "learning_rate": 3.8002306197468983e-06,
      "log_odds_chosen": 0.17079176008701324,
      "log_odds_ratio": -0.613702654838562,
      "logits/chosen": 0.19200792908668518,
      "logits/rejected": -1.0982946157455444,
      "logps/chosen": -1.6556106805801392,
      "logps/rejected": -1.795432686805725,
      "loss": 1.7958,
      "nll_loss": 1.734451174736023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16556105017662048,
      "rewards/margins": 0.01398220844566822,
      "rewards/rejected": -0.17954328656196594,
      "step": 292
    },
    {
      "epoch": 0.46178092986603625,
      "grad_norm": 0.22741490602493286,
      "learning_rate": 3.7978290016990367e-06,
      "log_odds_chosen": 0.48492002487182617,
      "log_odds_ratio": -0.4839008152484894,
      "logits/chosen": 0.17027397453784943,
      "logits/rejected": -1.0292561054229736,
      "logps/chosen": -1.6551365852355957,
      "logps/rejected": -2.065377712249756,
      "loss": 1.7785,
      "nll_loss": 1.7300668954849243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16551366448402405,
      "rewards/margins": 0.0410241037607193,
      "rewards/rejected": -0.20653776824474335,
      "step": 293
    },
    {
      "epoch": 0.46335697399527187,
      "grad_norm": 0.2619555592536926,
      "learning_rate": 3.795413801911776e-06,
      "log_odds_chosen": 0.54489666223526,
      "log_odds_ratio": -0.46421873569488525,
      "logits/chosen": 0.15529164671897888,
      "logits/rejected": -1.1627665758132935,
      "logps/chosen": -1.5853267908096313,
      "logps/rejected": -2.041583299636841,
      "loss": 1.7285,
      "nll_loss": 1.6820851564407349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15853269398212433,
      "rewards/margins": 0.045625634491443634,
      "rewards/rejected": -0.20415832102298737,
      "step": 294
    },
    {
      "epoch": 0.4649330181245075,
      "grad_norm": 0.2782156765460968,
      "learning_rate": 3.7929850386307965e-06,
      "log_odds_chosen": 0.323818564414978,
      "log_odds_ratio": -0.5496729612350464,
      "logits/chosen": 0.2609451711177826,
      "logits/rejected": -0.9133318662643433,
      "logps/chosen": -1.7928597927093506,
      "logps/rejected": -2.07020902633667,
      "loss": 1.9066,
      "nll_loss": 1.851614236831665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17928597331047058,
      "rewards/margins": 0.027734942734241486,
      "rewards/rejected": -0.20702092349529266,
      "step": 295
    },
    {
      "epoch": 0.4665090622537431,
      "grad_norm": 0.2524731755256653,
      "learning_rate": 3.790542730204245e-06,
      "log_odds_chosen": 0.38338762521743774,
      "log_odds_ratio": -0.5222858190536499,
      "logits/chosen": 0.2131703644990921,
      "logits/rejected": -1.1159170866012573,
      "logps/chosen": -1.7220946550369263,
      "logps/rejected": -2.048020124435425,
      "loss": 1.8309,
      "nll_loss": 1.7786983251571655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1722094714641571,
      "rewards/margins": 0.03259255737066269,
      "rewards/rejected": -0.2048020213842392,
      "step": 296
    },
    {
      "epoch": 0.46808510638297873,
      "grad_norm": 0.2552836835384369,
      "learning_rate": 3.7880868950825935e-06,
      "log_odds_chosen": 0.3997831344604492,
      "log_odds_ratio": -0.513701856136322,
      "logits/chosen": 0.1777328997850418,
      "logits/rejected": -1.2488939762115479,
      "logps/chosen": -1.6790422201156616,
      "logps/rejected": -2.0147104263305664,
      "loss": 1.7804,
      "nll_loss": 1.7290263175964355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16790422797203064,
      "rewards/margins": 0.03356681764125824,
      "rewards/rejected": -0.20147106051445007,
      "step": 297
    },
    {
      "epoch": 0.46966115051221435,
      "grad_norm": 0.2645845115184784,
      "learning_rate": 3.7856175518185058e-06,
      "log_odds_chosen": 0.39919474720954895,
      "log_odds_ratio": -0.519536554813385,
      "logits/chosen": 0.04900103062391281,
      "logits/rejected": -1.2728874683380127,
      "logps/chosen": -1.6557228565216064,
      "logps/rejected": -1.9891810417175293,
      "loss": 1.7843,
      "nll_loss": 1.7323014736175537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16557228565216064,
      "rewards/margins": 0.033345822244882584,
      "rewards/rejected": -0.19891811907291412,
      "step": 298
    },
    {
      "epoch": 0.47123719464145,
      "grad_norm": 0.25159481167793274,
      "learning_rate": 3.7831347190666883e-06,
      "log_odds_chosen": 0.4714580476284027,
      "log_odds_ratio": -0.49071937799453735,
      "logits/chosen": 0.12509863078594208,
      "logits/rejected": -1.183892846107483,
      "logps/chosen": -1.6513078212738037,
      "logps/rejected": -2.048064708709717,
      "loss": 1.7823,
      "nll_loss": 1.7332583665847778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16513077914714813,
      "rewards/margins": 0.03967569023370743,
      "rewards/rejected": -0.20480646193027496,
      "step": 299
    },
    {
      "epoch": 0.4728132387706856,
      "grad_norm": 0.25482556223869324,
      "learning_rate": 3.780638415583759e-06,
      "log_odds_chosen": 0.27312323451042175,
      "log_odds_ratio": -0.5755019187927246,
      "logits/chosen": 0.24975763261318207,
      "logits/rejected": -0.9788644909858704,
      "logps/chosen": -1.6345561742782593,
      "logps/rejected": -1.8644256591796875,
      "loss": 1.7747,
      "nll_loss": 1.7171214818954468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16345562040805817,
      "rewards/margins": 0.022986939176917076,
      "rewards/rejected": -0.1864425539970398,
      "step": 300
    },
    {
      "epoch": 0.4743892828999212,
      "grad_norm": 0.25458070635795593,
      "learning_rate": 3.7781286602280967e-06,
      "log_odds_chosen": 0.16361647844314575,
      "log_odds_ratio": -0.6239433288574219,
      "logits/chosen": 0.1774168461561203,
      "logits/rejected": -1.2051948308944702,
      "logps/chosen": -1.616410493850708,
      "logps/rejected": -1.745263934135437,
      "loss": 1.7654,
      "nll_loss": 1.702994465827942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16164104640483856,
      "rewards/margins": 0.012885356321930885,
      "rewards/rejected": -0.1745263934135437,
      "step": 301
    },
    {
      "epoch": 0.47596532702915684,
      "grad_norm": 0.23742206394672394,
      "learning_rate": 3.7756054719597044e-06,
      "log_odds_chosen": 0.2543914318084717,
      "log_odds_ratio": -0.59993976354599,
      "logits/chosen": 0.06885389983654022,
      "logits/rejected": -1.19141685962677,
      "logps/chosen": -1.6490998268127441,
      "logps/rejected": -1.869534969329834,
      "loss": 1.7688,
      "nll_loss": 1.7087764739990234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1649099886417389,
      "rewards/margins": 0.02204350382089615,
      "rewards/rejected": -0.18695348501205444,
      "step": 302
    },
    {
      "epoch": 0.47754137115839246,
      "grad_norm": 0.22604241967201233,
      "learning_rate": 3.773068869840066e-06,
      "log_odds_chosen": 0.19994798302650452,
      "log_odds_ratio": -0.6074256300926208,
      "logits/chosen": 0.28619179129600525,
      "logits/rejected": -1.2311829328536987,
      "logps/chosen": -1.6744049787521362,
      "logps/rejected": -1.838005542755127,
      "loss": 1.7948,
      "nll_loss": 1.7340208292007446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1674405187368393,
      "rewards/margins": 0.016360044479370117,
      "rewards/rejected": -0.1838005632162094,
      "step": 303
    },
    {
      "epoch": 0.4791174152876281,
      "grad_norm": 0.25295767188072205,
      "learning_rate": 3.770518873031997e-06,
      "log_odds_chosen": 0.4454966187477112,
      "log_odds_ratio": -0.4978073537349701,
      "logits/chosen": 0.06649523228406906,
      "logits/rejected": -0.8774638772010803,
      "logps/chosen": -1.5495182275772095,
      "logps/rejected": -1.9150521755218506,
      "loss": 1.6921,
      "nll_loss": 1.6423286199569702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15495182573795319,
      "rewards/margins": 0.03655339777469635,
      "rewards/rejected": -0.19150523841381073,
      "step": 304
    },
    {
      "epoch": 0.4806934594168637,
      "grad_norm": 0.26729831099510193,
      "learning_rate": 3.7679555007995065e-06,
      "log_odds_chosen": 0.39848801493644714,
      "log_odds_ratio": -0.5314606428146362,
      "logits/chosen": 0.18635906279087067,
      "logits/rejected": -1.1083000898361206,
      "logps/chosen": -1.671202301979065,
      "logps/rejected": -2.0063977241516113,
      "loss": 1.8023,
      "nll_loss": 1.7491679191589355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16712023317813873,
      "rewards/margins": 0.033519547432661057,
      "rewards/rejected": -0.2006397843360901,
      "step": 305
    },
    {
      "epoch": 0.48226950354609927,
      "grad_norm": 0.25258681178092957,
      "learning_rate": 3.7653787725076464e-06,
      "log_odds_chosen": 0.25875869393348694,
      "log_odds_ratio": -0.5776093602180481,
      "logits/chosen": 0.18513011932373047,
      "logits/rejected": -1.0163553953170776,
      "logps/chosen": -1.6290266513824463,
      "logps/rejected": -1.844577670097351,
      "loss": 1.7547,
      "nll_loss": 1.696903109550476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16290265321731567,
      "rewards/margins": 0.021555107086896896,
      "rewards/rejected": -0.18445777893066406,
      "step": 306
    },
    {
      "epoch": 0.4838455476753349,
      "grad_norm": 0.26785412430763245,
      "learning_rate": 3.7627887076223685e-06,
      "log_odds_chosen": 0.36140918731689453,
      "log_odds_ratio": -0.5329678058624268,
      "logits/chosen": 0.2078455090522766,
      "logits/rejected": -0.8232326507568359,
      "logps/chosen": -1.6960173845291138,
      "logps/rejected": -1.9999104738235474,
      "loss": 1.8036,
      "nll_loss": 1.7503188848495483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16960173845291138,
      "rewards/margins": 0.03038930706679821,
      "rewards/rejected": -0.19999106228351593,
      "step": 307
    },
    {
      "epoch": 0.4854215918045705,
      "grad_norm": 0.22155308723449707,
      "learning_rate": 3.7601853257103765e-06,
      "log_odds_chosen": 0.21894103288650513,
      "log_odds_ratio": -0.5963100790977478,
      "logits/chosen": 0.103122279047966,
      "logits/rejected": -1.134813904762268,
      "logps/chosen": -1.5974401235580444,
      "logps/rejected": -1.7806645631790161,
      "loss": 1.7083,
      "nll_loss": 1.6486657857894897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1597440242767334,
      "rewards/margins": 0.018322426825761795,
      "rewards/rejected": -0.1780664473772049,
      "step": 308
    },
    {
      "epoch": 0.48699763593380613,
      "grad_norm": 0.22800187766551971,
      "learning_rate": 3.7575686464389767e-06,
      "log_odds_chosen": 0.34263283014297485,
      "log_odds_ratio": -0.5459380745887756,
      "logits/chosen": 0.1387854516506195,
      "logits/rejected": -1.1660175323486328,
      "logps/chosen": -1.5741075277328491,
      "logps/rejected": -1.856614351272583,
      "loss": 1.7095,
      "nll_loss": 1.6548840999603271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15741074085235596,
      "rewards/margins": 0.02825068309903145,
      "rewards/rejected": -0.1856614202260971,
      "step": 309
    },
    {
      "epoch": 0.48857368006304175,
      "grad_norm": 0.22426706552505493,
      "learning_rate": 3.7549386895759315e-06,
      "log_odds_chosen": 0.37626713514328003,
      "log_odds_ratio": -0.5249163508415222,
      "logits/chosen": 0.11323234438896179,
      "logits/rejected": -1.1059436798095703,
      "logps/chosen": -1.5328314304351807,
      "logps/rejected": -1.8388895988464355,
      "loss": 1.656,
      "nll_loss": 1.6035394668579102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15328313410282135,
      "rewards/margins": 0.030605828389525414,
      "rewards/rejected": -0.18388895690441132,
      "step": 310
    },
    {
      "epoch": 0.49014972419227737,
      "grad_norm": 0.21954846382141113,
      "learning_rate": 3.7522954749893086e-06,
      "log_odds_chosen": 0.5949737429618835,
      "log_odds_ratio": -0.45127981901168823,
      "logits/chosen": 0.03048582188785076,
      "logits/rejected": -1.2382416725158691,
      "logps/chosen": -1.5583537817001343,
      "logps/rejected": -2.053128719329834,
      "loss": 1.6949,
      "nll_loss": 1.6497327089309692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1558353751897812,
      "rewards/margins": 0.049477506428956985,
      "rewards/rejected": -0.20531287789344788,
      "step": 311
    },
    {
      "epoch": 0.491725768321513,
      "grad_norm": 0.24479185044765472,
      "learning_rate": 3.749639022647332e-06,
      "log_odds_chosen": 0.3550751507282257,
      "log_odds_ratio": -0.535753071308136,
      "logits/chosen": 0.09609371423721313,
      "logits/rejected": -1.0945216417312622,
      "logps/chosen": -1.6470805406570435,
      "logps/rejected": -1.9446109533309937,
      "loss": 1.7724,
      "nll_loss": 1.7188349962234497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16470806300640106,
      "rewards/margins": 0.029753029346466064,
      "rewards/rejected": -0.19446110725402832,
      "step": 312
    },
    {
      "epoch": 0.4933018124507486,
      "grad_norm": 0.21343177556991577,
      "learning_rate": 3.7469693526182304e-06,
      "log_odds_chosen": 0.507236659526825,
      "log_odds_ratio": -0.4877496659755707,
      "logits/chosen": 0.05133073404431343,
      "logits/rejected": -1.505554437637329,
      "logps/chosen": -1.5942163467407227,
      "logps/rejected": -2.0157418251037598,
      "loss": 1.7043,
      "nll_loss": 1.6554750204086304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1594216376543045,
      "rewards/margins": 0.04215255007147789,
      "rewards/rejected": -0.2015742063522339,
      "step": 313
    },
    {
      "epoch": 0.49487785657998423,
      "grad_norm": 0.22851817309856415,
      "learning_rate": 3.744286485070085e-06,
      "log_odds_chosen": 0.5064301490783691,
      "log_odds_ratio": -0.4742722511291504,
      "logits/chosen": 0.0719347670674324,
      "logits/rejected": -1.217496633529663,
      "logps/chosen": -1.5423535108566284,
      "logps/rejected": -1.9602735042572021,
      "loss": 1.6577,
      "nll_loss": 1.6102428436279297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1542353630065918,
      "rewards/margins": 0.041791994124650955,
      "rewards/rejected": -0.19602736830711365,
      "step": 314
    },
    {
      "epoch": 0.49645390070921985,
      "grad_norm": 0.2481304556131363,
      "learning_rate": 3.7415904402706795e-06,
      "log_odds_chosen": 0.41338393092155457,
      "log_odds_ratio": -0.5167226791381836,
      "logits/chosen": 0.10534465312957764,
      "logits/rejected": -1.2418715953826904,
      "logps/chosen": -1.742213249206543,
      "logps/rejected": -2.095057964324951,
      "loss": 1.8526,
      "nll_loss": 1.8009579181671143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17422132194042206,
      "rewards/margins": 0.03528450429439545,
      "rewards/rejected": -0.2095058262348175,
      "step": 315
    },
    {
      "epoch": 0.4980299448384555,
      "grad_norm": 0.22114680707454681,
      "learning_rate": 3.7388812385873435e-06,
      "log_odds_chosen": 0.35585927963256836,
      "log_odds_ratio": -0.5332568883895874,
      "logits/chosen": 0.07114619016647339,
      "logits/rejected": -1.2164775133132935,
      "logps/chosen": -1.577876091003418,
      "logps/rejected": -1.8701857328414917,
      "loss": 1.6942,
      "nll_loss": 1.6408575773239136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15778760612010956,
      "rewards/margins": 0.029230959713459015,
      "rewards/rejected": -0.18701857328414917,
      "step": 316
    },
    {
      "epoch": 0.4996059889676911,
      "grad_norm": 0.24303023517131805,
      "learning_rate": 3.7361589004868033e-06,
      "log_odds_chosen": 0.4969717562198639,
      "log_odds_ratio": -0.4814871549606323,
      "logits/chosen": 0.17601065337657928,
      "logits/rejected": -1.2875845432281494,
      "logps/chosen": -1.6603622436523438,
      "logps/rejected": -2.0823590755462646,
      "loss": 1.7619,
      "nll_loss": 1.7137192487716675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1660362333059311,
      "rewards/margins": 0.04219968616962433,
      "rewards/rejected": -0.20823588967323303,
      "step": 317
    },
    {
      "epoch": 0.5011820330969267,
      "grad_norm": 0.22220100462436676,
      "learning_rate": 3.733423446535022e-06,
      "log_odds_chosen": 0.5156369209289551,
      "log_odds_ratio": -0.4770677089691162,
      "logits/chosen": 0.1485612392425537,
      "logits/rejected": -1.373117446899414,
      "logps/chosen": -1.6504617929458618,
      "logps/rejected": -2.0905652046203613,
      "loss": 1.7536,
      "nll_loss": 1.705846905708313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16504618525505066,
      "rewards/margins": 0.04401035234332085,
      "rewards/rejected": -0.2090565413236618,
      "step": 318
    },
    {
      "epoch": 0.5027580772261623,
      "grad_norm": 0.21169239282608032,
      "learning_rate": 3.7306748973970476e-06,
      "log_odds_chosen": 0.3826340436935425,
      "log_odds_ratio": -0.529701292514801,
      "logits/chosen": 0.07297363877296448,
      "logits/rejected": -1.5029369592666626,
      "logps/chosen": -1.5217207670211792,
      "logps/rejected": -1.831828236579895,
      "loss": 1.6678,
      "nll_loss": 1.6148674488067627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15217207372188568,
      "rewards/margins": 0.031010746955871582,
      "rewards/rejected": -0.18318283557891846,
      "step": 319
    },
    {
      "epoch": 0.5043341213553979,
      "grad_norm": 0.21747496724128723,
      "learning_rate": 3.7279132738368564e-06,
      "log_odds_chosen": 0.18434260785579681,
      "log_odds_ratio": -0.6090730428695679,
      "logits/chosen": 0.13470004498958588,
      "logits/rejected": -1.289181113243103,
      "logps/chosen": -1.7176737785339355,
      "logps/rejected": -1.8703434467315674,
      "loss": 1.8286,
      "nll_loss": 1.7676782608032227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17176738381385803,
      "rewards/margins": 0.015266966074705124,
      "rewards/rejected": -0.18703433871269226,
      "step": 320
    },
    {
      "epoch": 0.5059101654846335,
      "grad_norm": 0.22482265532016754,
      "learning_rate": 3.725138596717195e-06,
      "log_odds_chosen": 0.4353731870651245,
      "log_odds_ratio": -0.5029094815254211,
      "logits/chosen": 0.05846483260393143,
      "logits/rejected": -1.3245011568069458,
      "logps/chosen": -1.5730060338974,
      "logps/rejected": -1.933168888092041,
      "loss": 1.6842,
      "nll_loss": 1.6339199542999268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15730062127113342,
      "rewards/margins": 0.03601628169417381,
      "rewards/rejected": -0.19331689178943634,
      "step": 321
    },
    {
      "epoch": 0.5074862096138691,
      "grad_norm": 0.23347902297973633,
      "learning_rate": 3.7223508869994244e-06,
      "log_odds_chosen": 0.4694395065307617,
      "log_odds_ratio": -0.4926353096961975,
      "logits/chosen": 0.08627428859472275,
      "logits/rejected": -1.3811330795288086,
      "logps/chosen": -1.6119287014007568,
      "logps/rejected": -2.0031211376190186,
      "loss": 1.7265,
      "nll_loss": 1.6772775650024414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1611928790807724,
      "rewards/margins": 0.03911925479769707,
      "rewards/rejected": -0.20031213760375977,
      "step": 322
    },
    {
      "epoch": 0.5090622537431048,
      "grad_norm": 0.24362680315971375,
      "learning_rate": 3.7195501657433594e-06,
      "log_odds_chosen": 0.3797358572483063,
      "log_odds_ratio": -0.5255128741264343,
      "logits/chosen": 0.03265100345015526,
      "logits/rejected": -1.3387643098831177,
      "logps/chosen": -1.6188209056854248,
      "logps/rejected": -1.9326517581939697,
      "loss": 1.7415,
      "nll_loss": 1.6889467239379883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16188210248947144,
      "rewards/margins": 0.03138309717178345,
      "rewards/rejected": -0.19326519966125488,
      "step": 323
    },
    {
      "epoch": 0.5106382978723404,
      "grad_norm": 0.21358193457126617,
      "learning_rate": 3.716736454107111e-06,
      "log_odds_chosen": 0.5095394253730774,
      "log_odds_ratio": -0.4752754867076874,
      "logits/chosen": 0.04177181422710419,
      "logits/rejected": -1.0887494087219238,
      "logps/chosen": -1.5133510828018188,
      "logps/rejected": -1.9286960363388062,
      "loss": 1.6277,
      "nll_loss": 1.5801358222961426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15133510529994965,
      "rewards/margins": 0.04153449088335037,
      "rewards/rejected": -0.19286958873271942,
      "step": 324
    },
    {
      "epoch": 0.512214342001576,
      "grad_norm": 0.23502080142498016,
      "learning_rate": 3.7139097733469277e-06,
      "log_odds_chosen": 0.5284742712974548,
      "log_odds_ratio": -0.4676551818847656,
      "logits/chosen": 0.05847911164164543,
      "logits/rejected": -1.3190383911132812,
      "logps/chosen": -1.5592632293701172,
      "logps/rejected": -1.9967985153198242,
      "loss": 1.677,
      "nll_loss": 1.6302053928375244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15592631697654724,
      "rewards/margins": 0.04375353828072548,
      "rewards/rejected": -0.19967986643314362,
      "step": 325
    },
    {
      "epoch": 0.5137903861308116,
      "grad_norm": 0.23256242275238037,
      "learning_rate": 3.711070144817032e-06,
      "log_odds_chosen": 0.4173673391342163,
      "log_odds_ratio": -0.5119081735610962,
      "logits/chosen": 0.047142788767814636,
      "logits/rejected": -1.1630587577819824,
      "logps/chosen": -1.6483253240585327,
      "logps/rejected": -2.0011093616485596,
      "loss": 1.7695,
      "nll_loss": 1.718288779258728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16483254730701447,
      "rewards/margins": 0.035278379917144775,
      "rewards/rejected": -0.20011092722415924,
      "step": 326
    },
    {
      "epoch": 0.5153664302600472,
      "grad_norm": 0.23888525366783142,
      "learning_rate": 3.708217589969461e-06,
      "log_odds_chosen": 0.5111429691314697,
      "log_odds_ratio": -0.4780721664428711,
      "logits/chosen": 0.0068323384039103985,
      "logits/rejected": -1.3913313150405884,
      "logps/chosen": -1.621565818786621,
      "logps/rejected": -2.0499422550201416,
      "loss": 1.7462,
      "nll_loss": 1.6983516216278076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1621565967798233,
      "rewards/margins": 0.04283764958381653,
      "rewards/rejected": -0.20499424636363983,
      "step": 327
    },
    {
      "epoch": 0.5169424743892829,
      "grad_norm": 0.23694045841693878,
      "learning_rate": 3.705352130353904e-06,
      "log_odds_chosen": 0.45277297496795654,
      "log_odds_ratio": -0.5033365488052368,
      "logits/chosen": 0.07880112528800964,
      "logits/rejected": -1.3383313417434692,
      "logps/chosen": -1.539336085319519,
      "logps/rejected": -1.9113584756851196,
      "loss": 1.6711,
      "nll_loss": 1.6207761764526367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15393361449241638,
      "rewards/margins": 0.03720225393772125,
      "rewards/rejected": -0.19113586843013763,
      "step": 328
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 0.22416533529758453,
      "learning_rate": 3.7024737876175404e-06,
      "log_odds_chosen": 0.4236825108528137,
      "log_odds_ratio": -0.5054708123207092,
      "logits/chosen": 0.021605733782052994,
      "logits/rejected": -1.1384456157684326,
      "logps/chosen": -1.5932283401489258,
      "logps/rejected": -1.9415353536605835,
      "loss": 1.7001,
      "nll_loss": 1.6495964527130127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1593228429555893,
      "rewards/margins": 0.03483070060610771,
      "rewards/rejected": -0.1941535323858261,
      "step": 329
    },
    {
      "epoch": 0.5200945626477541,
      "grad_norm": 0.23309147357940674,
      "learning_rate": 3.699582583504874e-06,
      "log_odds_chosen": 0.2958032786846161,
      "log_odds_ratio": -0.5629885196685791,
      "logits/chosen": 0.07190592586994171,
      "logits/rejected": -1.3420836925506592,
      "logps/chosen": -1.7306642532348633,
      "logps/rejected": -1.9801539182662964,
      "loss": 1.8318,
      "nll_loss": 1.7754981517791748,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1730664223432541,
      "rewards/margins": 0.02494896575808525,
      "rewards/rejected": -0.19801540672779083,
      "step": 330
    },
    {
      "epoch": 0.5216706067769897,
      "grad_norm": 0.21672366559505463,
      "learning_rate": 3.696678539857571e-06,
      "log_odds_chosen": 0.38817960023880005,
      "log_odds_ratio": -0.5281521677970886,
      "logits/chosen": 0.02449723891913891,
      "logits/rejected": -1.3222875595092773,
      "logps/chosen": -1.7110434770584106,
      "logps/rejected": -2.0422046184539795,
      "loss": 1.8193,
      "nll_loss": 1.766467571258545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17110435664653778,
      "rewards/margins": 0.03311609849333763,
      "rewards/rejected": -0.20422044396400452,
      "step": 331
    },
    {
      "epoch": 0.5232466509062254,
      "grad_norm": 0.22159616649150848,
      "learning_rate": 3.6937616786142956e-06,
      "log_odds_chosen": 0.5169373750686646,
      "log_odds_ratio": -0.47212016582489014,
      "logits/chosen": 0.048412833362817764,
      "logits/rejected": -1.0862215757369995,
      "logps/chosen": -1.5307966470718384,
      "logps/rejected": -1.9572421312332153,
      "loss": 1.6629,
      "nll_loss": 1.6156394481658936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15307967364788055,
      "rewards/margins": 0.04264454543590546,
      "rewards/rejected": -0.1957242339849472,
      "step": 332
    },
    {
      "epoch": 0.524822695035461,
      "grad_norm": 0.19886189699172974,
      "learning_rate": 3.6908320218105393e-06,
      "log_odds_chosen": 0.32757923007011414,
      "log_odds_ratio": -0.5524694919586182,
      "logits/chosen": 0.028493206948041916,
      "logits/rejected": -1.2768640518188477,
      "logps/chosen": -1.5223734378814697,
      "logps/rejected": -1.792266845703125,
      "loss": 1.637,
      "nll_loss": 1.5817255973815918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15223734080791473,
      "rewards/margins": 0.02698933705687523,
      "rewards/rejected": -0.17922668159008026,
      "step": 333
    },
    {
      "epoch": 0.5263987391646966,
      "grad_norm": 0.1958778351545334,
      "learning_rate": 3.6878895915784607e-06,
      "log_odds_chosen": 0.4811401963233948,
      "log_odds_ratio": -0.4867932200431824,
      "logits/chosen": 0.06704150140285492,
      "logits/rejected": -1.2202537059783936,
      "logps/chosen": -1.4886175394058228,
      "logps/rejected": -1.8793421983718872,
      "loss": 1.5997,
      "nll_loss": 1.5510001182556152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14886175096035004,
      "rewards/margins": 0.039072465151548386,
      "rewards/rejected": -0.18793421983718872,
      "step": 334
    },
    {
      "epoch": 0.5279747832939322,
      "grad_norm": 0.2485465407371521,
      "learning_rate": 3.6849344101467147e-06,
      "log_odds_chosen": 0.393343985080719,
      "log_odds_ratio": -0.5225814580917358,
      "logits/chosen": 0.059318412095308304,
      "logits/rejected": -1.4150105714797974,
      "logps/chosen": -1.5716626644134521,
      "logps/rejected": -1.8979530334472656,
      "loss": 1.6926,
      "nll_loss": 1.6403700113296509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1571662873029709,
      "rewards/margins": 0.032629020512104034,
      "rewards/rejected": -0.18979531526565552,
      "step": 335
    },
    {
      "epoch": 0.5295508274231678,
      "grad_norm": 0.2148926705121994,
      "learning_rate": 3.6819664998402857e-06,
      "log_odds_chosen": 0.3840746283531189,
      "log_odds_ratio": -0.5230731964111328,
      "logits/chosen": 0.03537122905254364,
      "logits/rejected": -1.249131441116333,
      "logps/chosen": -1.6128778457641602,
      "logps/rejected": -1.9333324432373047,
      "loss": 1.7263,
      "nll_loss": 1.6739463806152344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16128778457641602,
      "rewards/margins": 0.03204546496272087,
      "rewards/rejected": -0.19333325326442719,
      "step": 336
    },
    {
      "epoch": 0.5311268715524035,
      "grad_norm": 0.22502197325229645,
      "learning_rate": 3.6789858830803186e-06,
      "log_odds_chosen": 0.4095141291618347,
      "log_odds_ratio": -0.5146152973175049,
      "logits/chosen": 0.054265327751636505,
      "logits/rejected": -1.2500349283218384,
      "logps/chosen": -1.6632660627365112,
      "logps/rejected": -2.0055625438690186,
      "loss": 1.7668,
      "nll_loss": 1.7153334617614746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1663266271352768,
      "rewards/margins": 0.03422961384057999,
      "rewards/rejected": -0.20055623352527618,
      "step": 337
    },
    {
      "epoch": 0.5327029156816391,
      "grad_norm": 0.21749760210514069,
      "learning_rate": 3.6759925823839486e-06,
      "log_odds_chosen": 0.3231702446937561,
      "log_odds_ratio": -0.549367368221283,
      "logits/chosen": 0.019689474254846573,
      "logits/rejected": -1.1351220607757568,
      "logps/chosen": -1.5444843769073486,
      "logps/rejected": -1.8077837228775024,
      "loss": 1.6811,
      "nll_loss": 1.6261268854141235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15444843471050262,
      "rewards/margins": 0.026329942047595978,
      "rewards/rejected": -0.180778369307518,
      "step": 338
    },
    {
      "epoch": 0.5342789598108747,
      "grad_norm": 0.2007884979248047,
      "learning_rate": 3.672986620364134e-06,
      "log_odds_chosen": 0.4424893260002136,
      "log_odds_ratio": -0.4993942081928253,
      "logits/chosen": 0.06502474844455719,
      "logits/rejected": -1.1232551336288452,
      "logps/chosen": -1.6207716464996338,
      "logps/rejected": -1.9894078969955444,
      "loss": 1.7286,
      "nll_loss": 1.6786723136901855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1620771735906601,
      "rewards/margins": 0.03686363995075226,
      "rewards/rejected": -0.19894081354141235,
      "step": 339
    },
    {
      "epoch": 0.5358550039401103,
      "grad_norm": 0.22352111339569092,
      "learning_rate": 3.669968019729481e-06,
      "log_odds_chosen": 0.5362953543663025,
      "log_odds_ratio": -0.4702385365962982,
      "logits/chosen": 0.00828593224287033,
      "logits/rejected": -1.415726661682129,
      "logps/chosen": -1.4964334964752197,
      "logps/rejected": -1.938150405883789,
      "loss": 1.6141,
      "nll_loss": 1.5671132802963257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14964337646961212,
      "rewards/margins": 0.044171661138534546,
      "rewards/rejected": -0.19381502270698547,
      "step": 340
    },
    {
      "epoch": 0.5374310480693459,
      "grad_norm": 0.21597124636173248,
      "learning_rate": 3.666936803284076e-06,
      "log_odds_chosen": 0.4687511920928955,
      "log_odds_ratio": -0.4885876774787903,
      "logits/chosen": 0.045443110167980194,
      "logits/rejected": -1.2455968856811523,
      "logps/chosen": -1.638263463973999,
      "logps/rejected": -2.0333070755004883,
      "loss": 1.7327,
      "nll_loss": 1.6838411092758179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1638263612985611,
      "rewards/margins": 0.039504360407590866,
      "rewards/rejected": -0.20333071053028107,
      "step": 341
    },
    {
      "epoch": 0.5390070921985816,
      "grad_norm": 0.2129712551832199,
      "learning_rate": 3.663892993927312e-06,
      "log_odds_chosen": 0.5479106903076172,
      "log_odds_ratio": -0.46075499057769775,
      "logits/chosen": -0.023554421961307526,
      "logits/rejected": -1.3818119764328003,
      "logps/chosen": -1.5896512269973755,
      "logps/rejected": -2.0474183559417725,
      "loss": 1.707,
      "nll_loss": 1.6609206199645996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1589651256799698,
      "rewards/margins": 0.04577670246362686,
      "rewards/rejected": -0.20474180579185486,
      "step": 342
    },
    {
      "epoch": 0.5405831363278172,
      "grad_norm": 0.20988072454929352,
      "learning_rate": 3.6608366146537136e-06,
      "log_odds_chosen": 0.6004074215888977,
      "log_odds_ratio": -0.4560423493385315,
      "logits/chosen": -0.054608047008514404,
      "logits/rejected": -1.2286299467086792,
      "logps/chosen": -1.5147725343704224,
      "logps/rejected": -2.0184264183044434,
      "loss": 1.6343,
      "nll_loss": 1.5887430906295776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15147726237773895,
      "rewards/margins": 0.050365399569272995,
      "rewards/rejected": -0.20184265077114105,
      "step": 343
    },
    {
      "epoch": 0.5421591804570528,
      "grad_norm": 0.21432410180568695,
      "learning_rate": 3.6577676885527674e-06,
      "log_odds_chosen": 0.46950504183769226,
      "log_odds_ratio": -0.49097535014152527,
      "logits/chosen": -0.030298583209514618,
      "logits/rejected": -1.479987382888794,
      "logps/chosen": -1.5775291919708252,
      "logps/rejected": -1.9668179750442505,
      "loss": 1.6785,
      "nll_loss": 1.6294326782226562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15775293111801147,
      "rewards/margins": 0.03892885893583298,
      "rewards/rejected": -0.19668179750442505,
      "step": 344
    },
    {
      "epoch": 0.5437352245862884,
      "grad_norm": 0.20334196090698242,
      "learning_rate": 3.654686238808744e-06,
      "log_odds_chosen": 0.45373964309692383,
      "log_odds_ratio": -0.4997464418411255,
      "logits/chosen": -0.05928083136677742,
      "logits/rejected": -1.4065111875534058,
      "logps/chosen": -1.6670969724655151,
      "logps/rejected": -2.049161434173584,
      "loss": 1.7703,
      "nll_loss": 1.7203750610351562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16670972108840942,
      "rewards/margins": 0.038206446915864944,
      "rewards/rejected": -0.20491616427898407,
      "step": 345
    },
    {
      "epoch": 0.545311268715524,
      "grad_norm": 0.20314708352088928,
      "learning_rate": 3.6515922887005245e-06,
      "log_odds_chosen": 0.6570188403129578,
      "log_odds_ratio": -0.42130351066589355,
      "logits/chosen": -0.15550881624221802,
      "logits/rejected": -1.41984224319458,
      "logps/chosen": -1.5048458576202393,
      "logps/rejected": -2.044806480407715,
      "loss": 1.6173,
      "nll_loss": 1.575134515762329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1504845917224884,
      "rewards/margins": 0.05399605259299278,
      "rewards/rejected": -0.20448064804077148,
      "step": 346
    },
    {
      "epoch": 0.5468873128447597,
      "grad_norm": 0.1935390830039978,
      "learning_rate": 3.6484858616014236e-06,
      "log_odds_chosen": 0.28654223680496216,
      "log_odds_ratio": -0.5621439218521118,
      "logits/chosen": 0.017233777791261673,
      "logits/rejected": -1.4268871545791626,
      "logps/chosen": -1.6293790340423584,
      "logps/rejected": -1.8657327890396118,
      "loss": 1.7299,
      "nll_loss": 1.6737170219421387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16293790936470032,
      "rewards/margins": 0.023635366931557655,
      "rewards/rejected": -0.18657328188419342,
      "step": 347
    },
    {
      "epoch": 0.5484633569739953,
      "grad_norm": 0.2078549563884735,
      "learning_rate": 3.6453669809790154e-06,
      "log_odds_chosen": 0.3306597173213959,
      "log_odds_ratio": -0.5512881278991699,
      "logits/chosen": 0.036211688071489334,
      "logits/rejected": -1.0868035554885864,
      "logps/chosen": -1.5622117519378662,
      "logps/rejected": -1.834636926651001,
      "loss": 1.6717,
      "nll_loss": 1.616525411605835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1562211811542511,
      "rewards/margins": 0.02724250964820385,
      "rewards/rejected": -0.1834636926651001,
      "step": 348
    },
    {
      "epoch": 0.5500394011032309,
      "grad_norm": 0.20183558762073517,
      "learning_rate": 3.642235670394952e-06,
      "log_odds_chosen": 0.42099136114120483,
      "log_odds_ratio": -0.5093228220939636,
      "logits/chosen": 0.008305356837809086,
      "logits/rejected": -1.0708895921707153,
      "logps/chosen": -1.5708250999450684,
      "logps/rejected": -1.9134736061096191,
      "loss": 1.689,
      "nll_loss": 1.6381096839904785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15708249807357788,
      "rewards/margins": 0.034264855086803436,
      "rewards/rejected": -0.19134736061096191,
      "step": 349
    },
    {
      "epoch": 0.5516154452324665,
      "grad_norm": 0.19396637380123138,
      "learning_rate": 3.63909195350479e-06,
      "log_odds_chosen": 0.40000632405281067,
      "log_odds_ratio": -0.5222204923629761,
      "logits/chosen": 0.0709347128868103,
      "logits/rejected": -1.2560487985610962,
      "logps/chosen": -1.6595628261566162,
      "logps/rejected": -1.9919466972351074,
      "loss": 1.7618,
      "nll_loss": 1.7095361948013306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1659562885761261,
      "rewards/margins": 0.03323838487267494,
      "rewards/rejected": -0.19919466972351074,
      "step": 350
    },
    {
      "epoch": 0.5531914893617021,
      "grad_norm": 0.20782439410686493,
      "learning_rate": 3.635935854057809e-06,
      "log_odds_chosen": 0.39174172282218933,
      "log_odds_ratio": -0.5250329375267029,
      "logits/chosen": -0.1102612316608429,
      "logits/rejected": -0.9730474948883057,
      "logps/chosen": -1.6109540462493896,
      "logps/rejected": -1.9376519918441772,
      "loss": 1.7198,
      "nll_loss": 1.6673376560211182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16109541058540344,
      "rewards/margins": 0.032669797539711,
      "rewards/rejected": -0.19376519322395325,
      "step": 351
    },
    {
      "epoch": 0.5547675334909378,
      "grad_norm": 0.19801416993141174,
      "learning_rate": 3.6327673958968327e-06,
      "log_odds_chosen": 0.5151069760322571,
      "log_odds_ratio": -0.47180619835853577,
      "logits/chosen": -0.06693591177463531,
      "logits/rejected": -1.3027480840682983,
      "logps/chosen": -1.4905201196670532,
      "logps/rejected": -1.9089621305465698,
      "loss": 1.6179,
      "nll_loss": 1.570717215538025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14905200898647308,
      "rewards/margins": 0.0418442040681839,
      "rewards/rejected": -0.19089621305465698,
      "step": 352
    },
    {
      "epoch": 0.5563435776201734,
      "grad_norm": 0.18420955538749695,
      "learning_rate": 3.6295866029580483e-06,
      "log_odds_chosen": 0.49202004075050354,
      "log_odds_ratio": -0.4803290367126465,
      "logits/chosen": -0.0715307965874672,
      "logits/rejected": -1.3764203786849976,
      "logps/chosen": -1.5352476835250854,
      "logps/rejected": -1.938019037246704,
      "loss": 1.6324,
      "nll_loss": 1.5843286514282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15352477133274078,
      "rewards/margins": 0.04027712717652321,
      "rewards/rejected": -0.1938018947839737,
      "step": 353
    },
    {
      "epoch": 0.557919621749409,
      "grad_norm": 0.19222892820835114,
      "learning_rate": 3.626393499270829e-06,
      "log_odds_chosen": 0.44654372334480286,
      "log_odds_ratio": -0.5027562975883484,
      "logits/chosen": -0.057136163115501404,
      "logits/rejected": -1.3216649293899536,
      "logps/chosen": -1.473515510559082,
      "logps/rejected": -1.8351870775222778,
      "loss": 1.6025,
      "nll_loss": 1.552269458770752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735153317451477,
      "rewards/margins": 0.03616715967655182,
      "rewards/rejected": -0.18351870775222778,
      "step": 354
    },
    {
      "epoch": 0.5594956658786446,
      "grad_norm": 0.198707714676857,
      "learning_rate": 3.6231881089575466e-06,
      "log_odds_chosen": 0.6093676686286926,
      "log_odds_ratio": -0.437423974275589,
      "logits/chosen": -0.1271413415670395,
      "logits/rejected": -1.4303815364837646,
      "logps/chosen": -1.5215667486190796,
      "logps/rejected": -2.027055263519287,
      "loss": 1.6059,
      "nll_loss": 1.5621567964553833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15215668082237244,
      "rewards/margins": 0.05054884031414986,
      "rewards/rejected": -0.202705517411232,
      "step": 355
    },
    {
      "epoch": 0.5610717100078803,
      "grad_norm": 0.26446157693862915,
      "learning_rate": 3.6199704562333945e-06,
      "log_odds_chosen": 0.4508443772792816,
      "log_odds_ratio": -0.5002342462539673,
      "logits/chosen": -0.09892217069864273,
      "logits/rejected": -1.3460979461669922,
      "logps/chosen": -1.494001030921936,
      "logps/rejected": -1.8570603132247925,
      "loss": 1.604,
      "nll_loss": 1.553969383239746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14940010011196136,
      "rewards/margins": 0.03630594164133072,
      "rewards/rejected": -0.18570604920387268,
      "step": 356
    },
    {
      "epoch": 0.5626477541371159,
      "grad_norm": 0.20473134517669678,
      "learning_rate": 3.6167405654062024e-06,
      "log_odds_chosen": 0.4710828363895416,
      "log_odds_ratio": -0.4989466369152069,
      "logits/chosen": -0.04282144084572792,
      "logits/rejected": -1.2687091827392578,
      "logps/chosen": -1.5530476570129395,
      "logps/rejected": -1.9394475221633911,
      "loss": 1.6523,
      "nll_loss": 1.602416753768921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15530475974082947,
      "rewards/margins": 0.03863998502492905,
      "rewards/rejected": -0.19394473731517792,
      "step": 357
    },
    {
      "epoch": 0.5642237982663515,
      "grad_norm": 0.18403887748718262,
      "learning_rate": 3.6134984608762515e-06,
      "log_odds_chosen": 0.5030588507652283,
      "log_odds_ratio": -0.476035475730896,
      "logits/chosen": -0.05343765392899513,
      "logits/rejected": -1.3506138324737549,
      "logps/chosen": -1.4735472202301025,
      "logps/rejected": -1.8805701732635498,
      "loss": 1.5937,
      "nll_loss": 1.546113133430481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735473692417145,
      "rewards/margins": 0.04070229083299637,
      "rewards/rejected": -0.1880570352077484,
      "step": 358
    },
    {
      "epoch": 0.5657998423955871,
      "grad_norm": 0.21042759716510773,
      "learning_rate": 3.6102441671360945e-06,
      "log_odds_chosen": 0.48499730229377747,
      "log_odds_ratio": -0.49012240767478943,
      "logits/chosen": -0.0872420072555542,
      "logits/rejected": -1.223256230354309,
      "logps/chosen": -1.6216155290603638,
      "logps/rejected": -2.0279018878936768,
      "loss": 1.7136,
      "nll_loss": 1.6645991802215576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16216154396533966,
      "rewards/margins": 0.04062866419553757,
      "rewards/rejected": -0.20279023051261902,
      "step": 359
    },
    {
      "epoch": 0.5673758865248227,
      "grad_norm": 0.199711412191391,
      "learning_rate": 3.6069777087703654e-06,
      "log_odds_chosen": 0.5050286650657654,
      "log_odds_ratio": -0.47864657640457153,
      "logits/chosen": -0.06602545827627182,
      "logits/rejected": -1.2074447870254517,
      "logps/chosen": -1.5410226583480835,
      "logps/rejected": -1.956892728805542,
      "loss": 1.6585,
      "nll_loss": 1.610638976097107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15410228073596954,
      "rewards/margins": 0.04158700257539749,
      "rewards/rejected": -0.19568926095962524,
      "step": 360
    },
    {
      "epoch": 0.5689519306540584,
      "grad_norm": 0.20484104752540588,
      "learning_rate": 3.6036991104555973e-06,
      "log_odds_chosen": 0.4478030502796173,
      "log_odds_ratio": -0.5068904757499695,
      "logits/chosen": -0.04828740656375885,
      "logits/rejected": -1.331907033920288,
      "logps/chosen": -1.6810355186462402,
      "logps/rejected": -2.061274290084839,
      "loss": 1.771,
      "nll_loss": 1.720302939414978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16810354590415955,
      "rewards/margins": 0.038023896515369415,
      "rewards/rejected": -0.20612744987010956,
      "step": 361
    },
    {
      "epoch": 0.570527974783294,
      "grad_norm": 0.194278284907341,
      "learning_rate": 3.600408396960034e-06,
      "log_odds_chosen": 0.5042534470558167,
      "log_odds_ratio": -0.4804871380329132,
      "logits/chosen": -0.02915314957499504,
      "logits/rejected": -0.9427077174186707,
      "logps/chosen": -1.4871515035629272,
      "logps/rejected": -1.8912396430969238,
      "loss": 1.582,
      "nll_loss": 1.533990502357483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14871516823768616,
      "rewards/margins": 0.04040880501270294,
      "rewards/rejected": -0.1891239732503891,
      "step": 362
    },
    {
      "epoch": 0.5721040189125296,
      "grad_norm": 0.1945444643497467,
      "learning_rate": 3.5971055931434447e-06,
      "log_odds_chosen": 0.43649476766586304,
      "log_odds_ratio": -0.506388783454895,
      "logits/chosen": -0.02089090645313263,
      "logits/rejected": -1.2006688117980957,
      "logps/chosen": -1.536198377609253,
      "logps/rejected": -1.8970880508422852,
      "loss": 1.6438,
      "nll_loss": 1.5931452512741089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15361982583999634,
      "rewards/margins": 0.036088977009058,
      "rewards/rejected": -0.18970881402492523,
      "step": 363
    },
    {
      "epoch": 0.5736800630417652,
      "grad_norm": 0.18837140500545502,
      "learning_rate": 3.5937907239569343e-06,
      "log_odds_chosen": 0.4648410677909851,
      "log_odds_ratio": -0.4977383613586426,
      "logits/chosen": -0.09922034293413162,
      "logits/rejected": -1.3353502750396729,
      "logps/chosen": -1.5880086421966553,
      "logps/rejected": -1.9711687564849854,
      "loss": 1.6969,
      "nll_loss": 1.647141456604004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15880087018013,
      "rewards/margins": 0.03831600397825241,
      "rewards/rejected": -0.197116881608963,
      "step": 364
    },
    {
      "epoch": 0.5752561071710008,
      "grad_norm": 0.22331880033016205,
      "learning_rate": 3.5904638144427572e-06,
      "log_odds_chosen": 0.2761245369911194,
      "log_odds_ratio": -0.5722551941871643,
      "logits/chosen": -0.0540853813290596,
      "logits/rejected": -1.0778639316558838,
      "logps/chosen": -1.6464744806289673,
      "logps/rejected": -1.874767780303955,
      "loss": 1.7439,
      "nll_loss": 1.6866995096206665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1646474450826645,
      "rewards/margins": 0.022829342633485794,
      "rewards/rejected": -0.18747679889202118,
      "step": 365
    },
    {
      "epoch": 0.5768321513002365,
      "grad_norm": 0.1964210569858551,
      "learning_rate": 3.5871248897341246e-06,
      "log_odds_chosen": 0.5015654563903809,
      "log_odds_ratio": -0.4797275960445404,
      "logits/chosen": -0.0888606533408165,
      "logits/rejected": -1.0420361757278442,
      "logps/chosen": -1.4869009256362915,
      "logps/rejected": -1.9011142253875732,
      "loss": 1.6095,
      "nll_loss": 1.5615687370300293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486901044845581,
      "rewards/margins": 0.04142133146524429,
      "rewards/rejected": -0.1901114135980606,
      "step": 366
    },
    {
      "epoch": 0.578408195429472,
      "grad_norm": 0.18352432548999786,
      "learning_rate": 3.5837739750550182e-06,
      "log_odds_chosen": 0.48427703976631165,
      "log_odds_ratio": -0.48840558528900146,
      "logits/chosen": -0.11341613531112671,
      "logits/rejected": -1.6361585855484009,
      "logps/chosen": -1.564819574356079,
      "logps/rejected": -1.968957543373108,
      "loss": 1.6656,
      "nll_loss": 1.6167792081832886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15648198127746582,
      "rewards/margins": 0.040413789451122284,
      "rewards/rejected": -0.1968957632780075,
      "step": 367
    },
    {
      "epoch": 0.5799842395587076,
      "grad_norm": 0.18227070569992065,
      "learning_rate": 3.5804110957199977e-06,
      "log_odds_chosen": 0.5225378274917603,
      "log_odds_ratio": -0.47141802310943604,
      "logits/chosen": 0.0010305908508598804,
      "logits/rejected": -1.1611210107803345,
      "logps/chosen": -1.5438573360443115,
      "logps/rejected": -1.9776092767715454,
      "loss": 1.6428,
      "nll_loss": 1.5957022905349731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543857306241989,
      "rewards/margins": 0.043375201523303986,
      "rewards/rejected": -0.1977609246969223,
      "step": 368
    },
    {
      "epoch": 0.5815602836879432,
      "grad_norm": 0.19467630982398987,
      "learning_rate": 3.577036277134011e-06,
      "log_odds_chosen": 0.5960920453071594,
      "log_odds_ratio": -0.4427749514579773,
      "logits/chosen": -0.08947373181581497,
      "logits/rejected": -1.3899809122085571,
      "logps/chosen": -1.5279539823532104,
      "logps/rejected": -2.0212056636810303,
      "loss": 1.6406,
      "nll_loss": 1.5962986946105957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15279540419578552,
      "rewards/margins": 0.04932515323162079,
      "rewards/rejected": -0.2021205574274063,
      "step": 369
    },
    {
      "epoch": 0.5831363278171788,
      "grad_norm": 0.20579133927822113,
      "learning_rate": 3.5736495447922e-06,
      "log_odds_chosen": 0.3752758502960205,
      "log_odds_ratio": -0.5272895693778992,
      "logits/chosen": -0.08593582361936569,
      "logits/rejected": -1.4058607816696167,
      "logps/chosen": -1.6539585590362549,
      "logps/rejected": -1.966200351715088,
      "loss": 1.7574,
      "nll_loss": 1.7046749591827393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1653958410024643,
      "rewards/margins": 0.03122418001294136,
      "rewards/rejected": -0.19662001729011536,
      "step": 370
    },
    {
      "epoch": 0.5847123719464145,
      "grad_norm": 0.17624422907829285,
      "learning_rate": 3.5702509242797096e-06,
      "log_odds_chosen": 0.6972737312316895,
      "log_odds_ratio": -0.41951215267181396,
      "logits/chosen": -0.09982432425022125,
      "logits/rejected": -1.3790462017059326,
      "logps/chosen": -1.4812397956848145,
      "logps/rejected": -2.058272361755371,
      "loss": 1.595,
      "nll_loss": 1.5530776977539062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14812399446964264,
      "rewards/margins": 0.05770324915647507,
      "rewards/rejected": -0.2058272510766983,
      "step": 371
    },
    {
      "epoch": 0.5862884160756501,
      "grad_norm": 0.18756203353405,
      "learning_rate": 3.566840441271495e-06,
      "log_odds_chosen": 0.6480004787445068,
      "log_odds_ratio": -0.4360523819923401,
      "logits/chosen": -0.05456085875630379,
      "logits/rejected": -1.45460045337677,
      "logps/chosen": -1.49797523021698,
      "logps/rejected": -2.0297532081604004,
      "loss": 1.5874,
      "nll_loss": 1.5438189506530762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14979752898216248,
      "rewards/margins": 0.053177814930677414,
      "rewards/rejected": -0.2029753178358078,
      "step": 372
    },
    {
      "epoch": 0.5878644602048857,
      "grad_norm": 0.19936488568782806,
      "learning_rate": 3.5634181215321265e-06,
      "log_odds_chosen": 0.6168785095214844,
      "log_odds_ratio": -0.43505510687828064,
      "logits/chosen": -0.044426970183849335,
      "logits/rejected": -1.275886058807373,
      "logps/chosen": -1.553296685218811,
      "logps/rejected": -2.0681307315826416,
      "loss": 1.6652,
      "nll_loss": 1.6216744184494019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15532967448234558,
      "rewards/margins": 0.051483407616615295,
      "rewards/rejected": -0.20681308209896088,
      "step": 373
    },
    {
      "epoch": 0.5894405043341213,
      "grad_norm": 0.1882179081439972,
      "learning_rate": 3.5599839909155947e-06,
      "log_odds_chosen": 0.48124241828918457,
      "log_odds_ratio": -0.4906978905200958,
      "logits/chosen": -0.06898046284914017,
      "logits/rejected": -1.3314614295959473,
      "logps/chosen": -1.5185297727584839,
      "logps/rejected": -1.914254903793335,
      "loss": 1.6201,
      "nll_loss": 1.5710440874099731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15185298025608063,
      "rewards/margins": 0.039572522044181824,
      "rewards/rejected": -0.19142550230026245,
      "step": 374
    },
    {
      "epoch": 0.5910165484633569,
      "grad_norm": 0.18996809422969818,
      "learning_rate": 3.556538075365116e-06,
      "log_odds_chosen": 0.5556556582450867,
      "log_odds_ratio": -0.46352192759513855,
      "logits/chosen": -0.028684724122285843,
      "logits/rejected": -1.3568097352981567,
      "logps/chosen": -1.5122661590576172,
      "logps/rejected": -1.9674909114837646,
      "loss": 1.6132,
      "nll_loss": 1.566803216934204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15122662484645844,
      "rewards/margins": 0.04552246257662773,
      "rewards/rejected": -0.19674909114837646,
      "step": 375
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 0.1981869488954544,
      "learning_rate": 3.5530804009129367e-06,
      "log_odds_chosen": 0.4985346794128418,
      "log_odds_ratio": -0.4772438406944275,
      "logits/chosen": -0.07949502021074295,
      "logits/rejected": -1.1893216371536255,
      "logps/chosen": -1.548764944076538,
      "logps/rejected": -1.961025357246399,
      "loss": 1.6379,
      "nll_loss": 1.590188980102539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548765003681183,
      "rewards/margins": 0.04122602939605713,
      "rewards/rejected": -0.19610252976417542,
      "step": 376
    },
    {
      "epoch": 0.5941686367218282,
      "grad_norm": 0.19308407604694366,
      "learning_rate": 3.5496109936801368e-06,
      "log_odds_chosen": 0.4907795190811157,
      "log_odds_ratio": -0.4855614900588989,
      "logits/chosen": -0.13920965790748596,
      "logits/rejected": -1.4071189165115356,
      "logps/chosen": -1.5704931020736694,
      "logps/rejected": -1.9831798076629639,
      "loss": 1.6681,
      "nll_loss": 1.6195642948150635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15704932808876038,
      "rewards/margins": 0.041268669068813324,
      "rewards/rejected": -0.1983179897069931,
      "step": 377
    },
    {
      "epoch": 0.5957446808510638,
      "grad_norm": 0.20303456485271454,
      "learning_rate": 3.546129879876429e-06,
      "log_odds_chosen": 0.35706377029418945,
      "log_odds_ratio": -0.5432414412498474,
      "logits/chosen": -0.023199474439024925,
      "logits/rejected": -1.086435317993164,
      "logps/chosen": -1.6312530040740967,
      "logps/rejected": -1.9287961721420288,
      "loss": 1.7262,
      "nll_loss": 1.6719212532043457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16312529146671295,
      "rewards/margins": 0.029754336923360825,
      "rewards/rejected": -0.19287963211536407,
      "step": 378
    },
    {
      "epoch": 0.5973207249802994,
      "grad_norm": 0.19225862622261047,
      "learning_rate": 3.5426370857999662e-06,
      "log_odds_chosen": 0.3647347092628479,
      "log_odds_ratio": -0.531550407409668,
      "logits/chosen": -0.07333146780729294,
      "logits/rejected": -1.2189780473709106,
      "logps/chosen": -1.6128994226455688,
      "logps/rejected": -1.9161568880081177,
      "loss": 1.7184,
      "nll_loss": 1.6652899980545044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16128993034362793,
      "rewards/margins": 0.030325744301080704,
      "rewards/rejected": -0.19161568582057953,
      "step": 379
    },
    {
      "epoch": 0.598896769109535,
      "grad_norm": 0.19579578936100006,
      "learning_rate": 3.53913263783714e-06,
      "log_odds_chosen": 0.41187989711761475,
      "log_odds_ratio": -0.5210331678390503,
      "logits/chosen": -0.1096126139163971,
      "logits/rejected": -1.3426750898361206,
      "logps/chosen": -1.5626639127731323,
      "logps/rejected": -1.9077504873275757,
      "loss": 1.6558,
      "nll_loss": 1.6037330627441406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15626640617847443,
      "rewards/margins": 0.03450864553451538,
      "rewards/rejected": -0.1907750368118286,
      "step": 380
    },
    {
      "epoch": 0.6004728132387707,
      "grad_norm": 0.18819411098957062,
      "learning_rate": 3.53561656246238e-06,
      "log_odds_chosen": 0.6348834037780762,
      "log_odds_ratio": -0.4286256432533264,
      "logits/chosen": -0.2015249878168106,
      "logits/rejected": -1.5831485986709595,
      "logps/chosen": -1.5856183767318726,
      "logps/rejected": -2.117565631866455,
      "loss": 1.6802,
      "nll_loss": 1.6373547315597534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1585618406534195,
      "rewards/margins": 0.053194742649793625,
      "rewards/rejected": -0.21175658702850342,
      "step": 381
    },
    {
      "epoch": 0.6020488573680063,
      "grad_norm": 0.18237391114234924,
      "learning_rate": 3.532088886237956e-06,
      "log_odds_chosen": 0.5271506905555725,
      "log_odds_ratio": -0.4673537015914917,
      "logits/chosen": -0.12271787226200104,
      "logits/rejected": -1.5163284540176392,
      "logps/chosen": -1.6115859746932983,
      "logps/rejected": -2.0531668663024902,
      "loss": 1.6911,
      "nll_loss": 1.6443783044815063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16115859150886536,
      "rewards/margins": 0.04415808618068695,
      "rewards/rejected": -0.2053166925907135,
      "step": 382
    },
    {
      "epoch": 0.6036249014972419,
      "grad_norm": 0.172901913523674,
      "learning_rate": 3.528549635813778e-06,
      "log_odds_chosen": 0.5418984293937683,
      "log_odds_ratio": -0.4654310643672943,
      "logits/chosen": -0.15981490910053253,
      "logits/rejected": -1.3621118068695068,
      "logps/chosen": -1.4450870752334595,
      "logps/rejected": -1.8853989839553833,
      "loss": 1.5496,
      "nll_loss": 1.5030344724655151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1445087045431137,
      "rewards/margins": 0.04403119534254074,
      "rewards/rejected": -0.18853990733623505,
      "step": 383
    },
    {
      "epoch": 0.6052009456264775,
      "grad_norm": 0.19568444788455963,
      "learning_rate": 3.524998837927192e-06,
      "log_odds_chosen": 0.572688639163971,
      "log_odds_ratio": -0.4493398666381836,
      "logits/chosen": -0.11590529978275299,
      "logits/rejected": -1.2641996145248413,
      "logps/chosen": -1.5668047666549683,
      "logps/rejected": -2.04402756690979,
      "loss": 1.6464,
      "nll_loss": 1.6014786958694458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15668046474456787,
      "rewards/margins": 0.04772229492664337,
      "rewards/rejected": -0.20440277457237244,
      "step": 384
    },
    {
      "epoch": 0.6067769897557131,
      "grad_norm": 0.17970573902130127,
      "learning_rate": 3.5214365194027797e-06,
      "log_odds_chosen": 0.5927567481994629,
      "log_odds_ratio": -0.4437498152256012,
      "logits/chosen": -0.1304638385772705,
      "logits/rejected": -1.3869986534118652,
      "logps/chosen": -1.4864726066589355,
      "logps/rejected": -1.9743757247924805,
      "loss": 1.579,
      "nll_loss": 1.5346585512161255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486472636461258,
      "rewards/margins": 0.04879031702876091,
      "rewards/rejected": -0.19743755459785461,
      "step": 385
    },
    {
      "epoch": 0.6083530338849488,
      "grad_norm": 0.209872767329216,
      "learning_rate": 3.517862707152157e-06,
      "log_odds_chosen": 0.43204668164253235,
      "log_odds_ratio": -0.5081393122673035,
      "logits/chosen": -0.04916413873434067,
      "logits/rejected": -1.1377692222595215,
      "logps/chosen": -1.6241986751556396,
      "logps/rejected": -1.984796166419983,
      "loss": 1.7062,
      "nll_loss": 1.6553831100463867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.162419855594635,
      "rewards/margins": 0.036059752106666565,
      "rewards/rejected": -0.19847962260246277,
      "step": 386
    },
    {
      "epoch": 0.6099290780141844,
      "grad_norm": 0.21656852960586548,
      "learning_rate": 3.5142774281737674e-06,
      "log_odds_chosen": 0.6231499314308167,
      "log_odds_ratio": -0.43155157566070557,
      "logits/chosen": -0.13221031427383423,
      "logits/rejected": -1.25809907913208,
      "logps/chosen": -1.6151927709579468,
      "logps/rejected": -2.1396982669830322,
      "loss": 1.6916,
      "nll_loss": 1.6484118700027466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16151930391788483,
      "rewards/margins": 0.05245053023099899,
      "rewards/rejected": -0.21396984159946442,
      "step": 387
    },
    {
      "epoch": 0.61150512214342,
      "grad_norm": 0.17429621517658234,
      "learning_rate": 3.5106807095526817e-06,
      "log_odds_chosen": 0.626970112323761,
      "log_odds_ratio": -0.43910783529281616,
      "logits/chosen": -0.14420942962169647,
      "logits/rejected": -1.505774974822998,
      "logps/chosen": -1.5748193264007568,
      "logps/rejected": -2.0963521003723145,
      "loss": 1.6681,
      "nll_loss": 1.62421715259552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15748193860054016,
      "rewards/margins": 0.052153266966342926,
      "rewards/rejected": -0.20963521301746368,
      "step": 388
    },
    {
      "epoch": 0.6130811662726556,
      "grad_norm": 0.18532711267471313,
      "learning_rate": 3.5070725784603905e-06,
      "log_odds_chosen": 0.5388447642326355,
      "log_odds_ratio": -0.4668474793434143,
      "logits/chosen": -0.21902410686016083,
      "logits/rejected": -1.2050061225891113,
      "logps/chosen": -1.4473650455474854,
      "logps/rejected": -1.8858047723770142,
      "loss": 1.5341,
      "nll_loss": 1.4874001741409302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14473651349544525,
      "rewards/margins": 0.04384396970272064,
      "rewards/rejected": -0.1885804831981659,
      "step": 389
    },
    {
      "epoch": 0.6146572104018913,
      "grad_norm": 0.20954233407974243,
      "learning_rate": 3.503453062154602e-06,
      "log_odds_chosen": 0.45336735248565674,
      "log_odds_ratio": -0.5033838748931885,
      "logits/chosen": -0.138980433344841,
      "logits/rejected": -1.1668821573257446,
      "logps/chosen": -1.611983060836792,
      "logps/rejected": -1.9972047805786133,
      "loss": 1.7107,
      "nll_loss": 1.6603914499282837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16119830310344696,
      "rewards/margins": 0.03852216154336929,
      "rewards/rejected": -0.19972047209739685,
      "step": 390
    },
    {
      "epoch": 0.6162332545311269,
      "grad_norm": 0.19238236546516418,
      "learning_rate": 3.499822187979032e-06,
      "log_odds_chosen": 0.45051056146621704,
      "log_odds_ratio": -0.502297043800354,
      "logits/chosen": -0.07424494624137878,
      "logits/rejected": -1.422592043876648,
      "logps/chosen": -1.5691637992858887,
      "logps/rejected": -1.9431030750274658,
      "loss": 1.6718,
      "nll_loss": 1.6215293407440186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15691637992858887,
      "rewards/margins": 0.03739393502473831,
      "rewards/rejected": -0.19431032240390778,
      "step": 391
    },
    {
      "epoch": 0.6178092986603625,
      "grad_norm": 0.1847350001335144,
      "learning_rate": 3.496179983363202e-06,
      "log_odds_chosen": 0.39740023016929626,
      "log_odds_ratio": -0.5181854963302612,
      "logits/chosen": -0.08911152184009552,
      "logits/rejected": -1.3096749782562256,
      "logps/chosen": -1.5776864290237427,
      "logps/rejected": -1.9072067737579346,
      "loss": 1.6621,
      "nll_loss": 1.610294222831726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1577686369419098,
      "rewards/margins": 0.03295203298330307,
      "rewards/rejected": -0.19072067737579346,
      "step": 392
    },
    {
      "epoch": 0.6193853427895981,
      "grad_norm": 0.20399923622608185,
      "learning_rate": 3.4925264758222268e-06,
      "log_odds_chosen": 0.6117810010910034,
      "log_odds_ratio": -0.43658775091171265,
      "logits/chosen": -0.14838597178459167,
      "logits/rejected": -1.166914939880371,
      "logps/chosen": -1.507293701171875,
      "logps/rejected": -2.0140793323516846,
      "loss": 1.6041,
      "nll_loss": 1.5604305267333984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15072937309741974,
      "rewards/margins": 0.050678566098213196,
      "rewards/rejected": -0.20140793919563293,
      "step": 393
    },
    {
      "epoch": 0.6209613869188337,
      "grad_norm": 0.38816723227500916,
      "learning_rate": 3.488861692956611e-06,
      "log_odds_chosen": 0.5418617129325867,
      "log_odds_ratio": -0.46506166458129883,
      "logits/chosen": -0.1680249571800232,
      "logits/rejected": -1.281784176826477,
      "logps/chosen": -1.5487433671951294,
      "logps/rejected": -2.0000991821289062,
      "loss": 1.6392,
      "nll_loss": 1.5926973819732666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15487435460090637,
      "rewards/margins": 0.04513557255268097,
      "rewards/rejected": -0.20000992715358734,
      "step": 394
    },
    {
      "epoch": 0.6225374310480694,
      "grad_norm": 0.1981612592935562,
      "learning_rate": 3.4851856624520394e-06,
      "log_odds_chosen": 0.5252317786216736,
      "log_odds_ratio": -0.4674740135669708,
      "logits/chosen": -0.1143113300204277,
      "logits/rejected": -1.3392878770828247,
      "logps/chosen": -1.5815508365631104,
      "logps/rejected": -2.017178773880005,
      "loss": 1.6669,
      "nll_loss": 1.620198130607605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15815508365631104,
      "rewards/margins": 0.04356277734041214,
      "rewards/rejected": -0.20171786844730377,
      "step": 395
    },
    {
      "epoch": 0.624113475177305,
      "grad_norm": 0.20509879291057587,
      "learning_rate": 3.4814984120791664e-06,
      "log_odds_chosen": 0.497008353471756,
      "log_odds_ratio": -0.47801053524017334,
      "logits/chosen": -0.11156664043664932,
      "logits/rejected": -1.1910896301269531,
      "logps/chosen": -1.553000807762146,
      "logps/rejected": -1.962215542793274,
      "loss": 1.6485,
      "nll_loss": 1.6007264852523804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553000807762146,
      "rewards/margins": 0.04092148691415787,
      "rewards/rejected": -0.19622156023979187,
      "step": 396
    },
    {
      "epoch": 0.6256895193065406,
      "grad_norm": 0.18052570521831512,
      "learning_rate": 3.477799969693407e-06,
      "log_odds_chosen": 0.4731709361076355,
      "log_odds_ratio": -0.49316927790641785,
      "logits/chosen": -0.07338425517082214,
      "logits/rejected": -1.334223747253418,
      "logps/chosen": -1.5508506298065186,
      "logps/rejected": -1.936098575592041,
      "loss": 1.6401,
      "nll_loss": 1.5908257961273193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15508508682250977,
      "rewards/margins": 0.03852478042244911,
      "rewards/rejected": -0.19360986351966858,
      "step": 397
    },
    {
      "epoch": 0.6272655634357762,
      "grad_norm": 0.20021112263202667,
      "learning_rate": 3.474090363234728e-06,
      "log_odds_chosen": 0.7609047889709473,
      "log_odds_ratio": -0.3980966806411743,
      "logits/chosen": -0.24316024780273438,
      "logits/rejected": -1.3844587802886963,
      "logps/chosen": -1.4845788478851318,
      "logps/rejected": -2.117990016937256,
      "loss": 1.5681,
      "nll_loss": 1.528298020362854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14845788478851318,
      "rewards/margins": 0.06334113329648972,
      "rewards/rejected": -0.2117990255355835,
      "step": 398
    },
    {
      "epoch": 0.6288416075650118,
      "grad_norm": 0.19712619483470917,
      "learning_rate": 3.4703696207274325e-06,
      "log_odds_chosen": 0.5112850666046143,
      "log_odds_ratio": -0.4771292209625244,
      "logits/chosen": -0.12579302489757538,
      "logits/rejected": -1.206828236579895,
      "logps/chosen": -1.5673775672912598,
      "logps/rejected": -1.9893169403076172,
      "loss": 1.6413,
      "nll_loss": 1.593619704246521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15673774480819702,
      "rewards/margins": 0.0421939492225647,
      "rewards/rejected": -0.19893169403076172,
      "step": 399
    },
    {
      "epoch": 0.6304176516942475,
      "grad_norm": 0.18029263615608215,
      "learning_rate": 3.4666377702799545e-06,
      "log_odds_chosen": 0.5222339630126953,
      "log_odds_ratio": -0.47047317028045654,
      "logits/chosen": -0.23173952102661133,
      "logits/rejected": -1.5240064859390259,
      "logps/chosen": -1.4771634340286255,
      "logps/rejected": -1.9015573263168335,
      "loss": 1.5785,
      "nll_loss": 1.5314639806747437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14771634340286255,
      "rewards/margins": 0.04243940860033035,
      "rewards/rejected": -0.1901557445526123,
      "step": 400
    },
    {
      "epoch": 0.6319936958234831,
      "grad_norm": 0.19452960789203644,
      "learning_rate": 3.4628948400846417e-06,
      "log_odds_chosen": 0.6402549147605896,
      "log_odds_ratio": -0.43367528915405273,
      "logits/chosen": -0.15246199071407318,
      "logits/rejected": -1.4411405324935913,
      "logps/chosen": -1.5569361448287964,
      "logps/rejected": -2.0934228897094727,
      "loss": 1.6454,
      "nll_loss": 1.60203218460083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15569362044334412,
      "rewards/margins": 0.053648658096790314,
      "rewards/rejected": -0.20934227108955383,
      "step": 401
    },
    {
      "epoch": 0.6335697399527187,
      "grad_norm": 0.2060081660747528,
      "learning_rate": 3.4591408584175426e-06,
      "log_odds_chosen": 0.5116357207298279,
      "log_odds_ratio": -0.4790383577346802,
      "logits/chosen": -0.1661422997713089,
      "logits/rejected": -1.3209583759307861,
      "logps/chosen": -1.5975854396820068,
      "logps/rejected": -2.0274882316589355,
      "loss": 1.6893,
      "nll_loss": 1.6414045095443726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1597585529088974,
      "rewards/margins": 0.042990271002054214,
      "rewards/rejected": -0.20274882018566132,
      "step": 402
    },
    {
      "epoch": 0.6351457840819543,
      "grad_norm": 0.1968107670545578,
      "learning_rate": 3.4553758536381974e-06,
      "log_odds_chosen": 0.5235389471054077,
      "log_odds_ratio": -0.47713008522987366,
      "logits/chosen": -0.15526294708251953,
      "logits/rejected": -1.3401628732681274,
      "logps/chosen": -1.554589033126831,
      "logps/rejected": -1.9898911714553833,
      "loss": 1.6418,
      "nll_loss": 1.5940942764282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15545889735221863,
      "rewards/margins": 0.04353022575378418,
      "rewards/rejected": -0.198989138007164,
      "step": 403
    },
    {
      "epoch": 0.6367218282111899,
      "grad_norm": 0.2039954662322998,
      "learning_rate": 3.451599854189418e-06,
      "log_odds_chosen": 0.5396761894226074,
      "log_odds_ratio": -0.46344685554504395,
      "logits/chosen": -0.11397916078567505,
      "logits/rejected": -0.9760464429855347,
      "logps/chosen": -1.5517972707748413,
      "logps/rejected": -1.998788595199585,
      "loss": 1.6507,
      "nll_loss": 1.6043689250946045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15517973899841309,
      "rewards/margins": 0.04469913989305496,
      "rewards/rejected": -0.19987887144088745,
      "step": 404
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 0.19308118522167206,
      "learning_rate": 3.4478128885970765e-06,
      "log_odds_chosen": 0.6018125414848328,
      "log_odds_ratio": -0.4381783902645111,
      "logits/chosen": -0.16679120063781738,
      "logits/rejected": -1.3316242694854736,
      "logps/chosen": -1.6154249906539917,
      "logps/rejected": -2.122253894805908,
      "loss": 1.7001,
      "nll_loss": 1.6562443971633911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16154250502586365,
      "rewards/margins": 0.05068288370966911,
      "rewards/rejected": -0.21222539246082306,
      "step": 405
    },
    {
      "epoch": 0.6398739164696612,
      "grad_norm": 0.19146420061588287,
      "learning_rate": 3.44401498546989e-06,
      "log_odds_chosen": 0.4198459982872009,
      "log_odds_ratio": -0.5137929320335388,
      "logits/chosen": -0.10405167937278748,
      "logits/rejected": -1.477805733680725,
      "logps/chosen": -1.5691133737564087,
      "logps/rejected": -1.9161850214004517,
      "loss": 1.6395,
      "nll_loss": 1.5880873203277588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15691134333610535,
      "rewards/margins": 0.03470717370510101,
      "rewards/rejected": -0.19161850214004517,
      "step": 406
    },
    {
      "epoch": 0.6414499605988968,
      "grad_norm": 0.19537797570228577,
      "learning_rate": 3.4402061734992005e-06,
      "log_odds_chosen": 0.5202116370201111,
      "log_odds_ratio": -0.47119492292404175,
      "logits/chosen": -0.1433197259902954,
      "logits/rejected": -1.3762876987457275,
      "logps/chosen": -1.5083684921264648,
      "logps/rejected": -1.9353201389312744,
      "loss": 1.6033,
      "nll_loss": 1.5561641454696655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15083685517311096,
      "rewards/margins": 0.042695142328739166,
      "rewards/rejected": -0.19353200495243073,
      "step": 407
    },
    {
      "epoch": 0.6430260047281324,
      "grad_norm": 0.20898084342479706,
      "learning_rate": 3.4363864814587656e-06,
      "log_odds_chosen": 0.49365922808647156,
      "log_odds_ratio": -0.4801666736602783,
      "logits/chosen": -0.25102052092552185,
      "logits/rejected": -0.9237027764320374,
      "logps/chosen": -1.4852203130722046,
      "logps/rejected": -1.8896757364273071,
      "loss": 1.582,
      "nll_loss": 1.5339844226837158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1485220193862915,
      "rewards/margins": 0.04044555127620697,
      "rewards/rejected": -0.18896758556365967,
      "step": 408
    },
    {
      "epoch": 0.644602048857368,
      "grad_norm": 0.2019755095243454,
      "learning_rate": 3.4325559382045343e-06,
      "log_odds_chosen": 0.4014512598514557,
      "log_odds_ratio": -0.5167407989501953,
      "logits/chosen": -0.22251050174236298,
      "logits/rejected": -1.485706090927124,
      "logps/chosen": -1.4902509450912476,
      "logps/rejected": -1.812532663345337,
      "loss": 1.581,
      "nll_loss": 1.5293045043945312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149025097489357,
      "rewards/margins": 0.032228171825408936,
      "rewards/rejected": -0.18125325441360474,
      "step": 409
    },
    {
      "epoch": 0.6461780929866037,
      "grad_norm": 0.20751270651817322,
      "learning_rate": 3.4287145726744295e-06,
      "log_odds_chosen": 0.5322403907775879,
      "log_odds_ratio": -0.46528854966163635,
      "logits/chosen": -0.24800266325473785,
      "logits/rejected": -1.3249245882034302,
      "logps/chosen": -1.478865623474121,
      "logps/rejected": -1.912164568901062,
      "loss": 1.5805,
      "nll_loss": 1.5339622497558594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14788655936717987,
      "rewards/margins": 0.0433298759162426,
      "rewards/rejected": -0.19121645390987396,
      "step": 410
    },
    {
      "epoch": 0.6477541371158393,
      "grad_norm": 0.20497266948223114,
      "learning_rate": 3.4248624138881335e-06,
      "log_odds_chosen": 0.4167112112045288,
      "log_odds_ratio": -0.5280964374542236,
      "logits/chosen": -0.146779865026474,
      "logits/rejected": -1.1234551668167114,
      "logps/chosen": -1.615991473197937,
      "logps/rejected": -1.9755668640136719,
      "loss": 1.7015,
      "nll_loss": 1.6486729383468628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16159915924072266,
      "rewards/margins": 0.035957541316747665,
      "rewards/rejected": -0.19755667448043823,
      "step": 411
    },
    {
      "epoch": 0.6493301812450749,
      "grad_norm": 0.2031044065952301,
      "learning_rate": 3.4209994909468672e-06,
      "log_odds_chosen": 0.6441755294799805,
      "log_odds_ratio": -0.4322493076324463,
      "logits/chosen": -0.27414995431900024,
      "logits/rejected": -1.0003337860107422,
      "logps/chosen": -1.5170173645019531,
      "logps/rejected": -2.0522003173828125,
      "loss": 1.6035,
      "nll_loss": 1.5602253675460815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15170171856880188,
      "rewards/margins": 0.05351831763982773,
      "rewards/rejected": -0.2052200585603714,
      "step": 412
    },
    {
      "epoch": 0.6509062253743105,
      "grad_norm": 0.20105931162834167,
      "learning_rate": 3.4171258330331667e-06,
      "log_odds_chosen": 0.43131789565086365,
      "log_odds_ratio": -0.5029494762420654,
      "logits/chosen": -0.11574351787567139,
      "logits/rejected": -1.0839128494262695,
      "logps/chosen": -1.603833556175232,
      "logps/rejected": -1.9600427150726318,
      "loss": 1.6866,
      "nll_loss": 1.6363379955291748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16038337349891663,
      "rewards/margins": 0.035620901733636856,
      "rewards/rejected": -0.19600427150726318,
      "step": 413
    },
    {
      "epoch": 0.6524822695035462,
      "grad_norm": 0.1987302601337433,
      "learning_rate": 3.4132414694106684e-06,
      "log_odds_chosen": 0.6570238471031189,
      "log_odds_ratio": -0.425781786441803,
      "logits/chosen": -0.15273043513298035,
      "logits/rejected": -1.4411128759384155,
      "logps/chosen": -1.510444164276123,
      "logps/rejected": -2.0561532974243164,
      "loss": 1.5935,
      "nll_loss": 1.550927758216858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15104441344738007,
      "rewards/margins": 0.05457089841365814,
      "rewards/rejected": -0.2056153267621994,
      "step": 414
    },
    {
      "epoch": 0.6540583136327817,
      "grad_norm": 0.20442619919776917,
      "learning_rate": 3.409346429423884e-06,
      "log_odds_chosen": 0.45281243324279785,
      "log_odds_ratio": -0.4946718215942383,
      "logits/chosen": -0.008420580998063087,
      "logits/rejected": -1.278180480003357,
      "logps/chosen": -1.5783418416976929,
      "logps/rejected": -1.952226996421814,
      "loss": 1.6571,
      "nll_loss": 1.607609748840332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15783418715000153,
      "rewards/margins": 0.03738852217793465,
      "rewards/rejected": -0.19522270560264587,
      "step": 415
    },
    {
      "epoch": 0.6556343577620173,
      "grad_norm": 0.1937103271484375,
      "learning_rate": 3.40544074249798e-06,
      "log_odds_chosen": 0.6057698726654053,
      "log_odds_ratio": -0.43932780623435974,
      "logits/chosen": -0.18488506972789764,
      "logits/rejected": -1.447567343711853,
      "logps/chosen": -1.568244218826294,
      "logps/rejected": -2.074944496154785,
      "loss": 1.6516,
      "nll_loss": 1.607670545578003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15682442486286163,
      "rewards/margins": 0.05067002773284912,
      "rewards/rejected": -0.20749445259571075,
      "step": 416
    },
    {
      "epoch": 0.6572104018912529,
      "grad_norm": 0.21314595639705658,
      "learning_rate": 3.401524438138556e-06,
      "log_odds_chosen": 0.46506115794181824,
      "log_odds_ratio": -0.4945875406265259,
      "logits/chosen": -0.12872271239757538,
      "logits/rejected": -1.0993512868881226,
      "logps/chosen": -1.5732535123825073,
      "logps/rejected": -1.9601188898086548,
      "loss": 1.6377,
      "nll_loss": 1.5882155895233154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573253571987152,
      "rewards/margins": 0.03868655115365982,
      "rewards/rejected": -0.19601190090179443,
      "step": 417
    },
    {
      "epoch": 0.6587864460204885,
      "grad_norm": 0.20356151461601257,
      "learning_rate": 3.39759754593142e-06,
      "log_odds_chosen": 0.49928921461105347,
      "log_odds_ratio": -0.4814523756504059,
      "logits/chosen": -0.18617013096809387,
      "logits/rejected": -1.2594208717346191,
      "logps/chosen": -1.6336315870285034,
      "logps/rejected": -2.05531644821167,
      "loss": 1.7071,
      "nll_loss": 1.6589891910552979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16336317360401154,
      "rewards/margins": 0.04216848313808441,
      "rewards/rejected": -0.20553165674209595,
      "step": 418
    },
    {
      "epoch": 0.6603624901497241,
      "grad_norm": 0.1998877227306366,
      "learning_rate": 3.3936600955423683e-06,
      "log_odds_chosen": 0.703155517578125,
      "log_odds_ratio": -0.4173457622528076,
      "logits/chosen": -0.10339093208312988,
      "logits/rejected": -1.0794388055801392,
      "logps/chosen": -1.526084542274475,
      "logps/rejected": -2.111715078353882,
      "loss": 1.613,
      "nll_loss": 1.57127845287323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526084691286087,
      "rewards/margins": 0.05856304615736008,
      "rewards/rejected": -0.21117150783538818,
      "step": 419
    },
    {
      "epoch": 0.6619385342789598,
      "grad_norm": 0.1923801749944687,
      "learning_rate": 3.3897121167169573e-06,
      "log_odds_chosen": 0.3884754776954651,
      "log_odds_ratio": -0.5280902981758118,
      "logits/chosen": -0.2249905914068222,
      "logits/rejected": -1.2855345010757446,
      "logps/chosen": -1.499492883682251,
      "logps/rejected": -1.8159973621368408,
      "loss": 1.5979,
      "nll_loss": 1.545102596282959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14994929730892181,
      "rewards/margins": 0.031650442630052567,
      "rewards/rejected": -0.18159973621368408,
      "step": 420
    },
    {
      "epoch": 0.6635145784081954,
      "grad_norm": 0.20332655310630798,
      "learning_rate": 3.38575363928028e-06,
      "log_odds_chosen": 0.5980296730995178,
      "log_odds_ratio": -0.444957435131073,
      "logits/chosen": -0.1456383615732193,
      "logits/rejected": -1.1062229871749878,
      "logps/chosen": -1.5091032981872559,
      "logps/rejected": -2.0027518272399902,
      "loss": 1.5809,
      "nll_loss": 1.5364110469818115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15091031789779663,
      "rewards/margins": 0.04936487227678299,
      "rewards/rejected": -0.20027519762516022,
      "step": 421
    },
    {
      "epoch": 0.665090622537431,
      "grad_norm": 0.19581378996372223,
      "learning_rate": 3.3817846931367452e-06,
      "log_odds_chosen": 0.40414246916770935,
      "log_odds_ratio": -0.5155224800109863,
      "logits/chosen": -0.22978416085243225,
      "logits/rejected": -1.1638431549072266,
      "logps/chosen": -1.4919337034225464,
      "logps/rejected": -1.8167966604232788,
      "loss": 1.5812,
      "nll_loss": 1.5296943187713623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14919337630271912,
      "rewards/margins": 0.032486289739608765,
      "rewards/rejected": -0.18167968094348907,
      "step": 422
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 0.18812742829322815,
      "learning_rate": 3.377805308269844e-06,
      "log_odds_chosen": 0.6795855760574341,
      "log_odds_ratio": -0.42459002137184143,
      "logits/chosen": -0.11801916360855103,
      "logits/rejected": -1.4967668056488037,
      "logps/chosen": -1.54996919631958,
      "logps/rejected": -2.1208114624023438,
      "loss": 1.6253,
      "nll_loss": 1.5828850269317627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15499693155288696,
      "rewards/margins": 0.05708422511816025,
      "rewards/rejected": -0.21208113431930542,
      "step": 423
    },
    {
      "epoch": 0.6682427107959023,
      "grad_norm": 0.18950577080249786,
      "learning_rate": 3.3738155147419275e-06,
      "log_odds_chosen": 0.6117570996284485,
      "log_odds_ratio": -0.44386616349220276,
      "logits/chosen": -0.24225133657455444,
      "logits/rejected": -1.1781508922576904,
      "logps/chosen": -1.5445609092712402,
      "logps/rejected": -2.0556564331054688,
      "loss": 1.6058,
      "nll_loss": 1.561427354812622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15445609390735626,
      "rewards/margins": 0.05110953003168106,
      "rewards/rejected": -0.20556563138961792,
      "step": 424
    },
    {
      "epoch": 0.6698187549251379,
      "grad_norm": 0.2043224424123764,
      "learning_rate": 3.3698153426939824e-06,
      "log_odds_chosen": 0.6969811916351318,
      "log_odds_ratio": -0.4145274758338928,
      "logits/chosen": -0.2313898205757141,
      "logits/rejected": -1.0220508575439453,
      "logps/chosen": -1.473904013633728,
      "logps/rejected": -2.0466439723968506,
      "loss": 1.5474,
      "nll_loss": 1.5059936046600342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14739039540290833,
      "rewards/margins": 0.05727401375770569,
      "rewards/rejected": -0.204664409160614,
      "step": 425
    },
    {
      "epoch": 0.6713947990543735,
      "grad_norm": 0.20883503556251526,
      "learning_rate": 3.3658048223453954e-06,
      "log_odds_chosen": 0.6612953543663025,
      "log_odds_ratio": -0.42457008361816406,
      "logits/chosen": -0.19671984016895294,
      "logits/rejected": -1.3579217195510864,
      "logps/chosen": -1.6156816482543945,
      "logps/rejected": -2.1756107807159424,
      "loss": 1.6623,
      "nll_loss": 1.6198010444641113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16156817972660065,
      "rewards/margins": 0.05599290132522583,
      "rewards/rejected": -0.21756106615066528,
      "step": 426
    },
    {
      "epoch": 0.6729708431836091,
      "grad_norm": 0.19400307536125183,
      "learning_rate": 3.3617839839937337e-06,
      "log_odds_chosen": 0.6474947333335876,
      "log_odds_ratio": -0.431326687335968,
      "logits/chosen": -0.11850985884666443,
      "logits/rejected": -1.453221321105957,
      "logps/chosen": -1.5760408639907837,
      "logps/rejected": -2.1231493949890137,
      "loss": 1.6424,
      "nll_loss": 1.59929358959198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15760409832000732,
      "rewards/margins": 0.054710857570171356,
      "rewards/rejected": -0.2123149335384369,
      "step": 427
    },
    {
      "epoch": 0.6745468873128447,
      "grad_norm": 0.20923368632793427,
      "learning_rate": 3.3577528580145107e-06,
      "log_odds_chosen": 0.33366745710372925,
      "log_odds_ratio": -0.5432456731796265,
      "logits/chosen": -0.14751064777374268,
      "logits/rejected": -1.2790288925170898,
      "logps/chosen": -1.5968459844589233,
      "logps/rejected": -1.8737503290176392,
      "loss": 1.678,
      "nll_loss": 1.6236892938613892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15968461334705353,
      "rewards/margins": 0.027690425515174866,
      "rewards/rejected": -0.1873750239610672,
      "step": 428
    },
    {
      "epoch": 0.6761229314420804,
      "grad_norm": 0.1988777220249176,
      "learning_rate": 3.353711474860956e-06,
      "log_odds_chosen": 0.5849066376686096,
      "log_odds_ratio": -0.45891109108924866,
      "logits/chosen": -0.18468442559242249,
      "logits/rejected": -1.206817865371704,
      "logps/chosen": -1.5448060035705566,
      "logps/rejected": -2.038527011871338,
      "loss": 1.6061,
      "nll_loss": 1.560197353363037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15448060631752014,
      "rewards/margins": 0.04937208443880081,
      "rewards/rejected": -0.20385268330574036,
      "step": 429
    },
    {
      "epoch": 0.677698975571316,
      "grad_norm": 0.1907055526971817,
      "learning_rate": 3.3496598650637916e-06,
      "log_odds_chosen": 0.5627853274345398,
      "log_odds_ratio": -0.45737147331237793,
      "logits/chosen": -0.213260680437088,
      "logits/rejected": -1.1075356006622314,
      "logps/chosen": -1.4453046321868896,
      "logps/rejected": -1.9003225564956665,
      "loss": 1.5279,
      "nll_loss": 1.4821749925613403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14453044533729553,
      "rewards/margins": 0.04550181329250336,
      "rewards/rejected": -0.19003227353096008,
      "step": 430
    },
    {
      "epoch": 0.6792750197005516,
      "grad_norm": 0.1916009783744812,
      "learning_rate": 3.3455980592309923e-06,
      "log_odds_chosen": 0.6095476150512695,
      "log_odds_ratio": -0.44210147857666016,
      "logits/chosen": -0.23830467462539673,
      "logits/rejected": -1.2122029066085815,
      "logps/chosen": -1.4246712923049927,
      "logps/rejected": -1.921633005142212,
      "loss": 1.515,
      "nll_loss": 1.4708294868469238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14246714115142822,
      "rewards/margins": 0.04969616234302521,
      "rewards/rejected": -0.19216330349445343,
      "step": 431
    },
    {
      "epoch": 0.6808510638297872,
      "grad_norm": 0.2036287635564804,
      "learning_rate": 3.341526088047562e-06,
      "log_odds_chosen": 0.5544840693473816,
      "log_odds_ratio": -0.46283283829689026,
      "logits/chosen": -0.18271717429161072,
      "logits/rejected": -1.2361128330230713,
      "logps/chosen": -1.612389326095581,
      "logps/rejected": -2.080324411392212,
      "loss": 1.6715,
      "nll_loss": 1.6252350807189941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16123893857002258,
      "rewards/margins": 0.046793513000011444,
      "rewards/rejected": -0.20803247392177582,
      "step": 432
    },
    {
      "epoch": 0.6824271079590228,
      "grad_norm": 0.21691077947616577,
      "learning_rate": 3.3374439822752972e-06,
      "log_odds_chosen": 0.4269826412200928,
      "log_odds_ratio": -0.5059152841567993,
      "logits/chosen": -0.1358519196510315,
      "logits/rejected": -1.0953370332717896,
      "logps/chosen": -1.5720406770706177,
      "logps/rejected": -1.927139401435852,
      "loss": 1.6598,
      "nll_loss": 1.609249472618103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15720407664775848,
      "rewards/margins": 0.03550989180803299,
      "rewards/rejected": -0.19271396100521088,
      "step": 433
    },
    {
      "epoch": 0.6840031520882585,
      "grad_norm": 0.25961580872535706,
      "learning_rate": 3.333351772752559e-06,
      "log_odds_chosen": 0.5796866416931152,
      "log_odds_ratio": -0.4593260884284973,
      "logits/chosen": -0.2504950165748596,
      "logits/rejected": -1.251611351966858,
      "logps/chosen": -1.6392018795013428,
      "logps/rejected": -2.125319004058838,
      "loss": 1.7019,
      "nll_loss": 1.655974268913269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16392017900943756,
      "rewards/margins": 0.048611730337142944,
      "rewards/rejected": -0.2125319093465805,
      "step": 434
    },
    {
      "epoch": 0.6855791962174941,
      "grad_norm": 0.20323769748210907,
      "learning_rate": 3.3292494903940338e-06,
      "log_odds_chosen": 0.6984251141548157,
      "log_odds_ratio": -0.41105496883392334,
      "logits/chosen": -0.16568951308727264,
      "logits/rejected": -1.2594187259674072,
      "logps/chosen": -1.5446348190307617,
      "logps/rejected": -2.125537395477295,
      "loss": 1.6162,
      "nll_loss": 1.5750484466552734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544634848833084,
      "rewards/margins": 0.058090247213840485,
      "rewards/rejected": -0.2125537395477295,
      "step": 435
    },
    {
      "epoch": 0.6871552403467297,
      "grad_norm": 0.18550781905651093,
      "learning_rate": 3.3251371661905063e-06,
      "log_odds_chosen": 0.5926481485366821,
      "log_odds_ratio": -0.45173323154449463,
      "logits/chosen": -0.19779422879219055,
      "logits/rejected": -1.1184755563735962,
      "logps/chosen": -1.3700029850006104,
      "logps/rejected": -1.8533730506896973,
      "loss": 1.4567,
      "nll_loss": 1.4115686416625977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13700029253959656,
      "rewards/margins": 0.04833700880408287,
      "rewards/rejected": -0.18533730506896973,
      "step": 436
    },
    {
      "epoch": 0.6887312844759653,
      "grad_norm": 0.19023758172988892,
      "learning_rate": 3.321014831208622e-06,
      "log_odds_chosen": 0.5900598764419556,
      "log_odds_ratio": -0.44593122601509094,
      "logits/chosen": -0.20720794796943665,
      "logits/rejected": -1.0833600759506226,
      "logps/chosen": -1.4328017234802246,
      "logps/rejected": -1.9078587293624878,
      "loss": 1.5262,
      "nll_loss": 1.4815677404403687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14328016340732574,
      "rewards/margins": 0.047505710273981094,
      "rewards/rejected": -0.19078589975833893,
      "step": 437
    },
    {
      "epoch": 0.6903073286052009,
      "grad_norm": 0.18916691839694977,
      "learning_rate": 3.316882516590652e-06,
      "log_odds_chosen": 0.6026834845542908,
      "log_odds_ratio": -0.43881577253341675,
      "logits/chosen": -0.19735684990882874,
      "logits/rejected": -1.3404430150985718,
      "logps/chosen": -1.4774234294891357,
      "logps/rejected": -1.9714411497116089,
      "loss": 1.5584,
      "nll_loss": 1.5145463943481445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477423459291458,
      "rewards/margins": 0.04940177500247955,
      "rewards/rejected": -0.19714412093162537,
      "step": 438
    },
    {
      "epoch": 0.6918833727344366,
      "grad_norm": 0.2226303368806839,
      "learning_rate": 3.31274025355426e-06,
      "log_odds_chosen": 0.5777285099029541,
      "log_odds_ratio": -0.45020100474357605,
      "logits/chosen": -0.2297353744506836,
      "logits/rejected": -1.1321128606796265,
      "logps/chosen": -1.5695414543151855,
      "logps/rejected": -2.051584243774414,
      "loss": 1.6201,
      "nll_loss": 1.57511568069458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15695415437221527,
      "rewards/margins": 0.04820426553487778,
      "rewards/rejected": -0.20515841245651245,
      "step": 439
    },
    {
      "epoch": 0.6934594168636722,
      "grad_norm": 0.20178461074829102,
      "learning_rate": 3.308588073392265e-06,
      "log_odds_chosen": 0.5347023606300354,
      "log_odds_ratio": -0.4697195291519165,
      "logits/chosen": -0.22104787826538086,
      "logits/rejected": -1.1682895421981812,
      "logps/chosen": -1.5996606349945068,
      "logps/rejected": -2.0496530532836914,
      "loss": 1.6537,
      "nll_loss": 1.606750249862671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15996606647968292,
      "rewards/margins": 0.04499925673007965,
      "rewards/rejected": -0.20496532320976257,
      "step": 440
    },
    {
      "epoch": 0.6950354609929078,
      "grad_norm": 0.19244441390037537,
      "learning_rate": 3.3044260074724035e-06,
      "log_odds_chosen": 0.6290772557258606,
      "log_odds_ratio": -0.438933789730072,
      "logits/chosen": -0.21869786083698273,
      "logits/rejected": -1.3959290981292725,
      "logps/chosen": -1.5120939016342163,
      "logps/rejected": -2.030512809753418,
      "loss": 1.588,
      "nll_loss": 1.5441324710845947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15120939910411835,
      "rewards/margins": 0.05184189975261688,
      "rewards/rejected": -0.20305128395557404,
      "step": 441
    },
    {
      "epoch": 0.6966115051221434,
      "grad_norm": 0.1978086680173874,
      "learning_rate": 3.300254087237097e-06,
      "log_odds_chosen": 0.5377320647239685,
      "log_odds_ratio": -0.46433717012405396,
      "logits/chosen": -0.18392041325569153,
      "logits/rejected": -1.2981102466583252,
      "logps/chosen": -1.4015955924987793,
      "logps/rejected": -1.8341153860092163,
      "loss": 1.5073,
      "nll_loss": 1.460841417312622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14015956223011017,
      "rewards/margins": 0.04325197637081146,
      "rewards/rejected": -0.18341155350208282,
      "step": 442
    },
    {
      "epoch": 0.698187549251379,
      "grad_norm": 0.19712939858436584,
      "learning_rate": 3.2960723442032105e-06,
      "log_odds_chosen": 0.7022231817245483,
      "log_odds_ratio": -0.4093382954597473,
      "logits/chosen": -0.24490830302238464,
      "logits/rejected": -1.5799317359924316,
      "logps/chosen": -1.6035810708999634,
      "logps/rejected": -2.200234889984131,
      "loss": 1.6564,
      "nll_loss": 1.6155025959014893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16035811603069305,
      "rewards/margins": 0.05966537445783615,
      "rewards/rejected": -0.2200234830379486,
      "step": 443
    },
    {
      "epoch": 0.6997635933806147,
      "grad_norm": 0.1867726445198059,
      "learning_rate": 3.291880809961814e-06,
      "log_odds_chosen": 0.6228445768356323,
      "log_odds_ratio": -0.4409523904323578,
      "logits/chosen": -0.16951556503772736,
      "logits/rejected": -1.1806188821792603,
      "logps/chosen": -1.5619224309921265,
      "logps/rejected": -2.079741954803467,
      "loss": 1.6322,
      "nll_loss": 1.5880740880966187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15619224309921265,
      "rewards/margins": 0.05178195238113403,
      "rewards/rejected": -0.20797421038150787,
      "step": 444
    },
    {
      "epoch": 0.7013396375098503,
      "grad_norm": 0.19691401720046997,
      "learning_rate": 3.2876795161779473e-06,
      "log_odds_chosen": 0.7107241153717041,
      "log_odds_ratio": -0.4021419882774353,
      "logits/chosen": -0.18867270648479462,
      "logits/rejected": -1.5382509231567383,
      "logps/chosen": -1.5732825994491577,
      "logps/rejected": -2.173732280731201,
      "loss": 1.6228,
      "nll_loss": 1.5826274156570435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.157328262925148,
      "rewards/margins": 0.06004498153924942,
      "rewards/rejected": -0.21737323701381683,
      "step": 445
    },
    {
      "epoch": 0.7029156816390859,
      "grad_norm": 0.20158085227012634,
      "learning_rate": 3.2834684945903776e-06,
      "log_odds_chosen": 0.5534508228302002,
      "log_odds_ratio": -0.4586279094219208,
      "logits/chosen": -0.24863602221012115,
      "logits/rejected": -1.2043968439102173,
      "logps/chosen": -1.5018632411956787,
      "logps/rejected": -1.9528242349624634,
      "loss": 1.5808,
      "nll_loss": 1.5349347591400146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15018633008003235,
      "rewards/margins": 0.04509609937667847,
      "rewards/rejected": -0.19528241455554962,
      "step": 446
    },
    {
      "epoch": 0.7044917257683215,
      "grad_norm": 0.20299731194972992,
      "learning_rate": 3.2792477770113624e-06,
      "log_odds_chosen": 0.4938434362411499,
      "log_odds_ratio": -0.4814365804195404,
      "logits/chosen": -0.33849644660949707,
      "logits/rejected": -1.3542628288269043,
      "logps/chosen": -1.5917766094207764,
      "logps/rejected": -2.002875328063965,
      "loss": 1.6474,
      "nll_loss": 1.5992237329483032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15917766094207764,
      "rewards/margins": 0.041109874844551086,
      "rewards/rejected": -0.20028753578662872,
      "step": 447
    },
    {
      "epoch": 0.7060677698975572,
      "grad_norm": 0.18600669503211975,
      "learning_rate": 3.275017395326407e-06,
      "log_odds_chosen": 0.47598177194595337,
      "log_odds_ratio": -0.4912503659725189,
      "logits/chosen": -0.15067262947559357,
      "logits/rejected": -1.1057360172271729,
      "logps/chosen": -1.479712963104248,
      "logps/rejected": -1.8618954420089722,
      "loss": 1.5626,
      "nll_loss": 1.5134308338165283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14797130227088928,
      "rewards/margins": 0.03821825981140137,
      "rewards/rejected": -0.18618956208229065,
      "step": 448
    },
    {
      "epoch": 0.7076438140267928,
      "grad_norm": 0.1918012499809265,
      "learning_rate": 3.2707773814940244e-06,
      "log_odds_chosen": 0.6454062461853027,
      "log_odds_ratio": -0.4280465841293335,
      "logits/chosen": -0.17873692512512207,
      "logits/rejected": -1.295732021331787,
      "logps/chosen": -1.507833480834961,
      "logps/rejected": -2.0341856479644775,
      "loss": 1.5887,
      "nll_loss": 1.5458601713180542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15078336000442505,
      "rewards/margins": 0.05263521522283554,
      "rewards/rejected": -0.20341856777668,
      "step": 449
    },
    {
      "epoch": 0.7092198581560284,
      "grad_norm": 0.1859048306941986,
      "learning_rate": 3.2665277675454935e-06,
      "log_odds_chosen": 0.6589299440383911,
      "log_odds_ratio": -0.434900164604187,
      "logits/chosen": -0.2396085262298584,
      "logits/rejected": -1.546783685684204,
      "logps/chosen": -1.5887700319290161,
      "logps/rejected": -2.151167392730713,
      "loss": 1.6406,
      "nll_loss": 1.5970611572265625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15887701511383057,
      "rewards/margins": 0.056239720433950424,
      "rewards/rejected": -0.2151167392730713,
      "step": 450
    },
    {
      "epoch": 0.710795902285264,
      "grad_norm": 0.18971043825149536,
      "learning_rate": 3.262268585584619e-06,
      "log_odds_chosen": 0.5915400981903076,
      "log_odds_ratio": -0.44323453307151794,
      "logits/chosen": -0.16203592717647552,
      "logits/rejected": -1.2521172761917114,
      "logps/chosen": -1.5381022691726685,
      "logps/rejected": -2.030604124069214,
      "loss": 1.6041,
      "nll_loss": 1.5598118305206299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15381023287773132,
      "rewards/margins": 0.049250174313783646,
      "rewards/rejected": -0.20306040346622467,
      "step": 451
    },
    {
      "epoch": 0.7123719464144996,
      "grad_norm": 0.2022084891796112,
      "learning_rate": 3.2579998677874853e-06,
      "log_odds_chosen": 0.6777637600898743,
      "log_odds_ratio": -0.4203702211380005,
      "logits/chosen": -0.24360840022563934,
      "logits/rejected": -1.232491135597229,
      "logps/chosen": -1.5734740495681763,
      "logps/rejected": -2.1456642150878906,
      "loss": 1.6256,
      "nll_loss": 1.5835988521575928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573474109172821,
      "rewards/margins": 0.0572190061211586,
      "rewards/rejected": -0.2145664095878601,
      "step": 452
    },
    {
      "epoch": 0.7139479905437353,
      "grad_norm": 0.1870575249195099,
      "learning_rate": 3.2537216464022155e-06,
      "log_odds_chosen": 0.6743752956390381,
      "log_odds_ratio": -0.4204288721084595,
      "logits/chosen": -0.22134186327457428,
      "logits/rejected": -1.2324929237365723,
      "logps/chosen": -1.5700197219848633,
      "logps/rejected": -2.1332030296325684,
      "loss": 1.6333,
      "nll_loss": 1.5912562608718872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15700197219848633,
      "rewards/margins": 0.05631833150982857,
      "rewards/rejected": -0.2133202999830246,
      "step": 453
    },
    {
      "epoch": 0.7155240346729709,
      "grad_norm": 0.1873411238193512,
      "learning_rate": 3.2494339537487314e-06,
      "log_odds_chosen": 0.5221689343452454,
      "log_odds_ratio": -0.4673764109611511,
      "logits/chosen": -0.24073247611522675,
      "logits/rejected": -1.4843155145645142,
      "logps/chosen": -1.574814796447754,
      "logps/rejected": -2.0072293281555176,
      "loss": 1.6482,
      "nll_loss": 1.6014328002929688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15748149156570435,
      "rewards/margins": 0.04324144124984741,
      "rewards/rejected": -0.20072293281555176,
      "step": 454
    },
    {
      "epoch": 0.7171000788022065,
      "grad_norm": 0.18600791692733765,
      "learning_rate": 3.2451368222185006e-06,
      "log_odds_chosen": 0.4506160616874695,
      "log_odds_ratio": -0.49926573038101196,
      "logits/chosen": -0.23936370015144348,
      "logits/rejected": -1.121845006942749,
      "logps/chosen": -1.5184142589569092,
      "logps/rejected": -1.8841001987457275,
      "loss": 1.5993,
      "nll_loss": 1.5493282079696655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1518414169549942,
      "rewards/margins": 0.03656858578324318,
      "rewards/rejected": -0.18841001391410828,
      "step": 455
    },
    {
      "epoch": 0.7186761229314421,
      "grad_norm": 0.17563648521900177,
      "learning_rate": 3.2408302842743007e-06,
      "log_odds_chosen": 0.6520451903343201,
      "log_odds_ratio": -0.4270837604999542,
      "logits/chosen": -0.19061179459095,
      "logits/rejected": -1.2755080461502075,
      "logps/chosen": -1.4686167240142822,
      "logps/rejected": -2.004254102706909,
      "loss": 1.5546,
      "nll_loss": 1.511888027191162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14686168730258942,
      "rewards/margins": 0.05356375128030777,
      "rewards/rejected": -0.2004254311323166,
      "step": 456
    },
    {
      "epoch": 0.7202521670606777,
      "grad_norm": 0.19001001119613647,
      "learning_rate": 3.2365143724499684e-06,
      "log_odds_chosen": 0.5796292424201965,
      "log_odds_ratio": -0.4465140104293823,
      "logits/chosen": -0.20313462615013123,
      "logits/rejected": -1.3108395338058472,
      "logps/chosen": -1.5164985656738281,
      "logps/rejected": -1.994590163230896,
      "loss": 1.6016,
      "nll_loss": 1.556924819946289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1516498476266861,
      "rewards/margins": 0.047809161245822906,
      "rewards/rejected": -0.1994590163230896,
      "step": 457
    },
    {
      "epoch": 0.7218282111899134,
      "grad_norm": 0.19313624501228333,
      "learning_rate": 3.2321891193501564e-06,
      "log_odds_chosen": 0.5773687958717346,
      "log_odds_ratio": -0.45744845271110535,
      "logits/chosen": -0.1832474172115326,
      "logits/rejected": -0.8535163998603821,
      "logps/chosen": -1.5230867862701416,
      "logps/rejected": -2.0000154972076416,
      "loss": 1.5894,
      "nll_loss": 1.5436917543411255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15230868756771088,
      "rewards/margins": 0.04769286513328552,
      "rewards/rejected": -0.2000015527009964,
      "step": 458
    },
    {
      "epoch": 0.723404255319149,
      "grad_norm": 0.1829526573419571,
      "learning_rate": 3.2278545576500858e-06,
      "log_odds_chosen": 0.815370500087738,
      "log_odds_ratio": -0.3753301799297333,
      "logits/chosen": -0.10391156375408173,
      "logits/rejected": -1.0528960227966309,
      "logps/chosen": -1.3729358911514282,
      "logps/rejected": -2.040825843811035,
      "loss": 1.4585,
      "nll_loss": 1.4209495782852173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13729359209537506,
      "rewards/margins": 0.06678898632526398,
      "rewards/rejected": -0.20408257842063904,
      "step": 459
    },
    {
      "epoch": 0.7249802994483846,
      "grad_norm": 0.19639159739017487,
      "learning_rate": 3.223510720095299e-06,
      "log_odds_chosen": 0.7596641778945923,
      "log_odds_ratio": -0.3919370770454407,
      "logits/chosen": -0.2874103784561157,
      "logits/rejected": -1.3473809957504272,
      "logps/chosen": -1.551926851272583,
      "logps/rejected": -2.19168758392334,
      "loss": 1.6071,
      "nll_loss": 1.5678837299346924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15519270300865173,
      "rewards/margins": 0.06397604942321777,
      "rewards/rejected": -0.2191687524318695,
      "step": 460
    },
    {
      "epoch": 0.7265563435776202,
      "grad_norm": 0.18149657547473907,
      "learning_rate": 3.2191576395014158e-06,
      "log_odds_chosen": 0.7072669863700867,
      "log_odds_ratio": -0.40362295508384705,
      "logits/chosen": -0.23113706707954407,
      "logits/rejected": -1.2299100160598755,
      "logps/chosen": -1.4852123260498047,
      "logps/rejected": -2.0701723098754883,
      "loss": 1.5528,
      "nll_loss": 1.5123920440673828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14852122962474823,
      "rewards/margins": 0.05849601700901985,
      "rewards/rejected": -0.20701724290847778,
      "step": 461
    },
    {
      "epoch": 0.7281323877068558,
      "grad_norm": 0.18942932784557343,
      "learning_rate": 3.2147953487538794e-06,
      "log_odds_chosen": 0.6708348989486694,
      "log_odds_ratio": -0.41837048530578613,
      "logits/chosen": -0.1631353497505188,
      "logits/rejected": -1.3306108713150024,
      "logps/chosen": -1.455315113067627,
      "logps/rejected": -2.0053393840789795,
      "loss": 1.5252,
      "nll_loss": 1.4833753108978271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14553150534629822,
      "rewards/margins": 0.05500244349241257,
      "rewards/rejected": -0.2005339413881302,
      "step": 462
    },
    {
      "epoch": 0.7297084318360915,
      "grad_norm": 0.20128397643566132,
      "learning_rate": 3.2104238808077133e-06,
      "log_odds_chosen": 0.5619516968727112,
      "log_odds_ratio": -0.4598609507083893,
      "logits/chosen": -0.1880580186843872,
      "logits/rejected": -1.0508508682250977,
      "logps/chosen": -1.4888397455215454,
      "logps/rejected": -1.9507625102996826,
      "loss": 1.5634,
      "nll_loss": 1.5174212455749512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14888396859169006,
      "rewards/margins": 0.04619229957461357,
      "rewards/rejected": -0.19507628679275513,
      "step": 463
    },
    {
      "epoch": 0.731284475965327,
      "grad_norm": 0.18228676915168762,
      "learning_rate": 3.2060432686872704e-06,
      "log_odds_chosen": 0.8119957447052002,
      "log_odds_ratio": -0.37603914737701416,
      "logits/chosen": -0.2555653750896454,
      "logits/rejected": -1.1474933624267578,
      "logps/chosen": -1.398220181465149,
      "logps/rejected": -2.059756278991699,
      "loss": 1.4829,
      "nll_loss": 1.4452736377716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13982200622558594,
      "rewards/margins": 0.0661536231637001,
      "rewards/rejected": -0.20597563683986664,
      "step": 464
    },
    {
      "epoch": 0.7328605200945626,
      "grad_norm": 0.18841765820980072,
      "learning_rate": 3.201653545485982e-06,
      "log_odds_chosen": 0.6405482888221741,
      "log_odds_ratio": -0.4303835332393646,
      "logits/chosen": -0.16426052153110504,
      "logits/rejected": -1.2122530937194824,
      "logps/chosen": -1.5435234308242798,
      "logps/rejected": -2.079439640045166,
      "loss": 1.599,
      "nll_loss": 1.5559314489364624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543523371219635,
      "rewards/margins": 0.053591638803482056,
      "rewards/rejected": -0.20794397592544556,
      "step": 465
    },
    {
      "epoch": 0.7344365642237982,
      "grad_norm": 0.20733259618282318,
      "learning_rate": 3.197254744366111e-06,
      "log_odds_chosen": 0.6555430889129639,
      "log_odds_ratio": -0.42545032501220703,
      "logits/chosen": -0.14602793753147125,
      "logits/rejected": -1.174269676208496,
      "logps/chosen": -1.4550955295562744,
      "logps/rejected": -1.9908273220062256,
      "loss": 1.53,
      "nll_loss": 1.4874444007873535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14550955593585968,
      "rewards/margins": 0.05357319116592407,
      "rewards/rejected": -0.19908274710178375,
      "step": 466
    },
    {
      "epoch": 0.7360126083530338,
      "grad_norm": 0.19345013797283173,
      "learning_rate": 3.192846898558498e-06,
      "log_odds_chosen": 0.5251720547676086,
      "log_odds_ratio": -0.46669673919677734,
      "logits/chosen": -0.18853773176670074,
      "logits/rejected": -1.2136728763580322,
      "logps/chosen": -1.5849101543426514,
      "logps/rejected": -2.023404359817505,
      "loss": 1.654,
      "nll_loss": 1.6073112487792969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15849103033542633,
      "rewards/margins": 0.04384942352771759,
      "rewards/rejected": -0.20234045386314392,
      "step": 467
    },
    {
      "epoch": 0.7375886524822695,
      "grad_norm": 0.19730189442634583,
      "learning_rate": 3.188430041362313e-06,
      "log_odds_chosen": 0.47901207208633423,
      "log_odds_ratio": -0.49027061462402344,
      "logits/chosen": -0.15382762253284454,
      "logits/rejected": -1.2334424257278442,
      "logps/chosen": -1.579338550567627,
      "logps/rejected": -1.979215383529663,
      "loss": 1.637,
      "nll_loss": 1.5880184173583984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15793386101722717,
      "rewards/margins": 0.03998768329620361,
      "rewards/rejected": -0.1979215443134308,
      "step": 468
    },
    {
      "epoch": 0.7391646966115051,
      "grad_norm": 0.19508811831474304,
      "learning_rate": 3.184004206144803e-06,
      "log_odds_chosen": 0.7172151803970337,
      "log_odds_ratio": -0.40794268250465393,
      "logits/chosen": -0.25483450293540955,
      "logits/rejected": -1.2210605144500732,
      "logps/chosen": -1.4742763042449951,
      "logps/rejected": -2.062727689743042,
      "loss": 1.5429,
      "nll_loss": 1.5020675659179688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14742764830589294,
      "rewards/margins": 0.05884512513875961,
      "rewards/rejected": -0.20627278089523315,
      "step": 469
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 0.1892954707145691,
      "learning_rate": 3.1795694263410386e-06,
      "log_odds_chosen": 0.7759031653404236,
      "log_odds_ratio": -0.39132991433143616,
      "logits/chosen": -0.21951723098754883,
      "logits/rejected": -1.2244356870651245,
      "logps/chosen": -1.465575933456421,
      "logps/rejected": -2.1147849559783936,
      "loss": 1.5275,
      "nll_loss": 1.488350749015808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14655759930610657,
      "rewards/margins": 0.06492087990045547,
      "rewards/rejected": -0.21147848665714264,
      "step": 470
    },
    {
      "epoch": 0.7423167848699763,
      "grad_norm": 0.18874920904636383,
      "learning_rate": 3.1751257354536634e-06,
      "log_odds_chosen": 0.5416457056999207,
      "log_odds_ratio": -0.4711780548095703,
      "logits/chosen": -0.17236806452274323,
      "logits/rejected": -1.3529729843139648,
      "logps/chosen": -1.4607828855514526,
      "logps/rejected": -1.9070994853973389,
      "loss": 1.5414,
      "nll_loss": 1.4942772388458252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14607828855514526,
      "rewards/margins": 0.04463164880871773,
      "rewards/rejected": -0.1907099336385727,
      "step": 471
    },
    {
      "epoch": 0.7438928289992119,
      "grad_norm": 0.19573341310024261,
      "learning_rate": 3.1706731670526394e-06,
      "log_odds_chosen": 0.604232132434845,
      "log_odds_ratio": -0.43818584084510803,
      "logits/chosen": -0.27332159876823425,
      "logits/rejected": -1.2359827756881714,
      "logps/chosen": -1.5119805335998535,
      "logps/rejected": -2.008622884750366,
      "loss": 1.5662,
      "nll_loss": 1.5224015712738037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15119807422161102,
      "rewards/margins": 0.04966423660516739,
      "rewards/rejected": -0.20086230337619781,
      "step": 472
    },
    {
      "epoch": 0.7454688731284476,
      "grad_norm": 0.19741228222846985,
      "learning_rate": 3.166211754774994e-06,
      "log_odds_chosen": 0.6473110914230347,
      "log_odds_ratio": -0.4283471703529358,
      "logits/chosen": -0.2195882350206375,
      "logits/rejected": -1.3575206995010376,
      "logps/chosen": -1.5655333995819092,
      "logps/rejected": -2.1075003147125244,
      "loss": 1.652,
      "nll_loss": 1.609170913696289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15655335783958435,
      "rewards/margins": 0.054196685552597046,
      "rewards/rejected": -0.2107500433921814,
      "step": 473
    },
    {
      "epoch": 0.7470449172576832,
      "grad_norm": 0.19509533047676086,
      "learning_rate": 3.1617415323245665e-06,
      "log_odds_chosen": 0.6397197246551514,
      "log_odds_ratio": -0.43335115909576416,
      "logits/chosen": -0.26395586133003235,
      "logits/rejected": -1.3436589241027832,
      "logps/chosen": -1.4741613864898682,
      "logps/rejected": -1.9915375709533691,
      "loss": 1.5348,
      "nll_loss": 1.491437554359436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1474161297082901,
      "rewards/margins": 0.05173762887716293,
      "rewards/rejected": -0.19915376603603363,
      "step": 474
    },
    {
      "epoch": 0.7486209613869188,
      "grad_norm": 0.18101076781749725,
      "learning_rate": 3.157262533471752e-06,
      "log_odds_chosen": 0.690365731716156,
      "log_odds_ratio": -0.41062238812446594,
      "logits/chosen": -0.1729176789522171,
      "logits/rejected": -1.3418751955032349,
      "logps/chosen": -1.5249015092849731,
      "logps/rejected": -2.1028430461883545,
      "loss": 1.5936,
      "nll_loss": 1.552560806274414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15249013900756836,
      "rewards/margins": 0.057794176042079926,
      "rewards/rejected": -0.2102843075990677,
      "step": 475
    },
    {
      "epoch": 0.7501970055161544,
      "grad_norm": 0.19821086525917053,
      "learning_rate": 3.1527747920532468e-06,
      "log_odds_chosen": 0.5523171424865723,
      "log_odds_ratio": -0.4564129710197449,
      "logits/chosen": -0.1943480521440506,
      "logits/rejected": -1.095949649810791,
      "logps/chosen": -1.4880762100219727,
      "logps/rejected": -1.9403318166732788,
      "loss": 1.5629,
      "nll_loss": 1.517241358757019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14880762994289398,
      "rewards/margins": 0.04522555693984032,
      "rewards/rejected": -0.1940331906080246,
      "step": 476
    },
    {
      "epoch": 0.75177304964539,
      "grad_norm": 0.19268104434013367,
      "learning_rate": 3.148278341971795e-06,
      "log_odds_chosen": 0.6278610229492188,
      "log_odds_ratio": -0.4335193932056427,
      "logits/chosen": -0.24915321171283722,
      "logits/rejected": -1.1845955848693848,
      "logps/chosen": -1.5173267126083374,
      "logps/rejected": -2.037794351577759,
      "loss": 1.5831,
      "nll_loss": 1.5396989583969116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517326831817627,
      "rewards/margins": 0.052046775817871094,
      "rewards/rejected": -0.2037794440984726,
      "step": 477
    },
    {
      "epoch": 0.7533490937746257,
      "grad_norm": 0.1977827399969101,
      "learning_rate": 3.143773217195929e-06,
      "log_odds_chosen": 0.7068151831626892,
      "log_odds_ratio": -0.40859490633010864,
      "logits/chosen": -0.26628124713897705,
      "logits/rejected": -1.3386417627334595,
      "logps/chosen": -1.5787765979766846,
      "logps/rejected": -2.174910306930542,
      "loss": 1.6357,
      "nll_loss": 1.5947997570037842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15787766873836517,
      "rewards/margins": 0.05961337685585022,
      "rewards/rejected": -0.2174910455942154,
      "step": 478
    },
    {
      "epoch": 0.7549251379038613,
      "grad_norm": 0.19414526224136353,
      "learning_rate": 3.139259451759714e-06,
      "log_odds_chosen": 0.555619478225708,
      "log_odds_ratio": -0.45742952823638916,
      "logits/chosen": -0.2637663781642914,
      "logits/rejected": -1.2724649906158447,
      "logps/chosen": -1.5820374488830566,
      "logps/rejected": -2.045403480529785,
      "loss": 1.6538,
      "nll_loss": 1.6080107688903809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15820375084877014,
      "rewards/margins": 0.04633659869432449,
      "rewards/rejected": -0.20454034209251404,
      "step": 479
    },
    {
      "epoch": 0.7565011820330969,
      "grad_norm": 0.1868637353181839,
      "learning_rate": 3.134737079762493e-06,
      "log_odds_chosen": 0.5565078854560852,
      "log_odds_ratio": -0.45784488320350647,
      "logits/chosen": -0.21357539296150208,
      "logits/rejected": -1.2584372758865356,
      "logps/chosen": -1.5111368894577026,
      "logps/rejected": -1.9670014381408691,
      "loss": 1.5766,
      "nll_loss": 1.5307987928390503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15111368894577026,
      "rewards/margins": 0.045586466789245605,
      "rewards/rejected": -0.19670015573501587,
      "step": 480
    },
    {
      "epoch": 0.7580772261623325,
      "grad_norm": 0.2013820856809616,
      "learning_rate": 3.130206135368626e-06,
      "log_odds_chosen": 0.5862029194831848,
      "log_odds_ratio": -0.4538579285144806,
      "logits/chosen": -0.2686524987220764,
      "logits/rejected": -0.9783179759979248,
      "logps/chosen": -1.43379545211792,
      "logps/rejected": -1.9084105491638184,
      "loss": 1.524,
      "nll_loss": 1.478649616241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433795690536499,
      "rewards/margins": 0.04746149852871895,
      "rewards/rejected": -0.19084104895591736,
      "step": 481
    },
    {
      "epoch": 0.7596532702915682,
      "grad_norm": 0.19040453433990479,
      "learning_rate": 3.1256666528072327e-06,
      "log_odds_chosen": 0.755387544631958,
      "log_odds_ratio": -0.39063921570777893,
      "logits/chosen": -0.25499293208122253,
      "logits/rejected": -1.0305991172790527,
      "logps/chosen": -1.4815821647644043,
      "logps/rejected": -2.1069517135620117,
      "loss": 1.5446,
      "nll_loss": 1.5055186748504639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1481582075357437,
      "rewards/margins": 0.06253695487976074,
      "rewards/rejected": -0.21069514751434326,
      "step": 482
    },
    {
      "epoch": 0.7612293144208038,
      "grad_norm": 0.18193352222442627,
      "learning_rate": 3.121118666371937e-06,
      "log_odds_chosen": 0.5603374242782593,
      "log_odds_ratio": -0.4579220414161682,
      "logits/chosen": -0.182877317070961,
      "logits/rejected": -1.5083105564117432,
      "logps/chosen": -1.5461608171463013,
      "logps/rejected": -2.0124778747558594,
      "loss": 1.6023,
      "nll_loss": 1.5564837455749512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1546160876750946,
      "rewards/margins": 0.046631716191768646,
      "rewards/rejected": -0.20124779641628265,
      "step": 483
    },
    {
      "epoch": 0.7628053585500394,
      "grad_norm": 0.20136219263076782,
      "learning_rate": 3.1165622104206034e-06,
      "log_odds_chosen": 0.7325907349586487,
      "log_odds_ratio": -0.39630693197250366,
      "logits/chosen": -0.3966868817806244,
      "logits/rejected": -1.1748623847961426,
      "logps/chosen": -1.4722321033477783,
      "logps/rejected": -2.079011917114258,
      "loss": 1.5403,
      "nll_loss": 1.500671148300171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14722320437431335,
      "rewards/margins": 0.06067798286676407,
      "rewards/rejected": -0.20790117979049683,
      "step": 484
    },
    {
      "epoch": 0.764381402679275,
      "grad_norm": 0.18245038390159607,
      "learning_rate": 3.1119973193750816e-06,
      "log_odds_chosen": 0.6451510190963745,
      "log_odds_ratio": -0.4264066815376282,
      "logits/chosen": -0.2894980013370514,
      "logits/rejected": -1.225103497505188,
      "logps/chosen": -1.4829157590866089,
      "logps/rejected": -2.0141286849975586,
      "loss": 1.5432,
      "nll_loss": 1.500580906867981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14829157292842865,
      "rewards/margins": 0.05312127619981766,
      "rewards/rejected": -0.2014128416776657,
      "step": 485
    },
    {
      "epoch": 0.7659574468085106,
      "grad_norm": 0.19604384899139404,
      "learning_rate": 3.1074240277209408e-06,
      "log_odds_chosen": 0.5782766938209534,
      "log_odds_ratio": -0.4525739848613739,
      "logits/chosen": -0.2679707705974579,
      "logits/rejected": -1.1799029111862183,
      "logps/chosen": -1.507366418838501,
      "logps/rejected": -1.9800548553466797,
      "loss": 1.5533,
      "nll_loss": 1.5080450773239136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15073664486408234,
      "rewards/margins": 0.04726884141564369,
      "rewards/rejected": -0.19800546765327454,
      "step": 486
    },
    {
      "epoch": 0.7675334909377463,
      "grad_norm": 0.21242745220661163,
      "learning_rate": 3.102842370007217e-06,
      "log_odds_chosen": 0.654798150062561,
      "log_odds_ratio": -0.42668747901916504,
      "logits/chosen": -0.18659040331840515,
      "logits/rejected": -1.0967789888381958,
      "logps/chosen": -1.5160582065582275,
      "logps/rejected": -2.0530829429626465,
      "loss": 1.5815,
      "nll_loss": 1.538811206817627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15160581469535828,
      "rewards/margins": 0.0537024661898613,
      "rewards/rejected": -0.20530828833580017,
      "step": 487
    },
    {
      "epoch": 0.7691095350669819,
      "grad_norm": 0.19127397239208221,
      "learning_rate": 3.0982523808461454e-06,
      "log_odds_chosen": 0.5358982086181641,
      "log_odds_ratio": -0.4667418897151947,
      "logits/chosen": -0.18278566002845764,
      "logits/rejected": -1.2197632789611816,
      "logps/chosen": -1.5576581954956055,
      "logps/rejected": -1.9974056482315063,
      "loss": 1.6145,
      "nll_loss": 1.567873239517212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557658314704895,
      "rewards/margins": 0.04397474229335785,
      "rewards/rejected": -0.19974057376384735,
      "step": 488
    },
    {
      "epoch": 0.7706855791962175,
      "grad_norm": 0.18905805051326752,
      "learning_rate": 3.0936540949129006e-06,
      "log_odds_chosen": 0.6154669523239136,
      "log_odds_ratio": -0.4424894452095032,
      "logits/chosen": -0.24286700785160065,
      "logits/rejected": -1.1417758464813232,
      "logps/chosen": -1.5341846942901611,
      "logps/rejected": -2.046876907348633,
      "loss": 1.5994,
      "nll_loss": 1.5551624298095703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15341848134994507,
      "rewards/margins": 0.05126923322677612,
      "rewards/rejected": -0.20468769967556,
      "step": 489
    },
    {
      "epoch": 0.7722616233254531,
      "grad_norm": 0.17579598724842072,
      "learning_rate": 3.0890475469453378e-06,
      "log_odds_chosen": 0.7146469354629517,
      "log_odds_ratio": -0.40994906425476074,
      "logits/chosen": -0.24190282821655273,
      "logits/rejected": -1.3688513040542603,
      "logps/chosen": -1.4117985963821411,
      "logps/rejected": -1.9928693771362305,
      "loss": 1.4803,
      "nll_loss": 1.439333438873291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411798745393753,
      "rewards/margins": 0.05810706689953804,
      "rewards/rejected": -0.19928693771362305,
      "step": 490
    },
    {
      "epoch": 0.7738376674546887,
      "grad_norm": 0.1882636994123459,
      "learning_rate": 3.0844327717437263e-06,
      "log_odds_chosen": 0.6613863110542297,
      "log_odds_ratio": -0.4273605942726135,
      "logits/chosen": -0.23976236581802368,
      "logits/rejected": -1.243269443511963,
      "logps/chosen": -1.437576413154602,
      "logps/rejected": -1.9814512729644775,
      "loss": 1.5097,
      "nll_loss": 1.4669883251190186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437576413154602,
      "rewards/margins": 0.05438747629523277,
      "rewards/rejected": -0.19814513623714447,
      "step": 491
    },
    {
      "epoch": 0.7754137115839244,
      "grad_norm": 0.18878327310085297,
      "learning_rate": 3.0798098041704892e-06,
      "log_odds_chosen": 0.478700190782547,
      "log_odds_ratio": -0.4873453974723816,
      "logits/chosen": -0.18805843591690063,
      "logits/rejected": -1.271687388420105,
      "logps/chosen": -1.3689371347427368,
      "logps/rejected": -1.7501403093338013,
      "loss": 1.4618,
      "nll_loss": 1.4130362272262573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13689371943473816,
      "rewards/margins": 0.03812030330300331,
      "rewards/rejected": -0.17501404881477356,
      "step": 492
    },
    {
      "epoch": 0.77698975571316,
      "grad_norm": 0.19235889613628387,
      "learning_rate": 3.0751786791499368e-06,
      "log_odds_chosen": 0.6095344424247742,
      "log_odds_ratio": -0.44119495153427124,
      "logits/chosen": -0.20893912017345428,
      "logits/rejected": -1.2534291744232178,
      "logps/chosen": -1.5530734062194824,
      "logps/rejected": -2.0629472732543945,
      "loss": 1.6223,
      "nll_loss": 1.5781906843185425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.155307337641716,
      "rewards/margins": 0.050987400114536285,
      "rewards/rejected": -0.2062947303056717,
      "step": 493
    },
    {
      "epoch": 0.7785657998423956,
      "grad_norm": 0.18117356300354004,
      "learning_rate": 3.070539431668008e-06,
      "log_odds_chosen": 0.6075527667999268,
      "log_odds_ratio": -0.4399612247943878,
      "logits/chosen": -0.2257227599620819,
      "logits/rejected": -1.1582225561141968,
      "logps/chosen": -1.5574135780334473,
      "logps/rejected": -2.067356824874878,
      "loss": 1.6169,
      "nll_loss": 1.5728775262832642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557413637638092,
      "rewards/margins": 0.05099431425333023,
      "rewards/rejected": -0.20673568546772003,
      "step": 494
    },
    {
      "epoch": 0.7801418439716312,
      "grad_norm": 0.1898123025894165,
      "learning_rate": 3.0658920967720018e-06,
      "log_odds_chosen": 0.7518050074577332,
      "log_odds_ratio": -0.39490705728530884,
      "logits/chosen": -0.35575562715530396,
      "logits/rejected": -1.1922754049301147,
      "logps/chosen": -1.4951189756393433,
      "logps/rejected": -2.1237242221832275,
      "loss": 1.5649,
      "nll_loss": 1.525448203086853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495118886232376,
      "rewards/margins": 0.06286052614450455,
      "rewards/rejected": -0.21237242221832275,
      "step": 495
    },
    {
      "epoch": 0.7817178881008668,
      "grad_norm": 0.1872013807296753,
      "learning_rate": 3.0612367095703116e-06,
      "log_odds_chosen": 0.6950463652610779,
      "log_odds_ratio": -0.4093022644519806,
      "logits/chosen": -0.21190622448921204,
      "logits/rejected": -1.4540730714797974,
      "logps/chosen": -1.5332351922988892,
      "logps/rejected": -2.1157336235046387,
      "loss": 1.5885,
      "nll_loss": 1.5475294589996338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15332353115081787,
      "rewards/margins": 0.05824984610080719,
      "rewards/rejected": -0.21157337725162506,
      "step": 496
    },
    {
      "epoch": 0.7832939322301025,
      "grad_norm": 0.18900524079799652,
      "learning_rate": 3.056573305232167e-06,
      "log_odds_chosen": 0.7527428865432739,
      "log_odds_ratio": -0.392995148897171,
      "logits/chosen": -0.2251337319612503,
      "logits/rejected": -1.3849214315414429,
      "logps/chosen": -1.4837510585784912,
      "logps/rejected": -2.1036033630371094,
      "loss": 1.5498,
      "nll_loss": 1.510545253753662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14837510883808136,
      "rewards/margins": 0.061985231935977936,
      "rewards/rejected": -0.2103603333234787,
      "step": 497
    },
    {
      "epoch": 0.7848699763593381,
      "grad_norm": 0.18676206469535828,
      "learning_rate": 3.051901918987359e-06,
      "log_odds_chosen": 0.7129929065704346,
      "log_odds_ratio": -0.4029238224029541,
      "logits/chosen": -0.3676823675632477,
      "logits/rejected": -1.319760799407959,
      "logps/chosen": -1.4286247491836548,
      "logps/rejected": -2.0117030143737793,
      "loss": 1.4979,
      "nll_loss": 1.4575592279434204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1428624838590622,
      "rewards/margins": 0.058307845145463943,
      "rewards/rejected": -0.20117032527923584,
      "step": 498
    },
    {
      "epoch": 0.7864460204885737,
      "grad_norm": 0.2265917956829071,
      "learning_rate": 3.047222586125979e-06,
      "log_odds_chosen": 0.725178599357605,
      "log_odds_ratio": -0.39900532364845276,
      "logits/chosen": -0.19229286909103394,
      "logits/rejected": -0.8906416296958923,
      "logps/chosen": -1.4400482177734375,
      "logps/rejected": -2.0356321334838867,
      "loss": 1.5177,
      "nll_loss": 1.4778459072113037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14400483667850494,
      "rewards/margins": 0.059558384120464325,
      "rewards/rejected": -0.20356319844722748,
      "step": 499
    },
    {
      "epoch": 0.7880220646178093,
      "grad_norm": 0.18137092888355255,
      "learning_rate": 3.042535341998152e-06,
      "log_odds_chosen": 0.5409306883811951,
      "log_odds_ratio": -0.46464020013809204,
      "logits/chosen": -0.10464775562286377,
      "logits/rejected": -1.2362757921218872,
      "logps/chosen": -1.5858546495437622,
      "logps/rejected": -2.039395332336426,
      "loss": 1.6316,
      "nll_loss": 1.5851597785949707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15858545899391174,
      "rewards/margins": 0.04535408318042755,
      "rewards/rejected": -0.2039395570755005,
      "step": 500
    },
    {
      "epoch": 0.789598108747045,
      "grad_norm": 0.19027630984783173,
      "learning_rate": 3.037840222013769e-06,
      "log_odds_chosen": 0.6238623261451721,
      "log_odds_ratio": -0.43770092725753784,
      "logits/chosen": -0.19144752621650696,
      "logits/rejected": -0.9809869527816772,
      "logps/chosen": -1.5221189260482788,
      "logps/rejected": -2.0402820110321045,
      "loss": 1.5793,
      "nll_loss": 1.5355498790740967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15221190452575684,
      "rewards/margins": 0.05181630700826645,
      "rewards/rejected": -0.2040282040834427,
      "step": 501
    },
    {
      "epoch": 0.7911741528762806,
      "grad_norm": 0.1850683093070984,
      "learning_rate": 3.033137261642219e-06,
      "log_odds_chosen": 0.770533561706543,
      "log_odds_ratio": -0.3894934058189392,
      "logits/chosen": -0.2244076132774353,
      "logits/rejected": -1.2155929803848267,
      "logps/chosen": -1.4710595607757568,
      "logps/rejected": -2.1106247901916504,
      "loss": 1.5271,
      "nll_loss": 1.4881579875946045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14710596203804016,
      "rewards/margins": 0.06395652145147324,
      "rewards/rejected": -0.211062490940094,
      "step": 502
    },
    {
      "epoch": 0.7927501970055162,
      "grad_norm": 0.1797146201133728,
      "learning_rate": 3.02842649641212e-06,
      "log_odds_chosen": 0.7118703126907349,
      "log_odds_ratio": -0.40820637345314026,
      "logits/chosen": -0.13694347441196442,
      "logits/rejected": -1.1051255464553833,
      "logps/chosen": -1.517691969871521,
      "logps/rejected": -2.113922595977783,
      "loss": 1.5806,
      "nll_loss": 1.5397837162017822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15176919102668762,
      "rewards/margins": 0.059623055160045624,
      "rewards/rejected": -0.21139223873615265,
      "step": 503
    },
    {
      "epoch": 0.7943262411347518,
      "grad_norm": 0.18583795428276062,
      "learning_rate": 3.0237079619110554e-06,
      "log_odds_chosen": 0.7813155651092529,
      "log_odds_ratio": -0.3830240070819855,
      "logits/chosen": -0.26007750630378723,
      "logits/rejected": -1.3784176111221313,
      "logps/chosen": -1.4832781553268433,
      "logps/rejected": -2.126810312271118,
      "loss": 1.5407,
      "nll_loss": 1.5024230480194092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14832782745361328,
      "rewards/margins": 0.06435321271419525,
      "rewards/rejected": -0.21268102526664734,
      "step": 504
    },
    {
      "epoch": 0.7959022852639874,
      "grad_norm": 0.19030435383319855,
      "learning_rate": 3.0189816937852976e-06,
      "log_odds_chosen": 0.7461392283439636,
      "log_odds_ratio": -0.39470726251602173,
      "logits/chosen": -0.3101636469364166,
      "logits/rejected": -1.3661364316940308,
      "logps/chosen": -1.4635733366012573,
      "logps/rejected": -2.0784850120544434,
      "loss": 1.5116,
      "nll_loss": 1.4720885753631592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14635735750198364,
      "rewards/margins": 0.06149115413427353,
      "rewards/rejected": -0.20784850418567657,
      "step": 505
    },
    {
      "epoch": 0.797478329393223,
      "grad_norm": 0.18456418812274933,
      "learning_rate": 3.014247727739546e-06,
      "log_odds_chosen": 0.8672016263008118,
      "log_odds_ratio": -0.35805395245552063,
      "logits/chosen": -0.25903183221817017,
      "logits/rejected": -1.3094465732574463,
      "logps/chosen": -1.4639673233032227,
      "logps/rejected": -2.1868860721588135,
      "loss": 1.5206,
      "nll_loss": 1.4848119020462036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1463967263698578,
      "rewards/margins": 0.07229189574718475,
      "rewards/rejected": -0.21868860721588135,
      "step": 506
    },
    {
      "epoch": 0.7990543735224587,
      "grad_norm": 0.1868259459733963,
      "learning_rate": 3.009506099536653e-06,
      "log_odds_chosen": 0.6214101314544678,
      "log_odds_ratio": -0.4331985414028168,
      "logits/chosen": -0.20833641290664673,
      "logits/rejected": -1.3431315422058105,
      "logps/chosen": -1.520611047744751,
      "logps/rejected": -2.035783529281616,
      "loss": 1.5605,
      "nll_loss": 1.5171860456466675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1520611047744751,
      "rewards/margins": 0.05151723325252533,
      "rewards/rejected": -0.20357833802700043,
      "step": 507
    },
    {
      "epoch": 0.8006304176516943,
      "grad_norm": 0.1810230165719986,
      "learning_rate": 3.0047568449973544e-06,
      "log_odds_chosen": 0.8515524864196777,
      "log_odds_ratio": -0.364116907119751,
      "logits/chosen": -0.2927939295768738,
      "logits/rejected": -1.36265230178833,
      "logps/chosen": -1.372568130493164,
      "logps/rejected": -2.067460298538208,
      "loss": 1.4345,
      "nll_loss": 1.3981022834777832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13725680112838745,
      "rewards/margins": 0.06948922574520111,
      "rewards/rejected": -0.20674602687358856,
      "step": 508
    },
    {
      "epoch": 0.8022064617809299,
      "grad_norm": 0.1987299770116806,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.5341671109199524,
      "log_odds_ratio": -0.46895769238471985,
      "logits/chosen": -0.25236931443214417,
      "logits/rejected": -1.1305702924728394,
      "logps/chosen": -1.4515990018844604,
      "logps/rejected": -1.8847415447235107,
      "loss": 1.5178,
      "nll_loss": 1.470890760421753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14515990018844604,
      "rewards/margins": 0.043314263224601746,
      "rewards/rejected": -0.1884741634130478,
      "step": 509
    },
    {
      "epoch": 0.8037825059101655,
      "grad_norm": 0.17731288075447083,
      "learning_rate": 2.9952356004802813e-06,
      "log_odds_chosen": 0.47338056564331055,
      "log_odds_ratio": -0.4881148934364319,
      "logits/chosen": -0.20929376780986786,
      "logits/rejected": -1.256529450416565,
      "logps/chosen": -1.55081045627594,
      "logps/rejected": -1.941744327545166,
      "loss": 1.6022,
      "nll_loss": 1.5533736944198608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15508104860782623,
      "rewards/margins": 0.039093393832445145,
      "rewards/rejected": -0.19417443871498108,
      "step": 510
    },
    {
      "epoch": 0.8053585500394012,
      "grad_norm": 0.1984761506319046,
      "learning_rate": 2.9904636824309625e-06,
      "log_odds_chosen": 0.5101518630981445,
      "log_odds_ratio": -0.47170472145080566,
      "logits/chosen": -0.23291411995887756,
      "logits/rejected": -0.954363226890564,
      "logps/chosen": -1.4608978033065796,
      "logps/rejected": -1.875770092010498,
      "loss": 1.5257,
      "nll_loss": 1.4785338640213013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14608977735042572,
      "rewards/margins": 0.041487231850624084,
      "rewards/rejected": -0.187577024102211,
      "step": 511
    },
    {
      "epoch": 0.8069345941686368,
      "grad_norm": 0.207067608833313,
      "learning_rate": 2.985684281901603e-06,
      "log_odds_chosen": 0.5463806390762329,
      "log_odds_ratio": -0.4624846875667572,
      "logits/chosen": -0.2076461762189865,
      "logits/rejected": -1.0934462547302246,
      "logps/chosen": -1.580552339553833,
      "logps/rejected": -2.033411741256714,
      "loss": 1.6271,
      "nll_loss": 1.5808457136154175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15805523097515106,
      "rewards/margins": 0.045285940170288086,
      "rewards/rejected": -0.20334117114543915,
      "step": 512
    },
    {
      "epoch": 0.8085106382978723,
      "grad_norm": 0.19801148772239685,
      "learning_rate": 2.980897434998293e-06,
      "log_odds_chosen": 0.7844054698944092,
      "log_odds_ratio": -0.38332870602607727,
      "logits/chosen": -0.27720344066619873,
      "logits/rejected": -1.1505428552627563,
      "logps/chosen": -1.4240094423294067,
      "logps/rejected": -2.0650014877319336,
      "loss": 1.4864,
      "nll_loss": 1.4480822086334229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14240095019340515,
      "rewards/margins": 0.06409922987222672,
      "rewards/rejected": -0.20650017261505127,
      "step": 513
    },
    {
      "epoch": 0.8100866824271079,
      "grad_norm": 0.2111295759677887,
      "learning_rate": 2.976103177883374e-06,
      "log_odds_chosen": 0.5484133362770081,
      "log_odds_ratio": -0.4681061804294586,
      "logits/chosen": -0.22659584879875183,
      "logits/rejected": -1.0482819080352783,
      "logps/chosen": -1.577925443649292,
      "logps/rejected": -2.0398576259613037,
      "loss": 1.6314,
      "nll_loss": 1.5846238136291504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15779253840446472,
      "rewards/margins": 0.04619321972131729,
      "rewards/rejected": -0.2039857655763626,
      "step": 514
    },
    {
      "epoch": 0.8116627265563435,
      "grad_norm": 0.1878584623336792,
      "learning_rate": 2.971301546775167e-06,
      "log_odds_chosen": 0.7004796862602234,
      "log_odds_ratio": -0.40559083223342896,
      "logits/chosen": -0.28382182121276855,
      "logits/rejected": -1.3315712213516235,
      "logps/chosen": -1.4837517738342285,
      "logps/rejected": -2.060093641281128,
      "loss": 1.5454,
      "nll_loss": 1.5048052072525024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14837519824504852,
      "rewards/margins": 0.057634174823760986,
      "rewards/rejected": -0.2060093730688095,
      "step": 515
    },
    {
      "epoch": 0.8132387706855791,
      "grad_norm": 0.1968100666999817,
      "learning_rate": 2.966492577947704e-06,
      "log_odds_chosen": 0.6331671476364136,
      "log_odds_ratio": -0.42988285422325134,
      "logits/chosen": -0.2608785927295685,
      "logits/rejected": -1.3226923942565918,
      "logps/chosen": -1.5938044786453247,
      "logps/rejected": -2.1265294551849365,
      "loss": 1.6597,
      "nll_loss": 1.6167157888412476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1593804508447647,
      "rewards/margins": 0.05327250435948372,
      "rewards/rejected": -0.21265296638011932,
      "step": 516
    },
    {
      "epoch": 0.8148148148148148,
      "grad_norm": 0.1844560205936432,
      "learning_rate": 2.9616763077304457e-06,
      "log_odds_chosen": 0.7213395833969116,
      "log_odds_ratio": -0.400473952293396,
      "logits/chosen": -0.2046661674976349,
      "logits/rejected": -1.3109172582626343,
      "logps/chosen": -1.5215797424316406,
      "logps/rejected": -2.121401309967041,
      "loss": 1.5903,
      "nll_loss": 1.550216555595398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1521579921245575,
      "rewards/margins": 0.059982165694236755,
      "rewards/rejected": -0.21214015781879425,
      "step": 517
    },
    {
      "epoch": 0.8163908589440504,
      "grad_norm": 0.18234987556934357,
      "learning_rate": 2.956852772508014e-06,
      "log_odds_chosen": 0.5562025904655457,
      "log_odds_ratio": -0.46611055731773376,
      "logits/chosen": -0.21094052493572235,
      "logits/rejected": -0.9477425217628479,
      "logps/chosen": -1.4239654541015625,
      "logps/rejected": -1.87904691696167,
      "loss": 1.4868,
      "nll_loss": 1.4401750564575195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14239655435085297,
      "rewards/margins": 0.04550812020897865,
      "rewards/rejected": -0.18790468573570251,
      "step": 518
    },
    {
      "epoch": 0.817966903073286,
      "grad_norm": 0.20299586653709412,
      "learning_rate": 2.952022008719914e-06,
      "log_odds_chosen": 0.42628350853919983,
      "log_odds_ratio": -0.5056624412536621,
      "logits/chosen": -0.19726808369159698,
      "logits/rejected": -0.9944378733634949,
      "logps/chosen": -1.5680383443832397,
      "logps/rejected": -1.919787883758545,
      "loss": 1.6248,
      "nll_loss": 1.5742021799087524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15680384635925293,
      "rewards/margins": 0.03517494350671768,
      "rewards/rejected": -0.19197878241539001,
      "step": 519
    },
    {
      "epoch": 0.8195429472025216,
      "grad_norm": 0.1842677742242813,
      "learning_rate": 2.9471840528602573e-06,
      "log_odds_chosen": 0.599527895450592,
      "log_odds_ratio": -0.4468288719654083,
      "logits/chosen": -0.3563230335712433,
      "logits/rejected": -1.0952818393707275,
      "logps/chosen": -1.509791612625122,
      "logps/rejected": -2.0040204524993896,
      "loss": 1.5632,
      "nll_loss": 1.518543004989624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1509791612625122,
      "rewards/margins": 0.04942290484905243,
      "rewards/rejected": -0.20040206611156464,
      "step": 520
    },
    {
      "epoch": 0.8211189913317573,
      "grad_norm": 0.19890955090522766,
      "learning_rate": 2.9423389414774914e-06,
      "log_odds_chosen": 0.6644148826599121,
      "log_odds_ratio": -0.42108988761901855,
      "logits/chosen": -0.31821855902671814,
      "logits/rejected": -1.353883981704712,
      "logps/chosen": -1.4735428094863892,
      "logps/rejected": -2.021632671356201,
      "loss": 1.5324,
      "nll_loss": 1.4902606010437012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735430479049683,
      "rewards/margins": 0.05480898544192314,
      "rewards/rejected": -0.20216327905654907,
      "step": 521
    },
    {
      "epoch": 0.8226950354609929,
      "grad_norm": 0.18903915584087372,
      "learning_rate": 2.9374867111741174e-06,
      "log_odds_chosen": 0.6556580662727356,
      "log_odds_ratio": -0.42317891120910645,
      "logits/chosen": -0.20167070627212524,
      "logits/rejected": -1.3227620124816895,
      "logps/chosen": -1.4989482164382935,
      "logps/rejected": -2.040137529373169,
      "loss": 1.5513,
      "nll_loss": 1.5090069770812988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1498948335647583,
      "rewards/margins": 0.054118938744068146,
      "rewards/rejected": -0.20401374995708466,
      "step": 522
    },
    {
      "epoch": 0.8242710795902285,
      "grad_norm": 0.19492323696613312,
      "learning_rate": 2.9326273986064177e-06,
      "log_odds_chosen": 0.7035064101219177,
      "log_odds_ratio": -0.4070170819759369,
      "logits/chosen": -0.24931682646274567,
      "logits/rejected": -1.230039119720459,
      "logps/chosen": -1.5197010040283203,
      "logps/rejected": -2.1036765575408936,
      "loss": 1.5835,
      "nll_loss": 1.5428440570831299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15197008848190308,
      "rewards/margins": 0.05839754641056061,
      "rewards/rejected": -0.21036764979362488,
      "step": 523
    },
    {
      "epoch": 0.8258471237194641,
      "grad_norm": 0.2022964209318161,
      "learning_rate": 2.9277610404841787e-06,
      "log_odds_chosen": 0.5629186630249023,
      "log_odds_ratio": -0.45205625891685486,
      "logits/chosen": -0.2926446795463562,
      "logits/rejected": -1.0600086450576782,
      "logps/chosen": -1.4968057870864868,
      "logps/rejected": -1.9585050344467163,
      "loss": 1.5554,
      "nll_loss": 1.5101723670959473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14968058466911316,
      "rewards/margins": 0.046169936656951904,
      "rewards/rejected": -0.19585050642490387,
      "step": 524
    },
    {
      "epoch": 0.8274231678486997,
      "grad_norm": 0.21457691490650177,
      "learning_rate": 2.9228876735704107e-06,
      "log_odds_chosen": 0.5206466317176819,
      "log_odds_ratio": -0.4695855677127838,
      "logits/chosen": -0.260994017124176,
      "logits/rejected": -1.1955437660217285,
      "logps/chosen": -1.3916503190994263,
      "logps/rejected": -1.804068922996521,
      "loss": 1.4608,
      "nll_loss": 1.4138046503067017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13916504383087158,
      "rewards/margins": 0.0412418469786644,
      "rewards/rejected": -0.18040689826011658,
      "step": 525
    },
    {
      "epoch": 0.8289992119779354,
      "grad_norm": 0.2018660604953766,
      "learning_rate": 2.9180073346810738e-06,
      "log_odds_chosen": 0.5272499918937683,
      "log_odds_ratio": -0.4673810303211212,
      "logits/chosen": -0.2688242793083191,
      "logits/rejected": -1.2237880229949951,
      "logps/chosen": -1.6564934253692627,
      "logps/rejected": -2.1022491455078125,
      "loss": 1.6945,
      "nll_loss": 1.6477755308151245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16564933955669403,
      "rewards/margins": 0.044575564563274384,
      "rewards/rejected": -0.210224911570549,
      "step": 526
    },
    {
      "epoch": 0.830575256107171,
      "grad_norm": 0.1852441281080246,
      "learning_rate": 2.9131200606847957e-06,
      "log_odds_chosen": 0.5951411128044128,
      "log_odds_ratio": -0.44759196043014526,
      "logits/chosen": -0.21725840866565704,
      "logits/rejected": -1.2157734632492065,
      "logps/chosen": -1.5244123935699463,
      "logps/rejected": -2.0174002647399902,
      "loss": 1.5821,
      "nll_loss": 1.5373502969741821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15244124829769135,
      "rewards/margins": 0.04929877072572708,
      "rewards/rejected": -0.20173999667167664,
      "step": 527
    },
    {
      "epoch": 0.8321513002364066,
      "grad_norm": 0.19027827680110931,
      "learning_rate": 2.9082258885025995e-06,
      "log_odds_chosen": 0.6706474423408508,
      "log_odds_ratio": -0.42060303688049316,
      "logits/chosen": -0.3629383146762848,
      "logits/rejected": -1.2364530563354492,
      "logps/chosen": -1.5734461545944214,
      "logps/rejected": -2.1350913047790527,
      "loss": 1.62,
      "nll_loss": 1.5778906345367432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15734460949897766,
      "rewards/margins": 0.056164517998695374,
      "rewards/rejected": -0.21350912749767303,
      "step": 528
    },
    {
      "epoch": 0.8337273443656422,
      "grad_norm": 0.19536390900611877,
      "learning_rate": 2.9033248551076167e-06,
      "log_odds_chosen": 0.5333554744720459,
      "log_odds_ratio": -0.46403801441192627,
      "logits/chosen": -0.21269378066062927,
      "logits/rejected": -1.0774928331375122,
      "logps/chosen": -1.6315186023712158,
      "logps/rejected": -2.0792744159698486,
      "loss": 1.6871,
      "nll_loss": 1.640698790550232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16315187513828278,
      "rewards/margins": 0.044775549322366714,
      "rewards/rejected": -0.207927405834198,
      "step": 529
    },
    {
      "epoch": 0.8353033884948778,
      "grad_norm": 0.19617049396038055,
      "learning_rate": 2.8984169975248138e-06,
      "log_odds_chosen": 0.7091816663742065,
      "log_odds_ratio": -0.4042532742023468,
      "logits/chosen": -0.2628781795501709,
      "logits/rejected": -1.142508864402771,
      "logps/chosen": -1.4974637031555176,
      "logps/rejected": -2.0825719833374023,
      "loss": 1.5639,
      "nll_loss": 1.5234944820404053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149746373295784,
      "rewards/margins": 0.05851082503795624,
      "rewards/rejected": -0.20825719833374023,
      "step": 530
    },
    {
      "epoch": 0.8368794326241135,
      "grad_norm": 0.18096669018268585,
      "learning_rate": 2.893502352830712e-06,
      "log_odds_chosen": 0.8515176773071289,
      "log_odds_ratio": -0.36125829815864563,
      "logits/chosen": -0.2902701795101166,
      "logits/rejected": -1.3389122486114502,
      "logps/chosen": -1.402230143547058,
      "logps/rejected": -2.1021738052368164,
      "loss": 1.4504,
      "nll_loss": 1.4142682552337646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14022301137447357,
      "rewards/margins": 0.06999434530735016,
      "rewards/rejected": -0.21021735668182373,
      "step": 531
    },
    {
      "epoch": 0.8384554767533491,
      "grad_norm": 0.1927252560853958,
      "learning_rate": 2.888580958153103e-06,
      "log_odds_chosen": 0.676819384098053,
      "log_odds_ratio": -0.4148905575275421,
      "logits/chosen": -0.30515238642692566,
      "logits/rejected": -1.272576928138733,
      "logps/chosen": -1.4520963430404663,
      "logps/rejected": -2.007037878036499,
      "loss": 1.5233,
      "nll_loss": 1.4817777872085571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1452096402645111,
      "rewards/margins": 0.05549415946006775,
      "rewards/rejected": -0.20070379972457886,
      "step": 532
    },
    {
      "epoch": 0.8400315208825847,
      "grad_norm": 0.19564174115657806,
      "learning_rate": 2.8836528506707733e-06,
      "log_odds_chosen": 0.7572544813156128,
      "log_odds_ratio": -0.39043116569519043,
      "logits/chosen": -0.28933510184288025,
      "logits/rejected": -1.214314341545105,
      "logps/chosen": -1.5183055400848389,
      "logps/rejected": -2.152989387512207,
      "loss": 1.5693,
      "nll_loss": 1.5302594900131226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1518305540084839,
      "rewards/margins": 0.06346838921308517,
      "rewards/rejected": -0.21529893577098846,
      "step": 533
    },
    {
      "epoch": 0.8416075650118203,
      "grad_norm": 0.18157677352428436,
      "learning_rate": 2.878718067613222e-06,
      "log_odds_chosen": 0.56183260679245,
      "log_odds_ratio": -0.4554942548274994,
      "logits/chosen": -0.2513996660709381,
      "logits/rejected": -1.1430789232254028,
      "logps/chosen": -1.5722049474716187,
      "logps/rejected": -2.0404675006866455,
      "loss": 1.6264,
      "nll_loss": 1.580894947052002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1572204977273941,
      "rewards/margins": 0.046826258301734924,
      "rewards/rejected": -0.20404675602912903,
      "step": 534
    },
    {
      "epoch": 0.843183609141056,
      "grad_norm": 0.19487778842449188,
      "learning_rate": 2.8737766462603763e-06,
      "log_odds_chosen": 0.6868119835853577,
      "log_odds_ratio": -0.4209638833999634,
      "logits/chosen": -0.23184634745121002,
      "logits/rejected": -1.023790955543518,
      "logps/chosen": -1.5112544298171997,
      "logps/rejected": -2.0872793197631836,
      "loss": 1.562,
      "nll_loss": 1.5198806524276733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1511254608631134,
      "rewards/margins": 0.057602494955062866,
      "rewards/rejected": -0.20872795581817627,
      "step": 535
    },
    {
      "epoch": 0.8447596532702916,
      "grad_norm": 0.18175427615642548,
      "learning_rate": 2.8688286239423167e-06,
      "log_odds_chosen": 0.5748245120048523,
      "log_odds_ratio": -0.45573556423187256,
      "logits/chosen": -0.24123987555503845,
      "logits/rejected": -1.2013285160064697,
      "logps/chosen": -1.5519789457321167,
      "logps/rejected": -2.0324244499206543,
      "loss": 1.597,
      "nll_loss": 1.5514299869537354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1551978886127472,
      "rewards/margins": 0.048044539988040924,
      "rewards/rejected": -0.2032424360513687,
      "step": 536
    },
    {
      "epoch": 0.8463356973995272,
      "grad_norm": 0.18757079541683197,
      "learning_rate": 2.8638740380389862e-06,
      "log_odds_chosen": 0.6606250405311584,
      "log_odds_ratio": -0.42141610383987427,
      "logits/chosen": -0.27542534470558167,
      "logits/rejected": -1.2933332920074463,
      "logps/chosen": -1.4590181112289429,
      "logps/rejected": -1.999814748764038,
      "loss": 1.5245,
      "nll_loss": 1.4823747873306274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14590182900428772,
      "rewards/margins": 0.054079651832580566,
      "rewards/rejected": -0.1999814808368683,
      "step": 537
    },
    {
      "epoch": 0.8479117415287628,
      "grad_norm": 0.18987531960010529,
      "learning_rate": 2.8589129259799164e-06,
      "log_odds_chosen": 0.7554072737693787,
      "log_odds_ratio": -0.39838239550590515,
      "logits/chosen": -0.2796870768070221,
      "logits/rejected": -1.1120367050170898,
      "logps/chosen": -1.533353328704834,
      "logps/rejected": -2.168038845062256,
      "loss": 1.5807,
      "nll_loss": 1.5408391952514648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1533353328704834,
      "rewards/margins": 0.0634685754776001,
      "rewards/rejected": -0.2168039083480835,
      "step": 538
    },
    {
      "epoch": 0.8494877856579984,
      "grad_norm": 0.18945148587226868,
      "learning_rate": 2.853945325243938e-06,
      "log_odds_chosen": 0.599510669708252,
      "log_odds_ratio": -0.43924230337142944,
      "logits/chosen": -0.2838573157787323,
      "logits/rejected": -1.1174099445343018,
      "logps/chosen": -1.5499876737594604,
      "logps/rejected": -2.0470681190490723,
      "loss": 1.5974,
      "nll_loss": 1.5535141229629517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1549987643957138,
      "rewards/margins": 0.0497080460190773,
      "rewards/rejected": -0.2047068178653717,
      "step": 539
    },
    {
      "epoch": 0.851063829787234,
      "grad_norm": 0.18368282914161682,
      "learning_rate": 2.848971273358903e-06,
      "log_odds_chosen": 0.7604623436927795,
      "log_odds_ratio": -0.38638702034950256,
      "logits/chosen": -0.339118093252182,
      "logits/rejected": -1.1363259553909302,
      "logps/chosen": -1.4819369316101074,
      "logps/rejected": -2.1129770278930664,
      "loss": 1.5168,
      "nll_loss": 1.4781277179718018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14819370210170746,
      "rewards/margins": 0.06310401856899261,
      "rewards/rejected": -0.21129772067070007,
      "step": 540
    },
    {
      "epoch": 0.8526398739164697,
      "grad_norm": 0.20985926687717438,
      "learning_rate": 2.843990807901397e-06,
      "log_odds_chosen": 0.5638411641120911,
      "log_odds_ratio": -0.45591410994529724,
      "logits/chosen": -0.24105775356292725,
      "logits/rejected": -0.9131582379341125,
      "logps/chosen": -1.569594383239746,
      "logps/rejected": -2.03865647315979,
      "loss": 1.6258,
      "nll_loss": 1.5802273750305176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1569594293832779,
      "rewards/margins": 0.046906210482120514,
      "rewards/rejected": -0.203865647315979,
      "step": 541
    },
    {
      "epoch": 0.8542159180457053,
      "grad_norm": 0.1977475881576538,
      "learning_rate": 2.839003966496458e-06,
      "log_odds_chosen": 0.7232897281646729,
      "log_odds_ratio": -0.41055354475975037,
      "logits/chosen": -0.16717611253261566,
      "logits/rejected": -1.2326276302337646,
      "logps/chosen": -1.5320007801055908,
      "logps/rejected": -2.135465621948242,
      "loss": 1.5915,
      "nll_loss": 1.55043625831604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15320007503032684,
      "rewards/margins": 0.06034648418426514,
      "rewards/rejected": -0.2135465443134308,
      "step": 542
    },
    {
      "epoch": 0.8557919621749409,
      "grad_norm": 0.17979498207569122,
      "learning_rate": 2.8340107868172905e-06,
      "log_odds_chosen": 0.6962199211120605,
      "log_odds_ratio": -0.4092254042625427,
      "logits/chosen": -0.2727755904197693,
      "logits/rejected": -1.2297756671905518,
      "logps/chosen": -1.3761147260665894,
      "logps/rejected": -1.9405932426452637,
      "loss": 1.4366,
      "nll_loss": 1.395681619644165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1376114785671234,
      "rewards/margins": 0.05644785612821579,
      "rewards/rejected": -0.1940593272447586,
      "step": 543
    },
    {
      "epoch": 0.8573680063041765,
      "grad_norm": 0.17581501603126526,
      "learning_rate": 2.8290113065849826e-06,
      "log_odds_chosen": 0.7592371106147766,
      "log_odds_ratio": -0.3903483748435974,
      "logits/chosen": -0.24543331563472748,
      "logits/rejected": -1.3742636442184448,
      "logps/chosen": -1.4833760261535645,
      "logps/rejected": -2.112884521484375,
      "loss": 1.5416,
      "nll_loss": 1.5025991201400757,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14833758771419525,
      "rewards/margins": 0.06295083463191986,
      "rewards/rejected": -0.2112884372472763,
      "step": 544
    },
    {
      "epoch": 0.8589440504334122,
      "grad_norm": 0.1828548163175583,
      "learning_rate": 2.8240055635682193e-06,
      "log_odds_chosen": 0.8841625452041626,
      "log_odds_ratio": -0.3575596213340759,
      "logits/chosen": -0.28380656242370605,
      "logits/rejected": -1.2711106538772583,
      "logps/chosen": -1.4537177085876465,
      "logps/rejected": -2.1900110244750977,
      "loss": 1.4926,
      "nll_loss": 1.4568507671356201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14537179470062256,
      "rewards/margins": 0.07362929731607437,
      "rewards/rejected": -0.21900108456611633,
      "step": 545
    },
    {
      "epoch": 0.8605200945626478,
      "grad_norm": 0.19725649058818817,
      "learning_rate": 2.8189935955829973e-06,
      "log_odds_chosen": 0.800959050655365,
      "log_odds_ratio": -0.3783296048641205,
      "logits/chosen": -0.26514673233032227,
      "logits/rejected": -1.0618634223937988,
      "logps/chosen": -1.5127689838409424,
      "logps/rejected": -2.1850204467773438,
      "loss": 1.5601,
      "nll_loss": 1.5222481489181519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15127691626548767,
      "rewards/margins": 0.0672251284122467,
      "rewards/rejected": -0.21850204467773438,
      "step": 546
    },
    {
      "epoch": 0.8620961386918834,
      "grad_norm": 0.19058074057102203,
      "learning_rate": 2.813975440492342e-06,
      "log_odds_chosen": 0.7043353319168091,
      "log_odds_ratio": -0.41070765256881714,
      "logits/chosen": -0.303777813911438,
      "logits/rejected": -1.2495200634002686,
      "logps/chosen": -1.5077966451644897,
      "logps/rejected": -2.09123158454895,
      "loss": 1.5703,
      "nll_loss": 1.5291858911514282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15077966451644897,
      "rewards/margins": 0.05834350734949112,
      "rewards/rejected": -0.2091231793165207,
      "step": 547
    },
    {
      "epoch": 0.863672182821119,
      "grad_norm": 0.20688802003860474,
      "learning_rate": 2.8089511362060182e-06,
      "log_odds_chosen": 0.5309877395629883,
      "log_odds_ratio": -0.465130478143692,
      "logits/chosen": -0.3078065812587738,
      "logits/rejected": -1.140541672706604,
      "logps/chosen": -1.6442089080810547,
      "logps/rejected": -2.0930800437927246,
      "loss": 1.6764,
      "nll_loss": 1.6298750638961792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16442088782787323,
      "rewards/margins": 0.04488709941506386,
      "rewards/rejected": -0.20930799841880798,
      "step": 548
    },
    {
      "epoch": 0.8652482269503546,
      "grad_norm": 0.19751368463039398,
      "learning_rate": 2.803920720680244e-06,
      "log_odds_chosen": 0.680429995059967,
      "log_odds_ratio": -0.41317814588546753,
      "logits/chosen": -0.2596677839756012,
      "logits/rejected": -1.2549047470092773,
      "logps/chosen": -1.5024569034576416,
      "logps/rejected": -2.0636579990386963,
      "loss": 1.5577,
      "nll_loss": 1.5163743495941162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15024568140506744,
      "rewards/margins": 0.05612010508775711,
      "rewards/rejected": -0.20636580884456635,
      "step": 549
    },
    {
      "epoch": 0.8668242710795903,
      "grad_norm": 0.20308832824230194,
      "learning_rate": 2.7988842319174075e-06,
      "log_odds_chosen": 0.7216042876243591,
      "log_odds_ratio": -0.4037800431251526,
      "logits/chosen": -0.38072335720062256,
      "logits/rejected": -1.336956262588501,
      "logps/chosen": -1.5705177783966064,
      "logps/rejected": -2.1780800819396973,
      "loss": 1.6097,
      "nll_loss": 1.5693087577819824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15705177187919617,
      "rewards/margins": 0.06075625121593475,
      "rewards/rejected": -0.21780803799629211,
      "step": 550
    },
    {
      "epoch": 0.8684003152088259,
      "grad_norm": 0.17556044459342957,
      "learning_rate": 2.7938417079657743e-06,
      "log_odds_chosen": 0.7328196167945862,
      "log_odds_ratio": -0.39776748418807983,
      "logits/chosen": -0.3215878903865814,
      "logits/rejected": -1.3354767560958862,
      "logps/chosen": -1.417104721069336,
      "logps/rejected": -2.0171279907226562,
      "loss": 1.4733,
      "nll_loss": 1.4334834814071655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14171047508716583,
      "rewards/margins": 0.060002319514751434,
      "rewards/rejected": -0.20171278715133667,
      "step": 551
    },
    {
      "epoch": 0.8699763593380615,
      "grad_norm": 0.1794237196445465,
      "learning_rate": 2.7887931869192047e-06,
      "log_odds_chosen": 0.639879584312439,
      "log_odds_ratio": -0.4270557761192322,
      "logits/chosen": -0.319123774766922,
      "logits/rejected": -1.2014118432998657,
      "logps/chosen": -1.499389886856079,
      "logps/rejected": -2.0281472206115723,
      "loss": 1.5499,
      "nll_loss": 1.5071454048156738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14993900060653687,
      "rewards/margins": 0.05287574976682663,
      "rewards/rejected": -0.2028147429227829,
      "step": 552
    },
    {
      "epoch": 0.8715524034672971,
      "grad_norm": 0.1923706829547882,
      "learning_rate": 2.783738706916865e-06,
      "log_odds_chosen": 0.7734880447387695,
      "log_odds_ratio": -0.386980801820755,
      "logits/chosen": -0.28918617963790894,
      "logits/rejected": -1.36086106300354,
      "logps/chosen": -1.5330867767333984,
      "logps/rejected": -2.1776344776153564,
      "loss": 1.5751,
      "nll_loss": 1.5364114046096802,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1533086895942688,
      "rewards/margins": 0.06445474177598953,
      "rewards/rejected": -0.21776342391967773,
      "step": 553
    },
    {
      "epoch": 0.8731284475965327,
      "grad_norm": 0.19311000406742096,
      "learning_rate": 2.7786783061429356e-06,
      "log_odds_chosen": 0.6223908066749573,
      "log_odds_ratio": -0.43119731545448303,
      "logits/chosen": -0.3557952344417572,
      "logits/rejected": -1.0282750129699707,
      "logps/chosen": -1.468430757522583,
      "logps/rejected": -1.9774752855300903,
      "loss": 1.5075,
      "nll_loss": 1.4643542766571045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1468430757522583,
      "rewards/margins": 0.05090443789958954,
      "rewards/rejected": -0.19774751365184784,
      "step": 554
    },
    {
      "epoch": 0.8747044917257684,
      "grad_norm": 0.20641811192035675,
      "learning_rate": 2.7736120228263287e-06,
      "log_odds_chosen": 0.7676912546157837,
      "log_odds_ratio": -0.38985297083854675,
      "logits/chosen": -0.271095871925354,
      "logits/rejected": -1.179206371307373,
      "logps/chosen": -1.4588353633880615,
      "logps/rejected": -2.0947041511535645,
      "loss": 1.5126,
      "nll_loss": 1.4736404418945312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14588356018066406,
      "rewards/margins": 0.06358686834573746,
      "rewards/rejected": -0.20947042107582092,
      "step": 555
    },
    {
      "epoch": 0.876280535855004,
      "grad_norm": 0.17092975974082947,
      "learning_rate": 2.768539895240394e-06,
      "log_odds_chosen": 0.7042695879936218,
      "log_odds_ratio": -0.4147370457649231,
      "logits/chosen": -0.2829998731613159,
      "logits/rejected": -1.239137053489685,
      "logps/chosen": -1.423582911491394,
      "logps/rejected": -1.995326042175293,
      "loss": 1.4746,
      "nll_loss": 1.433091640472412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14235830307006836,
      "rewards/margins": 0.057174310088157654,
      "rewards/rejected": -0.19953259825706482,
      "step": 556
    },
    {
      "epoch": 0.8778565799842396,
      "grad_norm": 0.19411912560462952,
      "learning_rate": 2.763461961702633e-06,
      "log_odds_chosen": 0.7051361203193665,
      "log_odds_ratio": -0.4103311002254486,
      "logits/chosen": -0.30394285917282104,
      "logits/rejected": -1.3334358930587769,
      "logps/chosen": -1.5348284244537354,
      "logps/rejected": -2.124016761779785,
      "loss": 1.562,
      "nll_loss": 1.5209531784057617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1534828543663025,
      "rewards/margins": 0.058918826282024384,
      "rewards/rejected": -0.21240167319774628,
      "step": 557
    },
    {
      "epoch": 0.8794326241134752,
      "grad_norm": 0.19036102294921875,
      "learning_rate": 2.758378260574409e-06,
      "log_odds_chosen": 0.64496248960495,
      "log_odds_ratio": -0.42458900809288025,
      "logits/chosen": -0.2918802797794342,
      "logits/rejected": -1.23005211353302,
      "logps/chosen": -1.5225822925567627,
      "logps/rejected": -2.0564723014831543,
      "loss": 1.5836,
      "nll_loss": 1.5411243438720703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1522582322359085,
      "rewards/margins": 0.05338897556066513,
      "rewards/rejected": -0.20564721524715424,
      "step": 558
    },
    {
      "epoch": 0.8810086682427108,
      "grad_norm": 0.18347354233264923,
      "learning_rate": 2.753288830260655e-06,
      "log_odds_chosen": 0.8980096578598022,
      "log_odds_ratio": -0.3557407557964325,
      "logits/chosen": -0.27621370553970337,
      "logits/rejected": -1.4203813076019287,
      "logps/chosen": -1.5196247100830078,
      "logps/rejected": -2.2784533500671387,
      "loss": 1.5597,
      "nll_loss": 1.5240896940231323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15196247398853302,
      "rewards/margins": 0.07588286697864532,
      "rewards/rejected": -0.22784532606601715,
      "step": 559
    },
    {
      "epoch": 0.8825847123719465,
      "grad_norm": 0.19377274811267853,
      "learning_rate": 2.7481937092095866e-06,
      "log_odds_chosen": 0.4594305753707886,
      "log_odds_ratio": -0.4980555474758148,
      "logits/chosen": -0.23293444514274597,
      "logits/rejected": -1.1925618648529053,
      "logps/chosen": -1.571787714958191,
      "logps/rejected": -1.9583957195281982,
      "loss": 1.608,
      "nll_loss": 1.5582351684570312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15717877447605133,
      "rewards/margins": 0.03866079822182655,
      "rewards/rejected": -0.1958395540714264,
      "step": 560
    },
    {
      "epoch": 0.8841607565011821,
      "grad_norm": 0.1824018657207489,
      "learning_rate": 2.7430929359124086e-06,
      "log_odds_chosen": 0.584967315196991,
      "log_odds_ratio": -0.4464002251625061,
      "logits/chosen": -0.2975609004497528,
      "logits/rejected": -0.9871141910552979,
      "logps/chosen": -1.4142837524414062,
      "logps/rejected": -1.886897087097168,
      "loss": 1.4856,
      "nll_loss": 1.4409171342849731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1414283812046051,
      "rewards/margins": 0.047261323779821396,
      "rewards/rejected": -0.1886896938085556,
      "step": 561
    },
    {
      "epoch": 0.8857368006304176,
      "grad_norm": 0.19575563073158264,
      "learning_rate": 2.737986548903029e-06,
      "log_odds_chosen": 0.9707343578338623,
      "log_odds_ratio": -0.3241675794124603,
      "logits/chosen": -0.48363178968429565,
      "logits/rejected": -1.2981561422348022,
      "logps/chosen": -1.393601417541504,
      "logps/rejected": -2.1967051029205322,
      "loss": 1.4536,
      "nll_loss": 1.4211547374725342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13936014473438263,
      "rewards/margins": 0.08031036704778671,
      "rewards/rejected": -0.21967050433158875,
      "step": 562
    },
    {
      "epoch": 0.8873128447596532,
      "grad_norm": 0.18978776037693024,
      "learning_rate": 2.7328745867577604e-06,
      "log_odds_chosen": 0.5844587087631226,
      "log_odds_ratio": -0.4450819492340088,
      "logits/chosen": -0.2737792432308197,
      "logits/rejected": -1.2348958253860474,
      "logps/chosen": -1.5510786771774292,
      "logps/rejected": -2.0364534854888916,
      "loss": 1.6081,
      "nll_loss": 1.5636062622070312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15510787069797516,
      "rewards/margins": 0.048537470400333405,
      "rewards/rejected": -0.20364536345005035,
      "step": 563
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 0.19481179118156433,
      "learning_rate": 2.727757088095037e-06,
      "log_odds_chosen": 0.7795647978782654,
      "log_odds_ratio": -0.3803843855857849,
      "logits/chosen": -0.28297457098960876,
      "logits/rejected": -1.269483208656311,
      "logps/chosen": -1.5389131307601929,
      "logps/rejected": -2.192189931869507,
      "loss": 1.5824,
      "nll_loss": 1.5444093942642212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15389132499694824,
      "rewards/margins": 0.06532768160104752,
      "rewards/rejected": -0.21921899914741516,
      "step": 564
    },
    {
      "epoch": 0.8904649330181245,
      "grad_norm": 0.1915532797574997,
      "learning_rate": 2.7226340915751156e-06,
      "log_odds_chosen": 0.6808120012283325,
      "log_odds_ratio": -0.41644683480262756,
      "logits/chosen": -0.2742800712585449,
      "logits/rejected": -1.0535697937011719,
      "logps/chosen": -1.5436722040176392,
      "logps/rejected": -2.11293625831604,
      "loss": 1.6009,
      "nll_loss": 1.559282660484314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15436722338199615,
      "rewards/margins": 0.05692639946937561,
      "rewards/rejected": -0.21129362285137177,
      "step": 565
    },
    {
      "epoch": 0.8920409771473601,
      "grad_norm": 0.18501175940036774,
      "learning_rate": 2.7175056358997887e-06,
      "log_odds_chosen": 0.5376628041267395,
      "log_odds_ratio": -0.4701690375804901,
      "logits/chosen": -0.20144999027252197,
      "logits/rejected": -1.336869478225708,
      "logps/chosen": -1.5175023078918457,
      "logps/rejected": -1.961540699005127,
      "loss": 1.5662,
      "nll_loss": 1.5192195177078247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15175022184848785,
      "rewards/margins": 0.04440385848283768,
      "rewards/rejected": -0.19615407288074493,
      "step": 566
    },
    {
      "epoch": 0.8936170212765957,
      "grad_norm": 0.1886647492647171,
      "learning_rate": 2.7123717598120892e-06,
      "log_odds_chosen": 0.790952205657959,
      "log_odds_ratio": -0.38624221086502075,
      "logits/chosen": -0.39366939663887024,
      "logits/rejected": -1.2957227230072021,
      "logps/chosen": -1.4675683975219727,
      "logps/rejected": -2.1293246746063232,
      "loss": 1.5171,
      "nll_loss": 1.4785038232803345,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1467568427324295,
      "rewards/margins": 0.06617565453052521,
      "rewards/rejected": -0.21293248236179352,
      "step": 567
    },
    {
      "epoch": 0.8951930654058313,
      "grad_norm": 0.1779090315103531,
      "learning_rate": 2.7072325020959985e-06,
      "log_odds_chosen": 0.7795268297195435,
      "log_odds_ratio": -0.38284146785736084,
      "logits/chosen": -0.22315669059753418,
      "logits/rejected": -0.9971106648445129,
      "logps/chosen": -1.3655681610107422,
      "logps/rejected": -1.9968560934066772,
      "loss": 1.4357,
      "nll_loss": 1.3973870277404785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13655683398246765,
      "rewards/margins": 0.06312879920005798,
      "rewards/rejected": -0.19968558847904205,
      "step": 568
    },
    {
      "epoch": 0.8967691095350669,
      "grad_norm": 0.20352758467197418,
      "learning_rate": 2.702087901576155e-06,
      "log_odds_chosen": 0.8048277497291565,
      "log_odds_ratio": -0.3727533519268036,
      "logits/chosen": -0.17761847376823425,
      "logits/rejected": -1.168001651763916,
      "logps/chosen": -1.5068871974945068,
      "logps/rejected": -2.180389881134033,
      "loss": 1.5388,
      "nll_loss": 1.5015687942504883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15068870782852173,
      "rewards/margins": 0.06735028326511383,
      "rewards/rejected": -0.21803900599479675,
      "step": 569
    },
    {
      "epoch": 0.8983451536643026,
      "grad_norm": 0.1944047510623932,
      "learning_rate": 2.6969379971175576e-06,
      "log_odds_chosen": 0.7388870716094971,
      "log_odds_ratio": -0.3958666920661926,
      "logits/chosen": -0.27514928579330444,
      "logits/rejected": -1.3670649528503418,
      "logps/chosen": -1.550680160522461,
      "logps/rejected": -2.1719138622283936,
      "loss": 1.6026,
      "nll_loss": 1.563027262687683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1550680249929428,
      "rewards/margins": 0.0621233806014061,
      "rewards/rejected": -0.2171914130449295,
      "step": 570
    },
    {
      "epoch": 0.8999211977935382,
      "grad_norm": 0.20092549920082092,
      "learning_rate": 2.6917828276252745e-06,
      "log_odds_chosen": 0.8257419466972351,
      "log_odds_ratio": -0.36857298016548157,
      "logits/chosen": -0.2644127309322357,
      "logits/rejected": -1.2231860160827637,
      "logps/chosen": -1.4628832340240479,
      "logps/rejected": -2.1461424827575684,
      "loss": 1.5094,
      "nll_loss": 1.4725046157836914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14628832042217255,
      "rewards/margins": 0.0683259516954422,
      "rewards/rejected": -0.21461427211761475,
      "step": 571
    },
    {
      "epoch": 0.9014972419227738,
      "grad_norm": 0.18869692087173462,
      "learning_rate": 2.686622432044149e-06,
      "log_odds_chosen": 0.6543504595756531,
      "log_odds_ratio": -0.4234500527381897,
      "logits/chosen": -0.270676851272583,
      "logits/rejected": -1.2422775030136108,
      "logps/chosen": -1.408610463142395,
      "logps/rejected": -1.9409794807434082,
      "loss": 1.4716,
      "nll_loss": 1.4292057752609253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14086103439331055,
      "rewards/margins": 0.053236909210681915,
      "rewards/rejected": -0.19409795105457306,
      "step": 572
    },
    {
      "epoch": 0.9030732860520094,
      "grad_norm": 0.1846873015165329,
      "learning_rate": 2.681456849358505e-06,
      "log_odds_chosen": 0.7041017413139343,
      "log_odds_ratio": -0.41950079798698425,
      "logits/chosen": -0.3534983992576599,
      "logits/rejected": -1.3136813640594482,
      "logps/chosen": -1.452791690826416,
      "logps/rejected": -2.034878969192505,
      "loss": 1.5064,
      "nll_loss": 1.4644930362701416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1452791690826416,
      "rewards/margins": 0.05820874124765396,
      "rewards/rejected": -0.20348790287971497,
      "step": 573
    },
    {
      "epoch": 0.904649330181245,
      "grad_norm": 0.1766337901353836,
      "learning_rate": 2.6762861185918528e-06,
      "log_odds_chosen": 1.059026837348938,
      "log_odds_ratio": -0.32642221450805664,
      "logits/chosen": -0.37803950905799866,
      "logits/rejected": -1.2840404510498047,
      "logps/chosen": -1.3629827499389648,
      "logps/rejected": -2.2469286918640137,
      "loss": 1.4144,
      "nll_loss": 1.381784439086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1362982839345932,
      "rewards/margins": 0.08839457482099533,
      "rewards/rejected": -0.22469285130500793,
      "step": 574
    },
    {
      "epoch": 0.9062253743104807,
      "grad_norm": 0.19243809580802917,
      "learning_rate": 2.6711102788065934e-06,
      "log_odds_chosen": 0.8111237287521362,
      "log_odds_ratio": -0.37883177399635315,
      "logits/chosen": -0.29021063446998596,
      "logits/rejected": -1.2170164585113525,
      "logps/chosen": -1.54361891746521,
      "logps/rejected": -2.2324376106262207,
      "loss": 1.595,
      "nll_loss": 1.5571472644805908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15436188876628876,
      "rewards/margins": 0.06888186931610107,
      "rewards/rejected": -0.22324374318122864,
      "step": 575
    },
    {
      "epoch": 0.9078014184397163,
      "grad_norm": 0.1997518390417099,
      "learning_rate": 2.665929369103724e-06,
      "log_odds_chosen": 0.5232582688331604,
      "log_odds_ratio": -0.4723392724990845,
      "logits/chosen": -0.2458963543176651,
      "logits/rejected": -0.9681642055511475,
      "logps/chosen": -1.5918190479278564,
      "logps/rejected": -2.023986339569092,
      "loss": 1.6313,
      "nll_loss": 1.5840752124786377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15918190777301788,
      "rewards/margins": 0.04321672022342682,
      "rewards/rejected": -0.2023986279964447,
      "step": 576
    },
    {
      "epoch": 0.9093774625689519,
      "grad_norm": 0.19454072415828705,
      "learning_rate": 2.6607434286225427e-06,
      "log_odds_chosen": 0.7112796902656555,
      "log_odds_ratio": -0.40778082609176636,
      "logits/chosen": -0.23803873360157013,
      "logits/rejected": -1.235764503479004,
      "logps/chosen": -1.4846206903457642,
      "logps/rejected": -2.076571464538574,
      "loss": 1.5408,
      "nll_loss": 1.500047206878662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14846207201480865,
      "rewards/margins": 0.05919507145881653,
      "rewards/rejected": -0.20765715837478638,
      "step": 577
    },
    {
      "epoch": 0.9109535066981875,
      "grad_norm": 0.18670088052749634,
      "learning_rate": 2.6555524965403533e-06,
      "log_odds_chosen": 0.9150053262710571,
      "log_odds_ratio": -0.33961746096611023,
      "logits/chosen": -0.3448455333709717,
      "logits/rejected": -1.2186771631240845,
      "logps/chosen": -1.4050543308258057,
      "logps/rejected": -2.161069631576538,
      "loss": 1.4671,
      "nll_loss": 1.4331532716751099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14050543308258057,
      "rewards/margins": 0.07560153305530548,
      "rewards/rejected": -0.21610695123672485,
      "step": 578
    },
    {
      "epoch": 0.9125295508274232,
      "grad_norm": 0.18531572818756104,
      "learning_rate": 2.6503566120721683e-06,
      "log_odds_chosen": 0.8721405863761902,
      "log_odds_ratio": -0.35218220949172974,
      "logits/chosen": -0.28859424591064453,
      "logits/rejected": -1.2842233180999756,
      "logps/chosen": -1.4932284355163574,
      "logps/rejected": -2.2227084636688232,
      "loss": 1.5441,
      "nll_loss": 1.508841872215271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14932283759117126,
      "rewards/margins": 0.07294801622629166,
      "rewards/rejected": -0.22227084636688232,
      "step": 579
    },
    {
      "epoch": 0.9141055949566588,
      "grad_norm": 0.18429619073867798,
      "learning_rate": 2.6451558144704126e-06,
      "log_odds_chosen": 0.8276745676994324,
      "log_odds_ratio": -0.3685353398323059,
      "logits/chosen": -0.4071681499481201,
      "logits/rejected": -1.240911602973938,
      "logps/chosen": -1.4572088718414307,
      "logps/rejected": -2.1435623168945312,
      "loss": 1.4972,
      "nll_loss": 1.4603551626205444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14572086930274963,
      "rewards/margins": 0.06863535195589066,
      "rewards/rejected": -0.2143562287092209,
      "step": 580
    },
    {
      "epoch": 0.9156816390858944,
      "grad_norm": 0.1941244900226593,
      "learning_rate": 2.6399501430246286e-06,
      "log_odds_chosen": 0.7177987098693848,
      "log_odds_ratio": -0.39787161350250244,
      "logits/chosen": -0.2887517213821411,
      "logits/rejected": -1.1689492464065552,
      "logps/chosen": -1.4312595129013062,
      "logps/rejected": -2.0155599117279053,
      "loss": 1.4782,
      "nll_loss": 1.4384539127349854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14312595129013062,
      "rewards/margins": 0.05843004584312439,
      "rewards/rejected": -0.201555997133255,
      "step": 581
    },
    {
      "epoch": 0.91725768321513,
      "grad_norm": 0.18542201817035675,
      "learning_rate": 2.634739637061177e-06,
      "log_odds_chosen": 0.5925948619842529,
      "log_odds_ratio": -0.4440345764160156,
      "logits/chosen": -0.3084586262702942,
      "logits/rejected": -1.1862590312957764,
      "logps/chosen": -1.4869333505630493,
      "logps/rejected": -1.9728469848632812,
      "loss": 1.5407,
      "nll_loss": 1.4962780475616455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14869333803653717,
      "rewards/margins": 0.04859134927392006,
      "rewards/rejected": -0.19728469848632812,
      "step": 582
    },
    {
      "epoch": 0.9188337273443656,
      "grad_norm": 0.19448573887348175,
      "learning_rate": 2.6295243359429423e-06,
      "log_odds_chosen": 0.8545708060264587,
      "log_odds_ratio": -0.3594302237033844,
      "logits/chosen": -0.2909431457519531,
      "logits/rejected": -1.28232741355896,
      "logps/chosen": -1.5018738508224487,
      "logps/rejected": -2.2196271419525146,
      "loss": 1.5477,
      "nll_loss": 1.5117415189743042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1501873880624771,
      "rewards/margins": 0.07177533954381943,
      "rewards/rejected": -0.22196273505687714,
      "step": 583
    },
    {
      "epoch": 0.9204097714736013,
      "grad_norm": 0.18692518770694733,
      "learning_rate": 2.624304279069033e-06,
      "log_odds_chosen": 0.8368571400642395,
      "log_odds_ratio": -0.36407989263534546,
      "logits/chosen": -0.26759836077690125,
      "logits/rejected": -1.1975818872451782,
      "logps/chosen": -1.4316425323486328,
      "logps/rejected": -2.1195216178894043,
      "loss": 1.5006,
      "nll_loss": 1.4641568660736084,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431642472743988,
      "rewards/margins": 0.06878793239593506,
      "rewards/rejected": -0.21195219457149506,
      "step": 584
    },
    {
      "epoch": 0.9219858156028369,
      "grad_norm": 0.19453899562358856,
      "learning_rate": 2.6190795058744854e-06,
      "log_odds_chosen": 0.7895195484161377,
      "log_odds_ratio": -0.38437679409980774,
      "logits/chosen": -0.3300016224384308,
      "logits/rejected": -1.2094289064407349,
      "logps/chosen": -1.5207749605178833,
      "logps/rejected": -2.185511589050293,
      "loss": 1.5679,
      "nll_loss": 1.5294761657714844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15207749605178833,
      "rewards/margins": 0.06647366285324097,
      "rewards/rejected": -0.2185511589050293,
      "step": 585
    },
    {
      "epoch": 0.9235618597320725,
      "grad_norm": 0.20601381361484528,
      "learning_rate": 2.6138500558299664e-06,
      "log_odds_chosen": 0.6721675992012024,
      "log_odds_ratio": -0.4189746081829071,
      "logits/chosen": -0.30033591389656067,
      "logits/rejected": -1.0976755619049072,
      "logps/chosen": -1.505319356918335,
      "logps/rejected": -2.0645265579223633,
      "loss": 1.5525,
      "nll_loss": 1.5105657577514648,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15053193271160126,
      "rewards/margins": 0.055920716375112534,
      "rewards/rejected": -0.2064526528120041,
      "step": 586
    },
    {
      "epoch": 0.9251379038613081,
      "grad_norm": 0.19991175830364227,
      "learning_rate": 2.608615968441472e-06,
      "log_odds_chosen": 0.7707000970840454,
      "log_odds_ratio": -0.3885457217693329,
      "logits/chosen": -0.3386705815792084,
      "logits/rejected": -1.2245614528656006,
      "logps/chosen": -1.5440244674682617,
      "logps/rejected": -2.1955931186676025,
      "loss": 1.5936,
      "nll_loss": 1.5547609329223633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544024497270584,
      "rewards/margins": 0.06515685468912125,
      "rewards/rejected": -0.21955931186676025,
      "step": 587
    },
    {
      "epoch": 0.9267139479905437,
      "grad_norm": 0.19368569552898407,
      "learning_rate": 2.6033772832500333e-06,
      "log_odds_chosen": 0.7014005780220032,
      "log_odds_ratio": -0.407523512840271,
      "logits/chosen": -0.369717538356781,
      "logits/rejected": -1.158593773841858,
      "logps/chosen": -1.5436320304870605,
      "logps/rejected": -2.1312358379364014,
      "loss": 1.5933,
      "nll_loss": 1.5525622367858887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.154363214969635,
      "rewards/margins": 0.0587603934109211,
      "rewards/rejected": -0.2131236046552658,
      "step": 588
    },
    {
      "epoch": 0.9282899921197794,
      "grad_norm": 0.1853727251291275,
      "learning_rate": 2.5981340398314146e-06,
      "log_odds_chosen": 0.7536918520927429,
      "log_odds_ratio": -0.39509516954421997,
      "logits/chosen": -0.3868734836578369,
      "logits/rejected": -1.2833107709884644,
      "logps/chosen": -1.4288896322250366,
      "logps/rejected": -2.04567289352417,
      "loss": 1.4835,
      "nll_loss": 1.4439959526062012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14288897812366486,
      "rewards/margins": 0.06167832016944885,
      "rewards/rejected": -0.2045673131942749,
      "step": 589
    },
    {
      "epoch": 0.929866036249015,
      "grad_norm": 0.19667775928974152,
      "learning_rate": 2.592886277795815e-06,
      "log_odds_chosen": 0.7321896553039551,
      "log_odds_ratio": -0.4019528031349182,
      "logits/chosen": -0.3382108211517334,
      "logits/rejected": -1.4951646327972412,
      "logps/chosen": -1.5511035919189453,
      "logps/rejected": -2.168656349182129,
      "loss": 1.6085,
      "nll_loss": 1.5683385133743286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15511035919189453,
      "rewards/margins": 0.06175526976585388,
      "rewards/rejected": -0.2168656438589096,
      "step": 590
    },
    {
      "epoch": 0.9314420803782506,
      "grad_norm": 0.19306617975234985,
      "learning_rate": 2.5876340367875706e-06,
      "log_odds_chosen": 0.8973714113235474,
      "log_odds_ratio": -0.34914836287498474,
      "logits/chosen": -0.3456706702709198,
      "logits/rejected": -1.403714895248413,
      "logps/chosen": -1.5034680366516113,
      "logps/rejected": -2.259228467941284,
      "loss": 1.5562,
      "nll_loss": 1.5212527513504028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503468006849289,
      "rewards/margins": 0.075576052069664,
      "rewards/rejected": -0.2259228676557541,
      "step": 591
    },
    {
      "epoch": 0.9330181245074862,
      "grad_norm": 0.18300200998783112,
      "learning_rate": 2.582377356484853e-06,
      "log_odds_chosen": 0.7273343801498413,
      "log_odds_ratio": -0.39778101444244385,
      "logits/chosen": -0.36989137530326843,
      "logits/rejected": -1.228708028793335,
      "logps/chosen": -1.3983983993530273,
      "logps/rejected": -1.9865732192993164,
      "loss": 1.4611,
      "nll_loss": 1.4213485717773438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13983985781669617,
      "rewards/margins": 0.058817483484745026,
      "rewards/rejected": -0.1986573338508606,
      "step": 592
    },
    {
      "epoch": 0.9345941686367218,
      "grad_norm": 0.1917629837989807,
      "learning_rate": 2.577116276599373e-06,
      "log_odds_chosen": 0.7001888751983643,
      "log_odds_ratio": -0.4097610116004944,
      "logits/chosen": -0.3433193564414978,
      "logits/rejected": -0.9953939914703369,
      "logps/chosen": -1.430060863494873,
      "logps/rejected": -2.000685930252075,
      "loss": 1.5045,
      "nll_loss": 1.463550329208374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430060863494873,
      "rewards/margins": 0.05706249177455902,
      "rewards/rejected": -0.20006857812404633,
      "step": 593
    },
    {
      "epoch": 0.9361702127659575,
      "grad_norm": 0.192801833152771,
      "learning_rate": 2.5718508368760737e-06,
      "log_odds_chosen": 0.729022741317749,
      "log_odds_ratio": -0.41591376066207886,
      "logits/chosen": -0.36778977513313293,
      "logits/rejected": -1.2273101806640625,
      "logps/chosen": -1.4380747079849243,
      "logps/rejected": -2.0406250953674316,
      "loss": 1.5031,
      "nll_loss": 1.4614698886871338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14380745589733124,
      "rewards/margins": 0.06025506556034088,
      "rewards/rejected": -0.2040625363588333,
      "step": 594
    },
    {
      "epoch": 0.9377462568951931,
      "grad_norm": 0.1826949417591095,
      "learning_rate": 2.5665810770928386e-06,
      "log_odds_chosen": 0.8492671847343445,
      "log_odds_ratio": -0.3606170415878296,
      "logits/chosen": -0.3790428042411804,
      "logits/rejected": -1.339025855064392,
      "logps/chosen": -1.4300899505615234,
      "logps/rejected": -2.1312625408172607,
      "loss": 1.4992,
      "nll_loss": 1.4631707668304443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430089920759201,
      "rewards/margins": 0.07011725008487701,
      "rewards/rejected": -0.2131262570619583,
      "step": 595
    },
    {
      "epoch": 0.9393223010244287,
      "grad_norm": 0.1927093267440796,
      "learning_rate": 2.5613070370601863e-06,
      "log_odds_chosen": 0.8199338912963867,
      "log_odds_ratio": -0.3674444258213043,
      "logits/chosen": -0.3175826370716095,
      "logits/rejected": -1.2436487674713135,
      "logps/chosen": -1.482853889465332,
      "logps/rejected": -2.1671628952026367,
      "loss": 1.5225,
      "nll_loss": 1.4857112169265747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1482853889465332,
      "rewards/margins": 0.06843091547489166,
      "rewards/rejected": -0.21671628952026367,
      "step": 596
    },
    {
      "epoch": 0.9408983451536643,
      "grad_norm": 0.1973896622657776,
      "learning_rate": 2.556028756620969e-06,
      "log_odds_chosen": 0.719043493270874,
      "log_odds_ratio": -0.4020449221134186,
      "logits/chosen": -0.3283490240573883,
      "logits/rejected": -1.361223816871643,
      "logps/chosen": -1.5103546380996704,
      "logps/rejected": -2.1082468032836914,
      "loss": 1.5453,
      "nll_loss": 1.5050854682922363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15103545784950256,
      "rewards/margins": 0.059789203107357025,
      "rewards/rejected": -0.21082466840744019,
      "step": 597
    },
    {
      "epoch": 0.9424743892829,
      "grad_norm": 0.19892551004886627,
      "learning_rate": 2.5507462756500747e-06,
      "log_odds_chosen": 0.8766101002693176,
      "log_odds_ratio": -0.36278003454208374,
      "logits/chosen": -0.22360502183437347,
      "logits/rejected": -1.159963846206665,
      "logps/chosen": -1.509950876235962,
      "logps/rejected": -2.2477831840515137,
      "loss": 1.5522,
      "nll_loss": 1.5159486532211304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15099509060382843,
      "rewards/margins": 0.07378323376178741,
      "rewards/rejected": -0.22477832436561584,
      "step": 598
    },
    {
      "epoch": 0.9440504334121356,
      "grad_norm": 0.21452422440052032,
      "learning_rate": 2.5454596340541245e-06,
      "log_odds_chosen": 0.7531306743621826,
      "log_odds_ratio": -0.39862990379333496,
      "logits/chosen": -0.42769062519073486,
      "logits/rejected": -1.2770119905471802,
      "logps/chosen": -1.5458983182907104,
      "logps/rejected": -2.180636167526245,
      "loss": 1.5887,
      "nll_loss": 1.5487922430038452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15458983182907104,
      "rewards/margins": 0.06347379088401794,
      "rewards/rejected": -0.2180636078119278,
      "step": 599
    },
    {
      "epoch": 0.9456264775413712,
      "grad_norm": 0.19176235795021057,
      "learning_rate": 2.5401688717711702e-06,
      "log_odds_chosen": 0.6955916881561279,
      "log_odds_ratio": -0.4089171886444092,
      "logits/chosen": -0.32068610191345215,
      "logits/rejected": -1.2139990329742432,
      "logps/chosen": -1.4765769243240356,
      "logps/rejected": -2.0512802600860596,
      "loss": 1.5338,
      "nll_loss": 1.4928854703903198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14765769243240356,
      "rewards/margins": 0.057470306754112244,
      "rewards/rejected": -0.2051279991865158,
      "step": 600
    },
    {
      "epoch": 0.9472025216706068,
      "grad_norm": 0.18111003935337067,
      "learning_rate": 2.5348740287703937e-06,
      "log_odds_chosen": 0.8933596611022949,
      "log_odds_ratio": -0.3462783694267273,
      "logits/chosen": -0.3431352376937866,
      "logits/rejected": -1.1345089673995972,
      "logps/chosen": -1.379542350769043,
      "logps/rejected": -2.1102828979492188,
      "loss": 1.4484,
      "nll_loss": 1.413806676864624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1379542350769043,
      "rewards/margins": 0.07307406514883041,
      "rewards/rejected": -0.2110283076763153,
      "step": 601
    },
    {
      "epoch": 0.9487785657998424,
      "grad_norm": 0.20724500715732574,
      "learning_rate": 2.529575145051805e-06,
      "log_odds_chosen": 0.8153917789459229,
      "log_odds_ratio": -0.37926796078681946,
      "logits/chosen": -0.35732099413871765,
      "logits/rejected": -1.1828902959823608,
      "logps/chosen": -1.4545261859893799,
      "logps/rejected": -2.1325621604919434,
      "loss": 1.5274,
      "nll_loss": 1.4894965887069702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.145452618598938,
      "rewards/margins": 0.06780360639095306,
      "rewards/rejected": -0.21325621008872986,
      "step": 602
    },
    {
      "epoch": 0.950354609929078,
      "grad_norm": 0.19453661143779755,
      "learning_rate": 2.52427226064594e-06,
      "log_odds_chosen": 0.8606705665588379,
      "log_odds_ratio": -0.3596607744693756,
      "logits/chosen": -0.4270009696483612,
      "logits/rejected": -1.275531530380249,
      "logps/chosen": -1.4924569129943848,
      "logps/rejected": -2.2116432189941406,
      "loss": 1.5262,
      "nll_loss": 1.4902009963989258,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14924567937850952,
      "rewards/margins": 0.07191863656044006,
      "rewards/rejected": -0.22116431593894958,
      "step": 603
    },
    {
      "epoch": 0.9519306540583137,
      "grad_norm": 0.1814199835062027,
      "learning_rate": 2.518965415613557e-06,
      "log_odds_chosen": 1.0144093036651611,
      "log_odds_ratio": -0.32488948106765747,
      "logits/chosen": -0.36412313580513,
      "logits/rejected": -1.228358268737793,
      "logps/chosen": -1.385294795036316,
      "logps/rejected": -2.225130796432495,
      "loss": 1.4487,
      "nll_loss": 1.4162005186080933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13852949440479279,
      "rewards/margins": 0.08398358523845673,
      "rewards/rejected": -0.2225130945444107,
      "step": 604
    },
    {
      "epoch": 0.9535066981875493,
      "grad_norm": 0.19638565182685852,
      "learning_rate": 2.513654650045336e-06,
      "log_odds_chosen": 0.7919636964797974,
      "log_odds_ratio": -0.3775080442428589,
      "logits/chosen": -0.3657665252685547,
      "logits/rejected": -1.4252887964248657,
      "logps/chosen": -1.4191608428955078,
      "logps/rejected": -2.0698001384735107,
      "loss": 1.4703,
      "nll_loss": 1.4324991703033447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14191608130931854,
      "rewards/margins": 0.0650639459490776,
      "rewards/rejected": -0.20698001980781555,
      "step": 605
    },
    {
      "epoch": 0.9550827423167849,
      "grad_norm": 0.19228637218475342,
      "learning_rate": 2.508340004061574e-06,
      "log_odds_chosen": 0.9405322074890137,
      "log_odds_ratio": -0.33786192536354065,
      "logits/chosen": -0.3879520893096924,
      "logits/rejected": -1.0622220039367676,
      "logps/chosen": -1.4367401599884033,
      "logps/rejected": -2.2200965881347656,
      "loss": 1.4927,
      "nll_loss": 1.4588732719421387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14367400109767914,
      "rewards/margins": 0.07833566516637802,
      "rewards/rejected": -0.22200968861579895,
      "step": 606
    },
    {
      "epoch": 0.9566587864460205,
      "grad_norm": 0.19371724128723145,
      "learning_rate": 2.503021517811882e-06,
      "log_odds_chosen": 0.848656415939331,
      "log_odds_ratio": -0.3596135377883911,
      "logits/chosen": -0.4238395094871521,
      "logits/rejected": -1.3214848041534424,
      "logps/chosen": -1.5094656944274902,
      "logps/rejected": -2.221879243850708,
      "loss": 1.5607,
      "nll_loss": 1.5247186422348022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15094658732414246,
      "rewards/margins": 0.0712413638830185,
      "rewards/rejected": -0.22218795120716095,
      "step": 607
    },
    {
      "epoch": 0.9582348305752562,
      "grad_norm": 0.18693163990974426,
      "learning_rate": 2.497699231474885e-06,
      "log_odds_chosen": 0.9408037662506104,
      "log_odds_ratio": -0.34199029207229614,
      "logits/chosen": -0.39725589752197266,
      "logits/rejected": -1.3671289682388306,
      "logps/chosen": -1.4043060541152954,
      "logps/rejected": -2.1896579265594482,
      "loss": 1.4475,
      "nll_loss": 1.4132511615753174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14043059945106506,
      "rewards/margins": 0.07853520661592484,
      "rewards/rejected": -0.2189657986164093,
      "step": 608
    },
    {
      "epoch": 0.9598108747044918,
      "grad_norm": 0.1944882571697235,
      "learning_rate": 2.4923731852579127e-06,
      "log_odds_chosen": 0.8854267001152039,
      "log_odds_ratio": -0.35672587156295776,
      "logits/chosen": -0.39640650153160095,
      "logits/rejected": -1.0155848264694214,
      "logps/chosen": -1.4322024583816528,
      "logps/rejected": -2.171299934387207,
      "loss": 1.4884,
      "nll_loss": 1.4527146816253662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14322024583816528,
      "rewards/margins": 0.07390976697206497,
      "rewards/rejected": -0.21713000535964966,
      "step": 609
    },
    {
      "epoch": 0.9613869188337274,
      "grad_norm": 0.18235981464385986,
      "learning_rate": 2.4870434193967017e-06,
      "log_odds_chosen": 1.020743727684021,
      "log_odds_ratio": -0.3157598078250885,
      "logits/chosen": -0.3316628038883209,
      "logits/rejected": -1.3475199937820435,
      "logps/chosen": -1.4312551021575928,
      "logps/rejected": -2.284390449523926,
      "loss": 1.4648,
      "nll_loss": 1.4331845045089722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431255042552948,
      "rewards/margins": 0.08531351387500763,
      "rewards/rejected": -0.22843901813030243,
      "step": 610
    },
    {
      "epoch": 0.9629629629629629,
      "grad_norm": 0.2012511044740677,
      "learning_rate": 2.481709974155086e-06,
      "log_odds_chosen": 1.0592254400253296,
      "log_odds_ratio": -0.31417110562324524,
      "logits/chosen": -0.313336580991745,
      "logits/rejected": -1.1823182106018066,
      "logps/chosen": -1.4482026100158691,
      "logps/rejected": -2.341215133666992,
      "loss": 1.4907,
      "nll_loss": 1.4593302011489868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14482025802135468,
      "rewards/margins": 0.08930124342441559,
      "rewards/rejected": -0.23412150144577026,
      "step": 611
    },
    {
      "epoch": 0.9645390070921985,
      "grad_norm": 0.21659031510353088,
      "learning_rate": 2.4763728898246983e-06,
      "log_odds_chosen": 1.1309854984283447,
      "log_odds_ratio": -0.28944361209869385,
      "logits/chosen": -0.2193741351366043,
      "logits/rejected": -1.2549587488174438,
      "logps/chosen": -1.3368651866912842,
      "logps/rejected": -2.258378744125366,
      "loss": 1.3732,
      "nll_loss": 1.344224214553833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13368651270866394,
      "rewards/margins": 0.09215137362480164,
      "rewards/rejected": -0.22583790123462677,
      "step": 612
    },
    {
      "epoch": 0.9661150512214342,
      "grad_norm": 0.17772729694843292,
      "learning_rate": 2.4710322067246607e-06,
      "log_odds_chosen": 0.868254542350769,
      "log_odds_ratio": -0.3553982675075531,
      "logits/chosen": -0.29353803396224976,
      "logits/rejected": -1.1652791500091553,
      "logps/chosen": -1.4668205976486206,
      "logps/rejected": -2.188347339630127,
      "loss": 1.5014,
      "nll_loss": 1.4658981561660767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14668206870555878,
      "rewards/margins": 0.07215268909931183,
      "rewards/rejected": -0.2188347578048706,
      "step": 613
    },
    {
      "epoch": 0.9676910953506698,
      "grad_norm": 0.18264222145080566,
      "learning_rate": 2.465687965201283e-06,
      "log_odds_chosen": 0.9211742877960205,
      "log_odds_ratio": -0.34152358770370483,
      "logits/chosen": -0.3418979048728943,
      "logits/rejected": -1.214658260345459,
      "logps/chosen": -1.370938777923584,
      "logps/rejected": -2.1272804737091064,
      "loss": 1.4328,
      "nll_loss": 1.3986839056015015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13709387183189392,
      "rewards/margins": 0.0756341814994812,
      "rewards/rejected": -0.21272805333137512,
      "step": 614
    },
    {
      "epoch": 0.9692671394799054,
      "grad_norm": 0.20092345774173737,
      "learning_rate": 2.4603402056277577e-06,
      "log_odds_chosen": 0.8775561451911926,
      "log_odds_ratio": -0.35456082224845886,
      "logits/chosen": -0.47761034965515137,
      "logits/rejected": -1.194373607635498,
      "logps/chosen": -1.42928946018219,
      "logps/rejected": -2.153602123260498,
      "loss": 1.4881,
      "nll_loss": 1.452675700187683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14292894303798676,
      "rewards/margins": 0.07243126630783081,
      "rewards/rejected": -0.21536020934581757,
      "step": 615
    },
    {
      "epoch": 0.970843183609141,
      "grad_norm": 0.2074611485004425,
      "learning_rate": 2.454988968403854e-06,
      "log_odds_chosen": 0.6692450046539307,
      "log_odds_ratio": -0.4163838028907776,
      "logits/chosen": -0.3169185519218445,
      "logits/rejected": -1.244554877281189,
      "logps/chosen": -1.4843294620513916,
      "logps/rejected": -2.0351569652557373,
      "loss": 1.5207,
      "nll_loss": 1.4790477752685547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14843295514583588,
      "rewards/margins": 0.05508272722363472,
      "rewards/rejected": -0.2035156786441803,
      "step": 616
    },
    {
      "epoch": 0.9724192277383766,
      "grad_norm": 0.1868940144777298,
      "learning_rate": 2.4496342939556133e-06,
      "log_odds_chosen": 0.8804638385772705,
      "log_odds_ratio": -0.35378497838974,
      "logits/chosen": -0.25230270624160767,
      "logits/rejected": -1.206047773361206,
      "logps/chosen": -1.4417803287506104,
      "logps/rejected": -2.172455072402954,
      "loss": 1.4935,
      "nll_loss": 1.4580992460250854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14417804777622223,
      "rewards/margins": 0.07306744158267975,
      "rewards/rejected": -0.21724548935890198,
      "step": 617
    },
    {
      "epoch": 0.9739952718676123,
      "grad_norm": 0.1944309026002884,
      "learning_rate": 2.444276222735043e-06,
      "log_odds_chosen": 0.8683879375457764,
      "log_odds_ratio": -0.35284215211868286,
      "logits/chosen": -0.4312930703163147,
      "logits/rejected": -1.319032073020935,
      "logps/chosen": -1.4063115119934082,
      "logps/rejected": -2.1204988956451416,
      "loss": 1.4791,
      "nll_loss": 1.4437929391860962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14063115417957306,
      "rewards/margins": 0.07141873985528946,
      "rewards/rejected": -0.21204988658428192,
      "step": 618
    },
    {
      "epoch": 0.9755713159968479,
      "grad_norm": 0.18848849833011627,
      "learning_rate": 2.4389147952198127e-06,
      "log_odds_chosen": 0.8780885934829712,
      "log_odds_ratio": -0.3516533672809601,
      "logits/chosen": -0.45707547664642334,
      "logits/rejected": -1.2535477876663208,
      "logps/chosen": -1.4462100267410278,
      "logps/rejected": -2.1745729446411133,
      "loss": 1.4776,
      "nll_loss": 1.442458987236023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14462099969387054,
      "rewards/margins": 0.07283630967140198,
      "rewards/rejected": -0.21745732426643372,
      "step": 619
    },
    {
      "epoch": 0.9771473601260835,
      "grad_norm": 0.17683973908424377,
      "learning_rate": 2.433550051912946e-06,
      "log_odds_chosen": 0.9966714382171631,
      "log_odds_ratio": -0.3203020989894867,
      "logits/chosen": -0.3856451213359833,
      "logits/rejected": -1.339879035949707,
      "logps/chosen": -1.3389415740966797,
      "logps/rejected": -2.1509854793548584,
      "loss": 1.3899,
      "nll_loss": 1.3578870296478271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.133894145488739,
      "rewards/margins": 0.08120439201593399,
      "rewards/rejected": -0.2150985449552536,
      "step": 620
    },
    {
      "epoch": 0.9787234042553191,
      "grad_norm": 0.20172865688800812,
      "learning_rate": 2.4281820333425167e-06,
      "log_odds_chosen": 0.9100432395935059,
      "log_odds_ratio": -0.3491798937320709,
      "logits/chosen": -0.3759186565876007,
      "logits/rejected": -1.181538701057434,
      "logps/chosen": -1.5494132041931152,
      "logps/rejected": -2.3239943981170654,
      "loss": 1.5749,
      "nll_loss": 1.539953351020813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15494130551815033,
      "rewards/margins": 0.07745814323425293,
      "rewards/rejected": -0.23239943385124207,
      "step": 621
    },
    {
      "epoch": 0.9802994483845547,
      "grad_norm": 0.19874054193496704,
      "learning_rate": 2.42281078006134e-06,
      "log_odds_chosen": 1.0897847414016724,
      "log_odds_ratio": -0.2926476001739502,
      "logits/chosen": -0.40737009048461914,
      "logits/rejected": -1.458004117012024,
      "logps/chosen": -1.5359320640563965,
      "logps/rejected": -2.4667205810546875,
      "loss": 1.5721,
      "nll_loss": 1.5428738594055176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15359321236610413,
      "rewards/margins": 0.0930788442492485,
      "rewards/rejected": -0.24667204916477203,
      "step": 622
    },
    {
      "epoch": 0.9818754925137904,
      "grad_norm": 0.2077869474887848,
      "learning_rate": 2.4174363326466703e-06,
      "log_odds_chosen": 0.8463531136512756,
      "log_odds_ratio": -0.36907657980918884,
      "logits/chosen": -0.4125167727470398,
      "logits/rejected": -1.4528597593307495,
      "logps/chosen": -1.4611895084381104,
      "logps/rejected": -2.172938346862793,
      "loss": 1.5072,
      "nll_loss": 1.4703348875045776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14611895382404327,
      "rewards/margins": 0.07117487490177155,
      "rewards/rejected": -0.217293843626976,
      "step": 623
    },
    {
      "epoch": 0.983451536643026,
      "grad_norm": 0.19583779573440552,
      "learning_rate": 2.41205873169989e-06,
      "log_odds_chosen": 0.8020501136779785,
      "log_odds_ratio": -0.37741777300834656,
      "logits/chosen": -0.30660000443458557,
      "logits/rejected": -0.9770889282226562,
      "logps/chosen": -1.456507921218872,
      "logps/rejected": -2.1197385787963867,
      "loss": 1.5077,
      "nll_loss": 1.4699840545654297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14565080404281616,
      "rewards/margins": 0.06632307171821594,
      "rewards/rejected": -0.2119738757610321,
      "step": 624
    },
    {
      "epoch": 0.9850275807722616,
      "grad_norm": 0.18521183729171753,
      "learning_rate": 2.4066780178462058e-06,
      "log_odds_chosen": 0.9543585777282715,
      "log_odds_ratio": -0.33902472257614136,
      "logits/chosen": -0.2833150327205658,
      "logits/rejected": -1.1624242067337036,
      "logps/chosen": -1.4858702421188354,
      "logps/rejected": -2.2942442893981934,
      "loss": 1.5244,
      "nll_loss": 1.4904963970184326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14858704805374146,
      "rewards/margins": 0.08083738386631012,
      "rewards/rejected": -0.22942441701889038,
      "step": 625
    },
    {
      "epoch": 0.9866036249014972,
      "grad_norm": 0.18581609427928925,
      "learning_rate": 2.40129423173434e-06,
      "log_odds_chosen": 0.8655983805656433,
      "log_odds_ratio": -0.36010363698005676,
      "logits/chosen": -0.33863046765327454,
      "logits/rejected": -1.2671010494232178,
      "logps/chosen": -1.4374041557312012,
      "logps/rejected": -2.153071403503418,
      "loss": 1.4971,
      "nll_loss": 1.4610904455184937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14374041557312012,
      "rewards/margins": 0.07156671583652496,
      "rewards/rejected": -0.2153071165084839,
      "step": 626
    },
    {
      "epoch": 0.9881796690307328,
      "grad_norm": 0.2106720507144928,
      "learning_rate": 2.3959074140362274e-06,
      "log_odds_chosen": 0.8903502225875854,
      "log_odds_ratio": -0.34879085421562195,
      "logits/chosen": -0.46892446279525757,
      "logits/rejected": -1.21754789352417,
      "logps/chosen": -1.4162951707839966,
      "logps/rejected": -2.1503169536590576,
      "loss": 1.4614,
      "nll_loss": 1.426543951034546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14162953197956085,
      "rewards/margins": 0.07340216636657715,
      "rewards/rejected": -0.215031698346138,
      "step": 627
    },
    {
      "epoch": 0.9897557131599685,
      "grad_norm": 0.22940939664840698,
      "learning_rate": 2.3905176054467007e-06,
      "log_odds_chosen": 1.085053563117981,
      "log_odds_ratio": -0.30001646280288696,
      "logits/chosen": -0.4437375068664551,
      "logits/rejected": -1.241646647453308,
      "logps/chosen": -1.4003148078918457,
      "logps/rejected": -2.309556722640991,
      "loss": 1.4569,
      "nll_loss": 1.4268685579299927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14003145694732666,
      "rewards/margins": 0.0909242108464241,
      "rewards/rejected": -0.23095567524433136,
      "step": 628
    },
    {
      "epoch": 0.9913317572892041,
      "grad_norm": 0.21125461161136627,
      "learning_rate": 2.3851248466831905e-06,
      "log_odds_chosen": 0.9441803097724915,
      "log_odds_ratio": -0.34541040658950806,
      "logits/chosen": -0.3368746042251587,
      "logits/rejected": -1.1423709392547607,
      "logps/chosen": -1.4025014638900757,
      "logps/rejected": -2.1895670890808105,
      "loss": 1.4411,
      "nll_loss": 1.4065353870391846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14025016129016876,
      "rewards/margins": 0.07870654761791229,
      "rewards/rejected": -0.21895670890808105,
      "step": 629
    },
    {
      "epoch": 0.9929078014184397,
      "grad_norm": 0.1939847767353058,
      "learning_rate": 2.379729178485412e-06,
      "log_odds_chosen": 0.8645562529563904,
      "log_odds_ratio": -0.35570305585861206,
      "logits/chosen": -0.38498225808143616,
      "logits/rejected": -1.2787247896194458,
      "logps/chosen": -1.5039817094802856,
      "logps/rejected": -2.2317099571228027,
      "loss": 1.5475,
      "nll_loss": 1.5119446516036987,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503981649875641,
      "rewards/margins": 0.07277284562587738,
      "rewards/rejected": -0.22317101061344147,
      "step": 630
    },
    {
      "epoch": 0.9944838455476753,
      "grad_norm": 0.22963358461856842,
      "learning_rate": 2.3743306416150636e-06,
      "log_odds_chosen": 0.906265139579773,
      "log_odds_ratio": -0.34214454889297485,
      "logits/chosen": -0.3415853679180145,
      "logits/rejected": -1.2291094064712524,
      "logps/chosen": -1.4594645500183105,
      "logps/rejected": -2.212763786315918,
      "loss": 1.5044,
      "nll_loss": 1.4702297449111938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1459464430809021,
      "rewards/margins": 0.07532992213964462,
      "rewards/rejected": -0.2212763875722885,
      "step": 631
    },
    {
      "epoch": 0.996059889676911,
      "grad_norm": 0.19589129090309143,
      "learning_rate": 2.368929276855512e-06,
      "log_odds_chosen": 0.7489076256752014,
      "log_odds_ratio": -0.38871490955352783,
      "logits/chosen": -0.3408065140247345,
      "logits/rejected": -1.1506099700927734,
      "logps/chosen": -1.565054178237915,
      "logps/rejected": -2.1968889236450195,
      "loss": 1.5864,
      "nll_loss": 1.5475003719329834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15650542080402374,
      "rewards/margins": 0.06318347156047821,
      "rewards/rejected": -0.21968887746334076,
      "step": 632
    },
    {
      "epoch": 0.9976359338061466,
      "grad_norm": 0.19811393320560455,
      "learning_rate": 2.363525125011487e-06,
      "log_odds_chosen": 0.8352795243263245,
      "log_odds_ratio": -0.3715529143810272,
      "logits/chosen": -0.382291316986084,
      "logits/rejected": -1.2505590915679932,
      "logps/chosen": -1.5816127061843872,
      "logps/rejected": -2.2938432693481445,
      "loss": 1.6263,
      "nll_loss": 1.589192271232605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15816128253936768,
      "rewards/margins": 0.07122305780649185,
      "rewards/rejected": -0.22938431799411774,
      "step": 633
    },
    {
      "epoch": 0.9992119779353822,
      "grad_norm": 0.19048024713993073,
      "learning_rate": 2.3581182269087755e-06,
      "log_odds_chosen": 1.2554948329925537,
      "log_odds_ratio": -0.2977595329284668,
      "logits/chosen": -0.3498196601867676,
      "logits/rejected": -1.3388783931732178,
      "logps/chosen": -1.39158034324646,
      "logps/rejected": -2.4365665912628174,
      "loss": 1.446,
      "nll_loss": 1.4162647724151611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13915805518627167,
      "rewards/margins": 0.10449861735105515,
      "rewards/rejected": -0.24365665018558502,
      "step": 634
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.24443046748638153,
      "learning_rate": 2.3527086233939097e-06,
      "log_odds_chosen": 0.987897515296936,
      "log_odds_ratio": -0.32027938961982727,
      "logits/chosen": -0.3751267194747925,
      "logits/rejected": -1.490554690361023,
      "logps/chosen": -1.471299171447754,
      "logps/rejected": -2.3032376766204834,
      "loss": 1.521,
      "nll_loss": 1.488999605178833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14712990820407867,
      "rewards/margins": 0.08319386839866638,
      "rewards/rejected": -0.23032376170158386,
      "step": 635
    },
    {
      "epoch": 1.0015760441292356,
      "grad_norm": 0.21896344423294067,
      "learning_rate": 2.347296355333861e-06,
      "log_odds_chosen": 0.8776235580444336,
      "log_odds_ratio": -0.3515409827232361,
      "logits/chosen": -0.45162782073020935,
      "logits/rejected": -1.1421799659729004,
      "logps/chosen": -1.4817332029342651,
      "logps/rejected": -2.217543363571167,
      "loss": 1.5256,
      "nll_loss": 1.490419864654541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14817334711551666,
      "rewards/margins": 0.07358100265264511,
      "rewards/rejected": -0.22175432741641998,
      "step": 636
    },
    {
      "epoch": 1.0031520882584712,
      "grad_norm": 0.20834492146968842,
      "learning_rate": 2.3418814636157283e-06,
      "log_odds_chosen": 0.7709816694259644,
      "log_odds_ratio": -0.3855873644351959,
      "logits/chosen": -0.3986705541610718,
      "logits/rejected": -1.3128241300582886,
      "logps/chosen": -1.5207031965255737,
      "logps/rejected": -2.162932872772217,
      "loss": 1.5551,
      "nll_loss": 1.5165715217590332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1520703136920929,
      "rewards/margins": 0.06422296166419983,
      "rewards/rejected": -0.21629327535629272,
      "step": 637
    },
    {
      "epoch": 1.0047281323877069,
      "grad_norm": 0.1976415365934372,
      "learning_rate": 2.336463989146434e-06,
      "log_odds_chosen": 0.9102830290794373,
      "log_odds_ratio": -0.3458578586578369,
      "logits/chosen": -0.4078303575515747,
      "logits/rejected": -1.1020984649658203,
      "logps/chosen": -1.4147393703460693,
      "logps/rejected": -2.1706314086914062,
      "loss": 1.4551,
      "nll_loss": 1.4205539226531982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14147396385669708,
      "rewards/margins": 0.07558919489383698,
      "rewards/rejected": -0.21706312894821167,
      "step": 638
    },
    {
      "epoch": 1.0063041765169425,
      "grad_norm": 0.19738516211509705,
      "learning_rate": 2.3310439728524074e-06,
      "log_odds_chosen": 0.945460319519043,
      "log_odds_ratio": -0.3303861618041992,
      "logits/chosen": -0.435433566570282,
      "logits/rejected": -1.2158054113388062,
      "logps/chosen": -1.4381906986236572,
      "logps/rejected": -2.2253530025482178,
      "loss": 1.4904,
      "nll_loss": 1.4573779106140137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14381906390190125,
      "rewards/margins": 0.07871624827384949,
      "rewards/rejected": -0.22253529727458954,
      "step": 639
    },
    {
      "epoch": 1.007880220646178,
      "grad_norm": 0.19811402261257172,
      "learning_rate": 2.325621455679286e-06,
      "log_odds_chosen": 1.0701115131378174,
      "log_odds_ratio": -0.30467408895492554,
      "logits/chosen": -0.3498419225215912,
      "logits/rejected": -1.1933352947235107,
      "logps/chosen": -1.4204521179199219,
      "logps/rejected": -2.317910671234131,
      "loss": 1.4577,
      "nll_loss": 1.427182912826538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14204521477222443,
      "rewards/margins": 0.08974585682153702,
      "rewards/rejected": -0.23179104924201965,
      "step": 640
    },
    {
      "epoch": 1.0094562647754137,
      "grad_norm": 0.18856221437454224,
      "learning_rate": 2.3201964785915953e-06,
      "log_odds_chosen": 0.7920856475830078,
      "log_odds_ratio": -0.37525761127471924,
      "logits/chosen": -0.44481194019317627,
      "logits/rejected": -1.0962393283843994,
      "logps/chosen": -1.4826533794403076,
      "logps/rejected": -2.1414012908935547,
      "loss": 1.5278,
      "nll_loss": 1.4902886152267456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14826534688472748,
      "rewards/margins": 0.06587478518486023,
      "rewards/rejected": -0.2141401171684265,
      "step": 641
    },
    {
      "epoch": 1.0110323089046493,
      "grad_norm": 0.20465238392353058,
      "learning_rate": 2.3147690825724457e-06,
      "log_odds_chosen": 0.7337130904197693,
      "log_odds_ratio": -0.3982602059841156,
      "logits/chosen": -0.37815558910369873,
      "logits/rejected": -1.1759490966796875,
      "logps/chosen": -1.5304032564163208,
      "logps/rejected": -2.1479077339172363,
      "loss": 1.5684,
      "nll_loss": 1.5285764932632446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1530403196811676,
      "rewards/margins": 0.06175045296549797,
      "rewards/rejected": -0.21479080617427826,
      "step": 642
    },
    {
      "epoch": 1.012608353033885,
      "grad_norm": 0.19686812162399292,
      "learning_rate": 2.3093393086232226e-06,
      "log_odds_chosen": 0.9920375347137451,
      "log_odds_ratio": -0.3182482123374939,
      "logits/chosen": -0.3506353795528412,
      "logits/rejected": -1.1600711345672607,
      "logps/chosen": -1.3906843662261963,
      "logps/rejected": -2.2114710807800293,
      "loss": 1.4468,
      "nll_loss": 1.4149943590164185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13906843960285187,
      "rewards/margins": 0.08207866549491882,
      "rewards/rejected": -0.2211471050977707,
      "step": 643
    },
    {
      "epoch": 1.0141843971631206,
      "grad_norm": 0.18937237560749054,
      "learning_rate": 2.3039071977632748e-06,
      "log_odds_chosen": 0.9956415891647339,
      "log_odds_ratio": -0.327029287815094,
      "logits/chosen": -0.43983203172683716,
      "logits/rejected": -1.4347132444381714,
      "logps/chosen": -1.4506264925003052,
      "logps/rejected": -2.2919723987579346,
      "loss": 1.488,
      "nll_loss": 1.4552818536758423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.145062655210495,
      "rewards/margins": 0.08413458615541458,
      "rewards/rejected": -0.22919724881649017,
      "step": 644
    },
    {
      "epoch": 1.0157604412923562,
      "grad_norm": 0.19914327561855316,
      "learning_rate": 2.2984727910296044e-06,
      "log_odds_chosen": 0.8058873414993286,
      "log_odds_ratio": -0.3733724057674408,
      "logits/chosen": -0.43496257066726685,
      "logits/rejected": -1.2451636791229248,
      "logps/chosen": -1.512019157409668,
      "logps/rejected": -2.1896913051605225,
      "loss": 1.5687,
      "nll_loss": 1.5314000844955444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15120193362236023,
      "rewards/margins": 0.06776722520589828,
      "rewards/rejected": -0.21896913647651672,
      "step": 645
    },
    {
      "epoch": 1.0173364854215918,
      "grad_norm": 0.21274791657924652,
      "learning_rate": 2.2930361294765594e-06,
      "log_odds_chosen": 0.9274235367774963,
      "log_odds_ratio": -0.3371463716030121,
      "logits/chosen": -0.3180553913116455,
      "logits/rejected": -1.3749959468841553,
      "logps/chosen": -1.5139557123184204,
      "logps/rejected": -2.298336982727051,
      "loss": 1.567,
      "nll_loss": 1.5332802534103394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15139557421207428,
      "rewards/margins": 0.07843814045190811,
      "rewards/rejected": -0.2298337072134018,
      "step": 646
    },
    {
      "epoch": 1.0189125295508275,
      "grad_norm": 0.19187109172344208,
      "learning_rate": 2.287597254175521e-06,
      "log_odds_chosen": 1.024778962135315,
      "log_odds_ratio": -0.3242001533508301,
      "logits/chosen": -0.3780481517314911,
      "logits/rejected": -1.2348451614379883,
      "logps/chosen": -1.3806902170181274,
      "logps/rejected": -2.23112154006958,
      "loss": 1.4468,
      "nll_loss": 1.414337396621704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1380690038204193,
      "rewards/margins": 0.08504310995340347,
      "rewards/rejected": -0.22311213612556458,
      "step": 647
    },
    {
      "epoch": 1.020488573680063,
      "grad_norm": 0.1909557431936264,
      "learning_rate": 2.2821562062145938e-06,
      "log_odds_chosen": 1.054296612739563,
      "log_odds_ratio": -0.31239694356918335,
      "logits/chosen": -0.37844640016555786,
      "logits/rejected": -1.3747087717056274,
      "logps/chosen": -1.5338422060012817,
      "logps/rejected": -2.4422154426574707,
      "loss": 1.5508,
      "nll_loss": 1.5195119380950928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15338420867919922,
      "rewards/margins": 0.09083734452724457,
      "rewards/rejected": -0.24422156810760498,
      "step": 648
    },
    {
      "epoch": 1.0220646178092987,
      "grad_norm": 0.21584700047969818,
      "learning_rate": 2.2767130266982967e-06,
      "log_odds_chosen": 0.7735684514045715,
      "log_odds_ratio": -0.3889700174331665,
      "logits/chosen": -0.44244903326034546,
      "logits/rejected": -1.033464789390564,
      "logps/chosen": -1.450426697731018,
      "logps/rejected": -2.0916318893432617,
      "loss": 1.5007,
      "nll_loss": 1.461808204650879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14504265785217285,
      "rewards/margins": 0.06412051618099213,
      "rewards/rejected": -0.20916318893432617,
      "step": 649
    },
    {
      "epoch": 1.0236406619385343,
      "grad_norm": 0.19665421545505524,
      "learning_rate": 2.271267756747251e-06,
      "log_odds_chosen": 0.9418151378631592,
      "log_odds_ratio": -0.33850473165512085,
      "logits/chosen": -0.362537145614624,
      "logits/rejected": -1.260590672492981,
      "logps/chosen": -1.4192744493484497,
      "logps/rejected": -2.200381278991699,
      "loss": 1.4545,
      "nll_loss": 1.420658826828003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14192745089530945,
      "rewards/margins": 0.07811066508293152,
      "rewards/rejected": -0.22003813087940216,
      "step": 650
    },
    {
      "epoch": 1.02521670606777,
      "grad_norm": 0.1848953515291214,
      "learning_rate": 2.265820437497871e-06,
      "log_odds_chosen": 0.7917524576187134,
      "log_odds_ratio": -0.3847227692604065,
      "logits/chosen": -0.3512873947620392,
      "logits/rejected": -1.0360175371170044,
      "logps/chosen": -1.4114691019058228,
      "logps/rejected": -2.0633513927459717,
      "loss": 1.4763,
      "nll_loss": 1.4377862215042114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411469280719757,
      "rewards/margins": 0.06518823653459549,
      "rewards/rejected": -0.2063351571559906,
      "step": 651
    },
    {
      "epoch": 1.0267927501970056,
      "grad_norm": 0.6830459833145142,
      "learning_rate": 2.26037111010205e-06,
      "log_odds_chosen": 0.8095988035202026,
      "log_odds_ratio": -0.3710176646709442,
      "logits/chosen": -0.37010836601257324,
      "logits/rejected": -1.1965558528900146,
      "logps/chosen": -1.4072270393371582,
      "logps/rejected": -2.069167375564575,
      "loss": 1.4573,
      "nll_loss": 1.4202206134796143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14072272181510925,
      "rewards/margins": 0.06619402021169662,
      "rewards/rejected": -0.20691676437854767,
      "step": 652
    },
    {
      "epoch": 1.0283687943262412,
      "grad_norm": 0.19480550289154053,
      "learning_rate": 2.254919815726856e-06,
      "log_odds_chosen": 0.809634268283844,
      "log_odds_ratio": -0.374568909406662,
      "logits/chosen": -0.42030924558639526,
      "logits/rejected": -1.1776269674301147,
      "logps/chosen": -1.49861478805542,
      "logps/rejected": -2.177069664001465,
      "loss": 1.5562,
      "nll_loss": 1.5187631845474243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14986149966716766,
      "rewards/margins": 0.06784550100564957,
      "rewards/rejected": -0.21770699322223663,
      "step": 653
    },
    {
      "epoch": 1.0299448384554768,
      "grad_norm": 0.1978185921907425,
      "learning_rate": 2.2494665955542127e-06,
      "log_odds_chosen": 1.1225138902664185,
      "log_odds_ratio": -0.2955932021141052,
      "logits/chosen": -0.507006049156189,
      "logits/rejected": -1.2943130731582642,
      "logps/chosen": -1.4122717380523682,
      "logps/rejected": -2.3514137268066406,
      "loss": 1.4548,
      "nll_loss": 1.4251939058303833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14122718572616577,
      "rewards/margins": 0.09391415864229202,
      "rewards/rejected": -0.23514136672019958,
      "step": 654
    },
    {
      "epoch": 1.0315208825847124,
      "grad_norm": 0.22019991278648376,
      "learning_rate": 2.2440114907805942e-06,
      "log_odds_chosen": 0.8686199188232422,
      "log_odds_ratio": -0.35541900992393494,
      "logits/chosen": -0.4805382490158081,
      "logits/rejected": -1.2325630187988281,
      "logps/chosen": -1.54087233543396,
      "logps/rejected": -2.273367166519165,
      "loss": 1.5652,
      "nll_loss": 1.5296111106872559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15408723056316376,
      "rewards/margins": 0.07324948906898499,
      "rewards/rejected": -0.22733671963214874,
      "step": 655
    },
    {
      "epoch": 1.033096926713948,
      "grad_norm": 0.20485281944274902,
      "learning_rate": 2.2385545426167112e-06,
      "log_odds_chosen": 0.8484992980957031,
      "log_odds_ratio": -0.3635559678077698,
      "logits/chosen": -0.3128311038017273,
      "logits/rejected": -1.4063711166381836,
      "logps/chosen": -1.5250205993652344,
      "logps/rejected": -2.235733985900879,
      "loss": 1.5729,
      "nll_loss": 1.5365853309631348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15250207483768463,
      "rewards/margins": 0.07107134163379669,
      "rewards/rejected": -0.22357340157032013,
      "step": 656
    },
    {
      "epoch": 1.0346729708431837,
      "grad_norm": 0.20302793383598328,
      "learning_rate": 2.2330957922872016e-06,
      "log_odds_chosen": 1.2126038074493408,
      "log_odds_ratio": -0.2739661931991577,
      "logits/chosen": -0.48024260997772217,
      "logits/rejected": -1.344617486000061,
      "logps/chosen": -1.4424711465835571,
      "logps/rejected": -2.4748642444610596,
      "loss": 1.4836,
      "nll_loss": 1.45619797706604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1442471295595169,
      "rewards/margins": 0.10323929786682129,
      "rewards/rejected": -0.247486412525177,
      "step": 657
    },
    {
      "epoch": 1.0362490149724193,
      "grad_norm": 0.18546706438064575,
      "learning_rate": 2.2276352810303166e-06,
      "log_odds_chosen": 1.1111518144607544,
      "log_odds_ratio": -0.2927999794483185,
      "logits/chosen": -0.40777477622032166,
      "logits/rejected": -1.3596274852752686,
      "logps/chosen": -1.3509349822998047,
      "logps/rejected": -2.2723915576934814,
      "loss": 1.402,
      "nll_loss": 1.3727084398269653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13509351015090942,
      "rewards/margins": 0.0921456441283226,
      "rewards/rejected": -0.22723914682865143,
      "step": 658
    },
    {
      "epoch": 1.037825059101655,
      "grad_norm": 0.19996628165245056,
      "learning_rate": 2.222173050097609e-06,
      "log_odds_chosen": 0.9192258715629578,
      "log_odds_ratio": -0.3399088978767395,
      "logits/chosen": -0.41702932119369507,
      "logits/rejected": -1.2896058559417725,
      "logps/chosen": -1.4616878032684326,
      "logps/rejected": -2.230022430419922,
      "loss": 1.5107,
      "nll_loss": 1.4766929149627686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461687833070755,
      "rewards/margins": 0.07683345675468445,
      "rewards/rejected": -0.22300224006175995,
      "step": 659
    },
    {
      "epoch": 1.0394011032308905,
      "grad_norm": 0.20843632519245148,
      "learning_rate": 2.2167091407536272e-06,
      "log_odds_chosen": 0.8826905488967896,
      "log_odds_ratio": -0.3511711657047272,
      "logits/chosen": -0.4367958605289459,
      "logits/rejected": -1.1302224397659302,
      "logps/chosen": -1.4807345867156982,
      "logps/rejected": -2.2163126468658447,
      "loss": 1.5372,
      "nll_loss": 1.5020701885223389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1480734646320343,
      "rewards/margins": 0.07355780899524689,
      "rewards/rejected": -0.22163125872612,
      "step": 660
    },
    {
      "epoch": 1.0409771473601261,
      "grad_norm": 0.2042941302061081,
      "learning_rate": 2.211243594275594e-06,
      "log_odds_chosen": 0.866117537021637,
      "log_odds_ratio": -0.36015111207962036,
      "logits/chosen": -0.44429972767829895,
      "logits/rejected": -1.0608841180801392,
      "logps/chosen": -1.5025889873504639,
      "logps/rejected": -2.2289912700653076,
      "loss": 1.5281,
      "nll_loss": 1.4920531511306763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15025891363620758,
      "rewards/margins": 0.07264022529125214,
      "rewards/rejected": -0.22289912402629852,
      "step": 661
    },
    {
      "epoch": 1.0425531914893618,
      "grad_norm": 0.20318742096424103,
      "learning_rate": 2.2057764519531034e-06,
      "log_odds_chosen": 0.9476841688156128,
      "log_odds_ratio": -0.3337540924549103,
      "logits/chosen": -0.3915112316608429,
      "logits/rejected": -1.1626735925674438,
      "logps/chosen": -1.4998177289962769,
      "logps/rejected": -2.3005120754241943,
      "loss": 1.5254,
      "nll_loss": 1.4919891357421875,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1499817669391632,
      "rewards/margins": 0.08006945252418518,
      "rewards/rejected": -0.2300512194633484,
      "step": 662
    },
    {
      "epoch": 1.0441292356185974,
      "grad_norm": 0.2087588757276535,
      "learning_rate": 2.2003077550878053e-06,
      "log_odds_chosen": 1.0450795888900757,
      "log_odds_ratio": -0.30361974239349365,
      "logits/chosen": -0.40248483419418335,
      "logits/rejected": -1.3018412590026855,
      "logps/chosen": -1.4865845441818237,
      "logps/rejected": -2.3726813793182373,
      "loss": 1.5079,
      "nll_loss": 1.477536916732788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14865845441818237,
      "rewards/margins": 0.08860968798398972,
      "rewards/rejected": -0.2372681349515915,
      "step": 663
    },
    {
      "epoch": 1.045705279747833,
      "grad_norm": 0.2127457708120346,
      "learning_rate": 2.1948375449930915e-06,
      "log_odds_chosen": 0.9164343476295471,
      "log_odds_ratio": -0.3439173698425293,
      "logits/chosen": -0.4028761386871338,
      "logits/rejected": -1.2685139179229736,
      "logps/chosen": -1.5317394733428955,
      "logps/rejected": -2.3123817443847656,
      "loss": 1.5591,
      "nll_loss": 1.5247348546981812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15317393839359283,
      "rewards/margins": 0.07806423306465149,
      "rewards/rejected": -0.23123815655708313,
      "step": 664
    },
    {
      "epoch": 1.0472813238770686,
      "grad_norm": 0.19403357803821564,
      "learning_rate": 2.189365862993787e-06,
      "log_odds_chosen": 0.9709364771842957,
      "log_odds_ratio": -0.3274074196815491,
      "logits/chosen": -0.3594281077384949,
      "logits/rejected": -1.1750539541244507,
      "logps/chosen": -1.5105366706848145,
      "logps/rejected": -2.3339757919311523,
      "loss": 1.5526,
      "nll_loss": 1.5199037790298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15105366706848145,
      "rewards/margins": 0.08234389871358871,
      "rewards/rejected": -0.23339758813381195,
      "step": 665
    },
    {
      "epoch": 1.0488573680063042,
      "grad_norm": 0.21466416120529175,
      "learning_rate": 2.1838927504258354e-06,
      "log_odds_chosen": 0.9618017673492432,
      "log_odds_ratio": -0.3279406428337097,
      "logits/chosen": -0.3783637285232544,
      "logits/rejected": -1.2408227920532227,
      "logps/chosen": -1.4690965414047241,
      "logps/rejected": -2.2774126529693604,
      "loss": 1.5044,
      "nll_loss": 1.4715588092803955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1469096541404724,
      "rewards/margins": 0.0808316096663475,
      "rewards/rejected": -0.2277412712574005,
      "step": 666
    },
    {
      "epoch": 1.0504334121355399,
      "grad_norm": 0.23139192163944244,
      "learning_rate": 2.178418248635988e-06,
      "log_odds_chosen": 0.8501680493354797,
      "log_odds_ratio": -0.36671140789985657,
      "logits/chosen": -0.41223394870758057,
      "logits/rejected": -0.982063353061676,
      "logps/chosen": -1.5426734685897827,
      "logps/rejected": -2.2661685943603516,
      "loss": 1.6098,
      "nll_loss": 1.5731281042099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15426737070083618,
      "rewards/margins": 0.07234951108694077,
      "rewards/rejected": -0.22661687433719635,
      "step": 667
    },
    {
      "epoch": 1.0520094562647755,
      "grad_norm": 0.2014761120080948,
      "learning_rate": 2.1729423989814912e-06,
      "log_odds_chosen": 0.8299447298049927,
      "log_odds_ratio": -0.36926859617233276,
      "logits/chosen": -0.444474995136261,
      "logits/rejected": -1.1502386331558228,
      "logps/chosen": -1.4649335145950317,
      "logps/rejected": -2.1578330993652344,
      "loss": 1.4937,
      "nll_loss": 1.456723928451538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1464933604001999,
      "rewards/margins": 0.06928994506597519,
      "rewards/rejected": -0.21578331291675568,
      "step": 668
    },
    {
      "epoch": 1.053585500394011,
      "grad_norm": 0.19710859656333923,
      "learning_rate": 2.167465242829774e-06,
      "log_odds_chosen": 0.8926303386688232,
      "log_odds_ratio": -0.3491077721118927,
      "logits/chosen": -0.4429822564125061,
      "logits/rejected": -1.1164703369140625,
      "logps/chosen": -1.5111136436462402,
      "logps/rejected": -2.2671446800231934,
      "loss": 1.5494,
      "nll_loss": 1.5144447088241577,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15111136436462402,
      "rewards/margins": 0.07560309767723083,
      "rewards/rejected": -0.22671447694301605,
      "step": 669
    },
    {
      "epoch": 1.0551615445232467,
      "grad_norm": 0.2001245766878128,
      "learning_rate": 2.1619868215581343e-06,
      "log_odds_chosen": 0.9937952160835266,
      "log_odds_ratio": -0.3201329708099365,
      "logits/chosen": -0.44702696800231934,
      "logits/rejected": -1.1192543506622314,
      "logps/chosen": -1.4394910335540771,
      "logps/rejected": -2.267472982406616,
      "loss": 1.4984,
      "nll_loss": 1.4663594961166382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14394909143447876,
      "rewards/margins": 0.08279821276664734,
      "rewards/rejected": -0.2267473191022873,
      "step": 670
    },
    {
      "epoch": 1.0567375886524824,
      "grad_norm": 0.19865983724594116,
      "learning_rate": 2.1565071765534287e-06,
      "log_odds_chosen": 0.8848907947540283,
      "log_odds_ratio": -0.3515958786010742,
      "logits/chosen": -0.37906569242477417,
      "logits/rejected": -1.203958511352539,
      "logps/chosen": -1.5192478895187378,
      "logps/rejected": -2.266707420349121,
      "loss": 1.5535,
      "nll_loss": 1.5183725357055664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15192480385303497,
      "rewards/margins": 0.07474595308303833,
      "rewards/rejected": -0.2266707420349121,
      "step": 671
    },
    {
      "epoch": 1.058313632781718,
      "grad_norm": 0.19839511811733246,
      "learning_rate": 2.1510263492117574e-06,
      "log_odds_chosen": 0.7389354705810547,
      "log_odds_ratio": -0.39493751525878906,
      "logits/chosen": -0.3656381368637085,
      "logits/rejected": -1.0514992475509644,
      "logps/chosen": -1.5041388273239136,
      "logps/rejected": -2.1215643882751465,
      "loss": 1.5606,
      "nll_loss": 1.5210720300674438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1504139006137848,
      "rewards/margins": 0.06174255162477493,
      "rewards/rejected": -0.21215642988681793,
      "step": 672
    },
    {
      "epoch": 1.0598896769109536,
      "grad_norm": 0.18806280195713043,
      "learning_rate": 2.1455443809381535e-06,
      "log_odds_chosen": 1.0643047094345093,
      "log_odds_ratio": -0.30673301219940186,
      "logits/chosen": -0.4542730152606964,
      "logits/rejected": -1.3698962926864624,
      "logps/chosen": -1.4343891143798828,
      "logps/rejected": -2.328421115875244,
      "loss": 1.4747,
      "nll_loss": 1.4440538883209229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1434389054775238,
      "rewards/margins": 0.08940321207046509,
      "rewards/rejected": -0.23284214735031128,
      "step": 673
    },
    {
      "epoch": 1.0614657210401892,
      "grad_norm": 0.20284530520439148,
      "learning_rate": 2.1400613131462697e-06,
      "log_odds_chosen": 0.9821515679359436,
      "log_odds_ratio": -0.3239043056964874,
      "logits/chosen": -0.3779458999633789,
      "logits/rejected": -1.2626372575759888,
      "logps/chosen": -1.581770420074463,
      "logps/rejected": -2.4273085594177246,
      "loss": 1.606,
      "nll_loss": 1.5736521482467651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15817704796791077,
      "rewards/margins": 0.08455383032560349,
      "rewards/rejected": -0.24273085594177246,
      "step": 674
    },
    {
      "epoch": 1.0630417651694248,
      "grad_norm": 0.19790387153625488,
      "learning_rate": 2.1345771872580628e-06,
      "log_odds_chosen": 0.9997245669364929,
      "log_odds_ratio": -0.33621615171432495,
      "logits/chosen": -0.4172746539115906,
      "logits/rejected": -1.2197551727294922,
      "logps/chosen": -1.4898098707199097,
      "logps/rejected": -2.345339059829712,
      "loss": 1.5191,
      "nll_loss": 1.4855258464813232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1489810049533844,
      "rewards/margins": 0.08555291593074799,
      "rewards/rejected": -0.2345339059829712,
      "step": 675
    },
    {
      "epoch": 1.0646178092986605,
      "grad_norm": 0.1856481432914734,
      "learning_rate": 2.1290920447034846e-06,
      "log_odds_chosen": 0.9516907334327698,
      "log_odds_ratio": -0.33092695474624634,
      "logits/chosen": -0.3683914542198181,
      "logits/rejected": -1.1827926635742188,
      "logps/chosen": -1.4661242961883545,
      "logps/rejected": -2.261784315109253,
      "loss": 1.5071,
      "nll_loss": 1.4740149974822998,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14661243557929993,
      "rewards/margins": 0.07956600189208984,
      "rewards/rejected": -0.22617843747138977,
      "step": 676
    },
    {
      "epoch": 1.066193853427896,
      "grad_norm": 0.20007124543190002,
      "learning_rate": 2.1236059269201683e-06,
      "log_odds_chosen": 0.8022679686546326,
      "log_odds_ratio": -0.37775808572769165,
      "logits/chosen": -0.4633864164352417,
      "logits/rejected": -1.1169921159744263,
      "logps/chosen": -1.5044833421707153,
      "logps/rejected": -2.1787495613098145,
      "loss": 1.531,
      "nll_loss": 1.4931955337524414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15044833719730377,
      "rewards/margins": 0.06742661446332932,
      "rewards/rejected": -0.21787495911121368,
      "step": 677
    },
    {
      "epoch": 1.0677698975571317,
      "grad_norm": 0.24462629854679108,
      "learning_rate": 2.1181188753531124e-06,
      "log_odds_chosen": 0.9251030683517456,
      "log_odds_ratio": -0.3388793468475342,
      "logits/chosen": -0.49078822135925293,
      "logits/rejected": -1.1590297222137451,
      "logps/chosen": -1.6351323127746582,
      "logps/rejected": -2.4362010955810547,
      "loss": 1.6569,
      "nll_loss": 1.6230173110961914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16351322829723358,
      "rewards/margins": 0.08010690659284592,
      "rewards/rejected": -0.2436201423406601,
      "step": 678
    },
    {
      "epoch": 1.0693459416863673,
      "grad_norm": 0.20519758760929108,
      "learning_rate": 2.112630931454371e-06,
      "log_odds_chosen": 1.0099736452102661,
      "log_odds_ratio": -0.3172469437122345,
      "logits/chosen": -0.40586379170417786,
      "logits/rejected": -1.129070520401001,
      "logps/chosen": -1.4049760103225708,
      "logps/rejected": -2.241764545440674,
      "loss": 1.4537,
      "nll_loss": 1.4220170974731445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1404976099729538,
      "rewards/margins": 0.08367887139320374,
      "rewards/rejected": -0.22417648136615753,
      "step": 679
    },
    {
      "epoch": 1.070921985815603,
      "grad_norm": 0.18968316912651062,
      "learning_rate": 2.10714213668274e-06,
      "log_odds_chosen": 0.8810672760009766,
      "log_odds_ratio": -0.3498786687850952,
      "logits/chosen": -0.41325268149375916,
      "logits/rejected": -1.050408959388733,
      "logps/chosen": -1.4516003131866455,
      "logps/rejected": -2.1854681968688965,
      "loss": 1.5047,
      "nll_loss": 1.469740390777588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1451600342988968,
      "rewards/margins": 0.0733867883682251,
      "rewards/rejected": -0.2185468226671219,
      "step": 680
    },
    {
      "epoch": 1.0724980299448386,
      "grad_norm": 0.20564420521259308,
      "learning_rate": 2.1016525325034403e-06,
      "log_odds_chosen": 1.1329858303070068,
      "log_odds_ratio": -0.3025621473789215,
      "logits/chosen": -0.4055512547492981,
      "logits/rejected": -1.177002191543579,
      "logps/chosen": -1.460391640663147,
      "logps/rejected": -2.4255709648132324,
      "loss": 1.4929,
      "nll_loss": 1.4626846313476562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460391730070114,
      "rewards/margins": 0.09651794284582138,
      "rewards/rejected": -0.2425571233034134,
      "step": 681
    },
    {
      "epoch": 1.074074074074074,
      "grad_norm": 0.1874896138906479,
      "learning_rate": 2.096162160387811e-06,
      "log_odds_chosen": 0.7920770645141602,
      "log_odds_ratio": -0.3788534998893738,
      "logits/chosen": -0.35121428966522217,
      "logits/rejected": -1.1225156784057617,
      "logps/chosen": -1.3969855308532715,
      "logps/rejected": -2.036137819290161,
      "loss": 1.4604,
      "nll_loss": 1.4225172996520996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1396985650062561,
      "rewards/margins": 0.06391523033380508,
      "rewards/rejected": -0.2036137878894806,
      "step": 682
    },
    {
      "epoch": 1.0756501182033098,
      "grad_norm": 0.182921901345253,
      "learning_rate": 2.09067106181299e-06,
      "log_odds_chosen": 1.046210527420044,
      "log_odds_ratio": -0.3095901608467102,
      "logits/chosen": -0.41022592782974243,
      "logits/rejected": -1.169036865234375,
      "logps/chosen": -1.3433701992034912,
      "logps/rejected": -2.2022109031677246,
      "loss": 1.3921,
      "nll_loss": 1.361114263534546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13433702290058136,
      "rewards/margins": 0.08588407933712006,
      "rewards/rejected": -0.22022110223770142,
      "step": 683
    },
    {
      "epoch": 1.0772261623325452,
      "grad_norm": 0.1878536343574524,
      "learning_rate": 2.0851792782616052e-06,
      "log_odds_chosen": 1.1520943641662598,
      "log_odds_ratio": -0.28656676411628723,
      "logits/chosen": -0.32152676582336426,
      "logits/rejected": -1.2533209323883057,
      "logps/chosen": -1.4375636577606201,
      "logps/rejected": -2.4150896072387695,
      "loss": 1.4809,
      "nll_loss": 1.4521974325180054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14375635981559753,
      "rewards/margins": 0.09775260835886002,
      "rewards/rejected": -0.24150897562503815,
      "step": 684
    },
    {
      "epoch": 1.078802206461781,
      "grad_norm": 0.19676533341407776,
      "learning_rate": 2.0796868512214576e-06,
      "log_odds_chosen": 1.0356510877609253,
      "log_odds_ratio": -0.3207407295703888,
      "logits/chosen": -0.5013701915740967,
      "logits/rejected": -1.1672927141189575,
      "logps/chosen": -1.4319782257080078,
      "logps/rejected": -2.3007633686065674,
      "loss": 1.4886,
      "nll_loss": 1.4565701484680176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14319781959056854,
      "rewards/margins": 0.08687852323055267,
      "rewards/rejected": -0.23007634282112122,
      "step": 685
    },
    {
      "epoch": 1.0803782505910164,
      "grad_norm": 0.18871276080608368,
      "learning_rate": 2.0741938221852103e-06,
      "log_odds_chosen": 0.9563425183296204,
      "log_odds_ratio": -0.3327597677707672,
      "logits/chosen": -0.38215094804763794,
      "logits/rejected": -1.17371666431427,
      "logps/chosen": -1.4323339462280273,
      "logps/rejected": -2.2335944175720215,
      "loss": 1.4835,
      "nll_loss": 1.4501866102218628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14323340356349945,
      "rewards/margins": 0.08012604713439941,
      "rewards/rejected": -0.22335945069789886,
      "step": 686
    },
    {
      "epoch": 1.081954294720252,
      "grad_norm": 0.2114379107952118,
      "learning_rate": 2.0687002326500743e-06,
      "log_odds_chosen": 1.0518598556518555,
      "log_odds_ratio": -0.30392950773239136,
      "logits/chosen": -0.4802509844303131,
      "logits/rejected": -1.2724312543869019,
      "logps/chosen": -1.4321997165679932,
      "logps/rejected": -2.3145012855529785,
      "loss": 1.4702,
      "nll_loss": 1.439772129058838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1432199776172638,
      "rewards/margins": 0.08823014050722122,
      "rewards/rejected": -0.2314501255750656,
      "step": 687
    },
    {
      "epoch": 1.0835303388494877,
      "grad_norm": 0.2489686757326126,
      "learning_rate": 2.0632061241174942e-06,
      "log_odds_chosen": 1.0363986492156982,
      "log_odds_ratio": -0.31842878460884094,
      "logits/chosen": -0.45514965057373047,
      "logits/rejected": -1.1771830320358276,
      "logps/chosen": -1.4814982414245605,
      "logps/rejected": -2.3607730865478516,
      "loss": 1.5135,
      "nll_loss": 1.4816317558288574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14814981818199158,
      "rewards/margins": 0.08792747557163239,
      "rewards/rejected": -0.23607729375362396,
      "step": 688
    },
    {
      "epoch": 1.0851063829787233,
      "grad_norm": 0.2037288397550583,
      "learning_rate": 2.0577115380928364e-06,
      "log_odds_chosen": 0.9761087894439697,
      "log_odds_ratio": -0.3295080065727234,
      "logits/chosen": -0.42405396699905396,
      "logits/rejected": -1.1710150241851807,
      "logps/chosen": -1.4554269313812256,
      "logps/rejected": -2.279367446899414,
      "loss": 1.4852,
      "nll_loss": 1.452261209487915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.145542711019516,
      "rewards/margins": 0.0823940634727478,
      "rewards/rejected": -0.2279367595911026,
      "step": 689
    },
    {
      "epoch": 1.086682427107959,
      "grad_norm": 0.1994817852973938,
      "learning_rate": 2.052216516085073e-06,
      "log_odds_chosen": 0.8603441715240479,
      "log_odds_ratio": -0.36642566323280334,
      "logits/chosen": -0.4325138032436371,
      "logits/rejected": -1.1853559017181396,
      "logps/chosen": -1.4103660583496094,
      "logps/rejected": -2.1263349056243896,
      "loss": 1.4657,
      "nll_loss": 1.4290837049484253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14103659987449646,
      "rewards/margins": 0.0715968906879425,
      "rewards/rejected": -0.21263349056243896,
      "step": 690
    },
    {
      "epoch": 1.0882584712371945,
      "grad_norm": 0.2532378137111664,
      "learning_rate": 2.0467210996064707e-06,
      "log_odds_chosen": 0.8362988829612732,
      "log_odds_ratio": -0.3687710165977478,
      "logits/chosen": -0.5061081647872925,
      "logits/rejected": -0.8434776067733765,
      "logps/chosen": -1.4668684005737305,
      "logps/rejected": -2.166229724884033,
      "loss": 1.5011,
      "nll_loss": 1.464253306388855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.146686851978302,
      "rewards/margins": 0.0699361190199852,
      "rewards/rejected": -0.2166229635477066,
      "step": 691
    },
    {
      "epoch": 1.0898345153664302,
      "grad_norm": 0.18716050684452057,
      "learning_rate": 2.0412253301722774e-06,
      "log_odds_chosen": 1.0597642660140991,
      "log_odds_ratio": -0.30042803287506104,
      "logits/chosen": -0.3812195062637329,
      "logits/rejected": -1.2872941493988037,
      "logps/chosen": -1.5144933462142944,
      "logps/rejected": -2.4189929962158203,
      "loss": 1.5514,
      "nll_loss": 1.5213247537612915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15144933760166168,
      "rewards/margins": 0.09044996649026871,
      "rewards/rejected": -0.2418992966413498,
      "step": 692
    },
    {
      "epoch": 1.0914105594956658,
      "grad_norm": 0.19066938757896423,
      "learning_rate": 2.0357292493004044e-06,
      "log_odds_chosen": 0.9988577961921692,
      "log_odds_ratio": -0.3188289701938629,
      "logits/chosen": -0.3816460967063904,
      "logits/rejected": -1.1948785781860352,
      "logps/chosen": -1.4389021396636963,
      "logps/rejected": -2.276630163192749,
      "loss": 1.4649,
      "nll_loss": 1.4330021142959595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14389020204544067,
      "rewards/margins": 0.08377279341220856,
      "rewards/rejected": -0.22766301035881042,
      "step": 693
    },
    {
      "epoch": 1.0929866036249014,
      "grad_norm": 0.18970440328121185,
      "learning_rate": 2.0302328985111193e-06,
      "log_odds_chosen": 1.078777551651001,
      "log_odds_ratio": -0.31302446126937866,
      "logits/chosen": -0.3268972337245941,
      "logits/rejected": -1.1950205564498901,
      "logps/chosen": -1.3950469493865967,
      "logps/rejected": -2.303654193878174,
      "loss": 1.4488,
      "nll_loss": 1.4174940586090088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13950470089912415,
      "rewards/margins": 0.09086073189973831,
      "rewards/rejected": -0.23036542534828186,
      "step": 694
    },
    {
      "epoch": 1.094562647754137,
      "grad_norm": 0.2068614363670349,
      "learning_rate": 2.0247363193267256e-06,
      "log_odds_chosen": 0.9605895280838013,
      "log_odds_ratio": -0.32965677976608276,
      "logits/chosen": -0.3405793607234955,
      "logits/rejected": -1.1863356828689575,
      "logps/chosen": -1.5005230903625488,
      "logps/rejected": -2.314649820327759,
      "loss": 1.5311,
      "nll_loss": 1.498124122619629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1500522941350937,
      "rewards/margins": 0.08141268789768219,
      "rewards/rejected": -0.23146499693393707,
      "step": 695
    },
    {
      "epoch": 1.0961386918833727,
      "grad_norm": 0.20434536039829254,
      "learning_rate": 2.019239553271255e-06,
      "log_odds_chosen": 1.090073823928833,
      "log_odds_ratio": -0.29843807220458984,
      "logits/chosen": -0.4116791784763336,
      "logits/rejected": -1.2559654712677002,
      "logps/chosen": -1.443864107131958,
      "logps/rejected": -2.3633673191070557,
      "loss": 1.485,
      "nll_loss": 1.455194354057312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1443863958120346,
      "rewards/margins": 0.09195032715797424,
      "rewards/rejected": -0.23633673787117004,
      "step": 696
    },
    {
      "epoch": 1.0977147360126083,
      "grad_norm": 0.19242364168167114,
      "learning_rate": 2.0137426418701488e-06,
      "log_odds_chosen": 1.0919839143753052,
      "log_odds_ratio": -0.30183202028274536,
      "logits/chosen": -0.3657260835170746,
      "logits/rejected": -1.1943174600601196,
      "logps/chosen": -1.4493064880371094,
      "logps/rejected": -2.379274368286133,
      "loss": 1.4899,
      "nll_loss": 1.4597445726394653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1449306458234787,
      "rewards/margins": 0.09299679845571518,
      "rewards/rejected": -0.23792743682861328,
      "step": 697
    },
    {
      "epoch": 1.099290780141844,
      "grad_norm": 0.18624840676784515,
      "learning_rate": 2.008245626649947e-06,
      "log_odds_chosen": 1.0154231786727905,
      "log_odds_ratio": -0.3290625810623169,
      "logits/chosen": -0.41685575246810913,
      "logits/rejected": -1.3414193391799927,
      "logps/chosen": -1.3938621282577515,
      "logps/rejected": -2.244748592376709,
      "loss": 1.4503,
      "nll_loss": 1.417410135269165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13938622176647186,
      "rewards/margins": 0.08508863300085068,
      "rewards/rejected": -0.22447484731674194,
      "step": 698
    },
    {
      "epoch": 1.1008668242710795,
      "grad_norm": 0.2152419537305832,
      "learning_rate": 2.0027485491379746e-06,
      "log_odds_chosen": 1.1011391878128052,
      "log_odds_ratio": -0.29639315605163574,
      "logits/chosen": -0.5617659687995911,
      "logits/rejected": -1.04752516746521,
      "logps/chosen": -1.5701590776443481,
      "logps/rejected": -2.5163984298706055,
      "loss": 1.5965,
      "nll_loss": 1.5668200254440308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15701590478420258,
      "rewards/margins": 0.09462396055459976,
      "rewards/rejected": -0.25163984298706055,
      "step": 699
    },
    {
      "epoch": 1.1024428684003151,
      "grad_norm": 0.20998311042785645,
      "learning_rate": 1.9972514508620256e-06,
      "log_odds_chosen": 0.9747617840766907,
      "log_odds_ratio": -0.3306126594543457,
      "logits/chosen": -0.4026050567626953,
      "logits/rejected": -1.1908594369888306,
      "logps/chosen": -1.425804853439331,
      "logps/rejected": -2.239183187484741,
      "loss": 1.4709,
      "nll_loss": 1.4377951622009277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14258049428462982,
      "rewards/margins": 0.0813378095626831,
      "rewards/rejected": -0.22391831874847412,
      "step": 700
    },
    {
      "epoch": 1.1040189125295508,
      "grad_norm": 0.1995379775762558,
      "learning_rate": 1.991754373350053e-06,
      "log_odds_chosen": 0.7445781826972961,
      "log_odds_ratio": -0.39829933643341064,
      "logits/chosen": -0.4401898980140686,
      "logits/rejected": -1.232159972190857,
      "logps/chosen": -1.4865683317184448,
      "logps/rejected": -2.1072041988372803,
      "loss": 1.5371,
      "nll_loss": 1.4973129034042358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14865684509277344,
      "rewards/margins": 0.062063585966825485,
      "rewards/rejected": -0.21072041988372803,
      "step": 701
    },
    {
      "epoch": 1.1055949566587864,
      "grad_norm": 0.19035221636295319,
      "learning_rate": 1.986257358129852e-06,
      "log_odds_chosen": 0.9606376886367798,
      "log_odds_ratio": -0.3305578827857971,
      "logits/chosen": -0.4272080957889557,
      "logits/rejected": -1.1499531269073486,
      "logps/chosen": -1.387117624282837,
      "logps/rejected": -2.174914598464966,
      "loss": 1.4338,
      "nll_loss": 1.4007118940353394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13871176540851593,
      "rewards/margins": 0.0787796825170517,
      "rewards/rejected": -0.21749144792556763,
      "step": 702
    },
    {
      "epoch": 1.107171000788022,
      "grad_norm": 0.27617543935775757,
      "learning_rate": 1.9807604467287453e-06,
      "log_odds_chosen": 0.9321651458740234,
      "log_odds_ratio": -0.33925166726112366,
      "logits/chosen": -0.507322371006012,
      "logits/rejected": -1.0021750926971436,
      "logps/chosen": -1.5034773349761963,
      "logps/rejected": -2.288116693496704,
      "loss": 1.5329,
      "nll_loss": 1.498948335647583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1503477245569229,
      "rewards/margins": 0.07846394181251526,
      "rewards/rejected": -0.22881166636943817,
      "step": 703
    },
    {
      "epoch": 1.1087470449172576,
      "grad_norm": 0.20091944932937622,
      "learning_rate": 1.9752636806732742e-06,
      "log_odds_chosen": 0.9638543128967285,
      "log_odds_ratio": -0.32648393511772156,
      "logits/chosen": -0.36100420355796814,
      "logits/rejected": -1.1802233457565308,
      "logps/chosen": -1.428181767463684,
      "logps/rejected": -2.2321414947509766,
      "loss": 1.4602,
      "nll_loss": 1.4275248050689697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14281818270683289,
      "rewards/margins": 0.08039598166942596,
      "rewards/rejected": -0.22321414947509766,
      "step": 704
    },
    {
      "epoch": 1.1103230890464932,
      "grad_norm": 0.21452394127845764,
      "learning_rate": 1.9697671014888805e-06,
      "log_odds_chosen": 1.0019042491912842,
      "log_odds_ratio": -0.3201969563961029,
      "logits/chosen": -0.541839063167572,
      "logits/rejected": -1.0848870277404785,
      "logps/chosen": -1.5007753372192383,
      "logps/rejected": -2.349736213684082,
      "loss": 1.5348,
      "nll_loss": 1.5028202533721924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15007753670215607,
      "rewards/margins": 0.08489609509706497,
      "rewards/rejected": -0.23497362434864044,
      "step": 705
    },
    {
      "epoch": 1.1118991331757289,
      "grad_norm": 0.19678017497062683,
      "learning_rate": 1.9642707506995954e-06,
      "log_odds_chosen": 1.1572978496551514,
      "log_odds_ratio": -0.2805708646774292,
      "logits/chosen": -0.4564618468284607,
      "logits/rejected": -1.2300448417663574,
      "logps/chosen": -1.439591407775879,
      "logps/rejected": -2.421959400177002,
      "loss": 1.4757,
      "nll_loss": 1.4476170539855957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1439591348171234,
      "rewards/margins": 0.09823679178953171,
      "rewards/rejected": -0.24219590425491333,
      "step": 706
    },
    {
      "epoch": 1.1134751773049645,
      "grad_norm": 0.1865488886833191,
      "learning_rate": 1.9587746698277232e-06,
      "log_odds_chosen": 0.8861472010612488,
      "log_odds_ratio": -0.35670971870422363,
      "logits/chosen": -0.45484602451324463,
      "logits/rejected": -1.0410692691802979,
      "logps/chosen": -1.4463295936584473,
      "logps/rejected": -2.1877095699310303,
      "loss": 1.485,
      "nll_loss": 1.4493314027786255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1446329802274704,
      "rewards/margins": 0.07413799315690994,
      "rewards/rejected": -0.21877093613147736,
      "step": 707
    },
    {
      "epoch": 1.1150512214342,
      "grad_norm": 0.22569124400615692,
      "learning_rate": 1.953278900393529e-06,
      "log_odds_chosen": 0.9650210738182068,
      "log_odds_ratio": -0.3372189998626709,
      "logits/chosen": -0.5083266496658325,
      "logits/rejected": -1.0619049072265625,
      "logps/chosen": -1.4939903020858765,
      "logps/rejected": -2.3115758895874023,
      "loss": 1.508,
      "nll_loss": 1.4742854833602905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1493990421295166,
      "rewards/margins": 0.08175858110189438,
      "rewards/rejected": -0.23115761578083038,
      "step": 708
    },
    {
      "epoch": 1.1166272655634357,
      "grad_norm": 0.20435523986816406,
      "learning_rate": 1.9477834839149274e-06,
      "log_odds_chosen": 0.9990788102149963,
      "log_odds_ratio": -0.3213644325733185,
      "logits/chosen": -0.48895981907844543,
      "logits/rejected": -1.1478885412216187,
      "logps/chosen": -1.3947725296020508,
      "logps/rejected": -2.225590705871582,
      "loss": 1.4564,
      "nll_loss": 1.4242619276046753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13947725296020508,
      "rewards/margins": 0.08308181166648865,
      "rewards/rejected": -0.22255906462669373,
      "step": 709
    },
    {
      "epoch": 1.1182033096926713,
      "grad_norm": 0.2267635315656662,
      "learning_rate": 1.942288461907164e-06,
      "log_odds_chosen": 1.034114122390747,
      "log_odds_ratio": -0.319720596075058,
      "logits/chosen": -0.48249855637550354,
      "logits/rejected": -1.1380438804626465,
      "logps/chosen": -1.3985915184020996,
      "logps/rejected": -2.2676467895507812,
      "loss": 1.4544,
      "nll_loss": 1.4224073886871338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1398591548204422,
      "rewards/margins": 0.08690553903579712,
      "rewards/rejected": -0.22676467895507812,
      "step": 710
    },
    {
      "epoch": 1.119779353821907,
      "grad_norm": 0.2050100713968277,
      "learning_rate": 1.936793875882505e-06,
      "log_odds_chosen": 1.0137135982513428,
      "log_odds_ratio": -0.3136872947216034,
      "logits/chosen": -0.2920651137828827,
      "logits/rejected": -1.1815156936645508,
      "logps/chosen": -1.5506312847137451,
      "logps/rejected": -2.4197158813476562,
      "loss": 1.6,
      "nll_loss": 1.56861412525177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15506313741207123,
      "rewards/margins": 0.08690845966339111,
      "rewards/rejected": -0.24197161197662354,
      "step": 711
    },
    {
      "epoch": 1.1213553979511426,
      "grad_norm": 0.20854681730270386,
      "learning_rate": 1.931299767349926e-06,
      "log_odds_chosen": 1.0215733051300049,
      "log_odds_ratio": -0.31999504566192627,
      "logits/chosen": -0.48796266317367554,
      "logits/rejected": -0.985529899597168,
      "logps/chosen": -1.3821855783462524,
      "logps/rejected": -2.221768856048584,
      "loss": 1.4298,
      "nll_loss": 1.3978039026260376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13821856677532196,
      "rewards/margins": 0.08395831286907196,
      "rewards/rejected": -0.22217689454555511,
      "step": 712
    },
    {
      "epoch": 1.1229314420803782,
      "grad_norm": 0.2199748158454895,
      "learning_rate": 1.9258061778147895e-06,
      "log_odds_chosen": 0.9356629848480225,
      "log_odds_ratio": -0.33955317735671997,
      "logits/chosen": -0.49804332852363586,
      "logits/rejected": -1.2766650915145874,
      "logps/chosen": -1.4927222728729248,
      "logps/rejected": -2.2841503620147705,
      "loss": 1.5338,
      "nll_loss": 1.4998347759246826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14927223324775696,
      "rewards/margins": 0.07914278656244278,
      "rewards/rejected": -0.22841502726078033,
      "step": 713
    },
    {
      "epoch": 1.1245074862096138,
      "grad_norm": 0.20225383341312408,
      "learning_rate": 1.9203131487785426e-06,
      "log_odds_chosen": 1.1917847394943237,
      "log_odds_ratio": -0.26949161291122437,
      "logits/chosen": -0.6047004461288452,
      "logits/rejected": -1.064903736114502,
      "logps/chosen": -1.3710243701934814,
      "logps/rejected": -2.367295742034912,
      "loss": 1.422,
      "nll_loss": 1.3950304985046387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13710243999958038,
      "rewards/margins": 0.09962713718414307,
      "rewards/rejected": -0.23672957718372345,
      "step": 714
    },
    {
      "epoch": 1.1260835303388494,
      "grad_norm": 0.20919670164585114,
      "learning_rate": 1.9148207217383946e-06,
      "log_odds_chosen": 1.2860711812973022,
      "log_odds_ratio": -0.2481311410665512,
      "logits/chosen": -0.4356050491333008,
      "logits/rejected": -1.265368938446045,
      "logps/chosen": -1.48536217212677,
      "logps/rejected": -2.5933916568756104,
      "loss": 1.4993,
      "nll_loss": 1.4744744300842285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14853623509407043,
      "rewards/margins": 0.11080293357372284,
      "rewards/rejected": -0.2593391537666321,
      "step": 715
    },
    {
      "epoch": 1.127659574468085,
      "grad_norm": 0.21525493264198303,
      "learning_rate": 1.9093289381870094e-06,
      "log_odds_chosen": 0.8300395607948303,
      "log_odds_ratio": -0.36800631880760193,
      "logits/chosen": -0.45656830072402954,
      "logits/rejected": -1.1527045965194702,
      "logps/chosen": -1.477571964263916,
      "logps/rejected": -2.1725220680236816,
      "loss": 1.5218,
      "nll_loss": 1.4849562644958496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14775718748569489,
      "rewards/margins": 0.06949502229690552,
      "rewards/rejected": -0.2172522246837616,
      "step": 716
    },
    {
      "epoch": 1.1292356185973207,
      "grad_norm": 0.20370449125766754,
      "learning_rate": 1.9038378396121895e-06,
      "log_odds_chosen": 1.0496752262115479,
      "log_odds_ratio": -0.3066055476665497,
      "logits/chosen": -0.27968233823776245,
      "logits/rejected": -1.3493554592132568,
      "logps/chosen": -1.5176069736480713,
      "logps/rejected": -2.417680263519287,
      "loss": 1.5503,
      "nll_loss": 1.5196748971939087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15176072716712952,
      "rewards/margins": 0.09000729769468307,
      "rewards/rejected": -0.241768017411232,
      "step": 717
    },
    {
      "epoch": 1.1308116627265563,
      "grad_norm": 0.19282495975494385,
      "learning_rate": 1.8983474674965597e-06,
      "log_odds_chosen": 0.9821950197219849,
      "log_odds_ratio": -0.3319460451602936,
      "logits/chosen": -0.44672128558158875,
      "logits/rejected": -1.2834670543670654,
      "logps/chosen": -1.452689528465271,
      "logps/rejected": -2.276275396347046,
      "loss": 1.493,
      "nll_loss": 1.4598461389541626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14526894688606262,
      "rewards/margins": 0.08235857635736465,
      "rewards/rejected": -0.22762753069400787,
      "step": 718
    },
    {
      "epoch": 1.132387706855792,
      "grad_norm": 0.20439912378787994,
      "learning_rate": 1.8928578633172603e-06,
      "log_odds_chosen": 1.1495616436004639,
      "log_odds_ratio": -0.27841079235076904,
      "logits/chosen": -0.43235668540000916,
      "logits/rejected": -1.3092814683914185,
      "logps/chosen": -1.4898877143859863,
      "logps/rejected": -2.4715890884399414,
      "loss": 1.5304,
      "nll_loss": 1.5025358200073242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1489887833595276,
      "rewards/margins": 0.09817016124725342,
      "rewards/rejected": -0.247158944606781,
      "step": 719
    },
    {
      "epoch": 1.1339637509850276,
      "grad_norm": 0.19554686546325684,
      "learning_rate": 1.8873690685456283e-06,
      "log_odds_chosen": 1.0817512273788452,
      "log_odds_ratio": -0.3004467785358429,
      "logits/chosen": -0.4738815128803253,
      "logits/rejected": -1.1428114175796509,
      "logps/chosen": -1.4302932024002075,
      "logps/rejected": -2.3406765460968018,
      "loss": 1.4668,
      "nll_loss": 1.4367115497589111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430293321609497,
      "rewards/margins": 0.09103834629058838,
      "rewards/rejected": -0.23406767845153809,
      "step": 720
    },
    {
      "epoch": 1.1355397951142632,
      "grad_norm": 0.19459304213523865,
      "learning_rate": 1.8818811246468872e-06,
      "log_odds_chosen": 1.1829216480255127,
      "log_odds_ratio": -0.2841511070728302,
      "logits/chosen": -0.43917208909988403,
      "logits/rejected": -1.2284621000289917,
      "logps/chosen": -1.4190592765808105,
      "logps/rejected": -2.420963764190674,
      "loss": 1.4574,
      "nll_loss": 1.4289494752883911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14190593361854553,
      "rewards/margins": 0.10019044578075409,
      "rewards/rejected": -0.24209639430046082,
      "step": 721
    },
    {
      "epoch": 1.1371158392434988,
      "grad_norm": 0.21598437428474426,
      "learning_rate": 1.876394073079832e-06,
      "log_odds_chosen": 1.150900959968567,
      "log_odds_ratio": -0.2852911353111267,
      "logits/chosen": -0.5301509499549866,
      "logits/rejected": -1.1085585355758667,
      "logps/chosen": -1.5470672845840454,
      "logps/rejected": -2.5410451889038086,
      "loss": 1.5725,
      "nll_loss": 1.5439379215240479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15470673143863678,
      "rewards/margins": 0.09939778596162796,
      "rewards/rejected": -0.25410449504852295,
      "step": 722
    },
    {
      "epoch": 1.1386918833727344,
      "grad_norm": 0.19716085493564606,
      "learning_rate": 1.8709079552965152e-06,
      "log_odds_chosen": 1.0834004878997803,
      "log_odds_ratio": -0.30721333622932434,
      "logits/chosen": -0.4459407925605774,
      "logits/rejected": -1.1097474098205566,
      "logps/chosen": -1.4277780055999756,
      "logps/rejected": -2.3393218517303467,
      "loss": 1.4663,
      "nll_loss": 1.435594081878662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14277780055999756,
      "rewards/margins": 0.09115439653396606,
      "rewards/rejected": -0.23393219709396362,
      "step": 723
    },
    {
      "epoch": 1.14026792750197,
      "grad_norm": 0.20371459424495697,
      "learning_rate": 1.8654228127419375e-06,
      "log_odds_chosen": 0.9284728169441223,
      "log_odds_ratio": -0.3380288779735565,
      "logits/chosen": -0.5102434754371643,
      "logits/rejected": -1.0988212823867798,
      "logps/chosen": -1.4394677877426147,
      "logps/rejected": -2.215912342071533,
      "loss": 1.4786,
      "nll_loss": 1.444777011871338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1439467817544937,
      "rewards/margins": 0.0776444748044014,
      "rewards/rejected": -0.2215912640094757,
      "step": 724
    },
    {
      "epoch": 1.1418439716312057,
      "grad_norm": 0.19320811331272125,
      "learning_rate": 1.8599386868537306e-06,
      "log_odds_chosen": 1.1900373697280884,
      "log_odds_ratio": -0.2687169313430786,
      "logits/chosen": -0.4567139744758606,
      "logits/rejected": -1.222866415977478,
      "logps/chosen": -1.3228245973587036,
      "logps/rejected": -2.3053786754608154,
      "loss": 1.3745,
      "nll_loss": 1.3476735353469849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13228245079517365,
      "rewards/margins": 0.09825541824102402,
      "rewards/rejected": -0.23053789138793945,
      "step": 725
    },
    {
      "epoch": 1.1434200157604413,
      "grad_norm": 0.2144141048192978,
      "learning_rate": 1.8544556190618464e-06,
      "log_odds_chosen": 1.0680809020996094,
      "log_odds_ratio": -0.298118531703949,
      "logits/chosen": -0.5212095975875854,
      "logits/rejected": -1.0703027248382568,
      "logps/chosen": -1.5135245323181152,
      "logps/rejected": -2.4238083362579346,
      "loss": 1.5403,
      "nll_loss": 1.5104382038116455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15135245025157928,
      "rewards/margins": 0.09102839231491089,
      "rewards/rejected": -0.24238082766532898,
      "step": 726
    },
    {
      "epoch": 1.144996059889677,
      "grad_norm": 0.2138589769601822,
      "learning_rate": 1.8489736507882424e-06,
      "log_odds_chosen": 1.310249924659729,
      "log_odds_ratio": -0.2613876760005951,
      "logits/chosen": -0.4429868459701538,
      "logits/rejected": -1.1538211107254028,
      "logps/chosen": -1.4238499402999878,
      "logps/rejected": -2.548556327819824,
      "loss": 1.4541,
      "nll_loss": 1.427971363067627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14238499104976654,
      "rewards/margins": 0.11247065663337708,
      "rewards/rejected": -0.2548556625843048,
      "step": 727
    },
    {
      "epoch": 1.1465721040189125,
      "grad_norm": 0.19712799787521362,
      "learning_rate": 1.8434928234465716e-06,
      "log_odds_chosen": 1.1413190364837646,
      "log_odds_ratio": -0.2848305404186249,
      "logits/chosen": -0.5482323169708252,
      "logits/rejected": -1.1361311674118042,
      "logps/chosen": -1.3952279090881348,
      "logps/rejected": -2.354105234146118,
      "loss": 1.4393,
      "nll_loss": 1.4107712507247925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13952279090881348,
      "rewards/margins": 0.09588774293661118,
      "rewards/rejected": -0.23541051149368286,
      "step": 728
    },
    {
      "epoch": 1.1481481481481481,
      "grad_norm": 0.20782619714736938,
      "learning_rate": 1.8380131784418657e-06,
      "log_odds_chosen": 1.1528797149658203,
      "log_odds_ratio": -0.2929360568523407,
      "logits/chosen": -0.40681442618370056,
      "logits/rejected": -1.204696774482727,
      "logps/chosen": -1.4477262496948242,
      "logps/rejected": -2.4311623573303223,
      "loss": 1.4914,
      "nll_loss": 1.4621214866638184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14477261900901794,
      "rewards/margins": 0.0983436107635498,
      "rewards/rejected": -0.24311622977256775,
      "step": 729
    },
    {
      "epoch": 1.1497241922773838,
      "grad_norm": 0.20481131970882416,
      "learning_rate": 1.8325347571702259e-06,
      "log_odds_chosen": 1.0821598768234253,
      "log_odds_ratio": -0.30018386244773865,
      "logits/chosen": -0.517960786819458,
      "logits/rejected": -1.2233322858810425,
      "logps/chosen": -1.4976677894592285,
      "logps/rejected": -2.422020673751831,
      "loss": 1.5164,
      "nll_loss": 1.4863377809524536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14976678788661957,
      "rewards/margins": 0.0924353078007698,
      "rewards/rejected": -0.24220210313796997,
      "step": 730
    },
    {
      "epoch": 1.1513002364066194,
      "grad_norm": 0.220895916223526,
      "learning_rate": 1.827057601018509e-06,
      "log_odds_chosen": 1.166505217552185,
      "log_odds_ratio": -0.2821538746356964,
      "logits/chosen": -0.5554810166358948,
      "logits/rejected": -1.2023346424102783,
      "logps/chosen": -1.3842658996582031,
      "logps/rejected": -2.357381582260132,
      "loss": 1.4384,
      "nll_loss": 1.4102226495742798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13842660188674927,
      "rewards/margins": 0.09731155633926392,
      "rewards/rejected": -0.23573815822601318,
      "step": 731
    },
    {
      "epoch": 1.152876280535855,
      "grad_norm": 0.2122703343629837,
      "learning_rate": 1.8215817513640122e-06,
      "log_odds_chosen": 1.1712156534194946,
      "log_odds_ratio": -0.28429552912712097,
      "logits/chosen": -0.4573318064212799,
      "logits/rejected": -1.1952359676361084,
      "logps/chosen": -1.4386603832244873,
      "logps/rejected": -2.429713249206543,
      "loss": 1.467,
      "nll_loss": 1.438591480255127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14386604726314545,
      "rewards/margins": 0.09910527616739273,
      "rewards/rejected": -0.24297131597995758,
      "step": 732
    },
    {
      "epoch": 1.1544523246650906,
      "grad_norm": 0.1990232616662979,
      "learning_rate": 1.8161072495741647e-06,
      "log_odds_chosen": 1.142410397529602,
      "log_odds_ratio": -0.28524211049079895,
      "logits/chosen": -0.43404704332351685,
      "logits/rejected": -1.2667694091796875,
      "logps/chosen": -1.3983262777328491,
      "logps/rejected": -2.3543591499328613,
      "loss": 1.4415,
      "nll_loss": 1.4130146503448486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13983263075351715,
      "rewards/margins": 0.09560327976942062,
      "rewards/rejected": -0.23543591797351837,
      "step": 733
    },
    {
      "epoch": 1.1560283687943262,
      "grad_norm": 0.20943394303321838,
      "learning_rate": 1.810634137006213e-06,
      "log_odds_chosen": 0.839725136756897,
      "log_odds_ratio": -0.36195269227027893,
      "logits/chosen": -0.527044951915741,
      "logits/rejected": -0.9334619641304016,
      "logps/chosen": -1.4857885837554932,
      "logps/rejected": -2.1882660388946533,
      "loss": 1.5329,
      "nll_loss": 1.4966964721679688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14857886731624603,
      "rewards/margins": 0.07024775445461273,
      "rewards/rejected": -0.21882662177085876,
      "step": 734
    },
    {
      "epoch": 1.1576044129235619,
      "grad_norm": 0.20068326592445374,
      "learning_rate": 1.805162455006908e-06,
      "log_odds_chosen": 0.9912508130073547,
      "log_odds_ratio": -0.32414722442626953,
      "logits/chosen": -0.45623910427093506,
      "logits/rejected": -1.0643543004989624,
      "logps/chosen": -1.4959688186645508,
      "logps/rejected": -2.336454153060913,
      "loss": 1.5152,
      "nll_loss": 1.4827982187271118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14959688484668732,
      "rewards/margins": 0.08404852449893951,
      "rewards/rejected": -0.23364540934562683,
      "step": 735
    },
    {
      "epoch": 1.1591804570527975,
      "grad_norm": 0.20332376658916473,
      "learning_rate": 1.799692244912195e-06,
      "log_odds_chosen": 0.9452228546142578,
      "log_odds_ratio": -0.3528313636779785,
      "logits/chosen": -0.46050795912742615,
      "logits/rejected": -0.9370215535163879,
      "logps/chosen": -1.4905999898910522,
      "logps/rejected": -2.2954702377319336,
      "loss": 1.5395,
      "nll_loss": 1.50423002243042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14906001091003418,
      "rewards/margins": 0.08048702031373978,
      "rewards/rejected": -0.22954703867435455,
      "step": 736
    },
    {
      "epoch": 1.160756501182033,
      "grad_norm": 0.20273813605308533,
      "learning_rate": 1.7942235480468964e-06,
      "log_odds_chosen": 1.05588960647583,
      "log_odds_ratio": -0.3099244236946106,
      "logits/chosen": -0.49892657995224,
      "logits/rejected": -1.08159601688385,
      "logps/chosen": -1.3859915733337402,
      "logps/rejected": -2.265604019165039,
      "loss": 1.4299,
      "nll_loss": 1.3989274501800537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13859917223453522,
      "rewards/margins": 0.08796123415231705,
      "rewards/rejected": -0.22656041383743286,
      "step": 737
    },
    {
      "epoch": 1.1623325453112687,
      "grad_norm": 0.20004484057426453,
      "learning_rate": 1.7887564057244062e-06,
      "log_odds_chosen": 1.0720691680908203,
      "log_odds_ratio": -0.3204352855682373,
      "logits/chosen": -0.4647495448589325,
      "logits/rejected": -1.1344199180603027,
      "logps/chosen": -1.3437665700912476,
      "logps/rejected": -2.235045909881592,
      "loss": 1.392,
      "nll_loss": 1.3599578142166138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.134376659989357,
      "rewards/margins": 0.08912792801856995,
      "rewards/rejected": -0.22350457310676575,
      "step": 738
    },
    {
      "epoch": 1.1639085894405043,
      "grad_norm": 0.21401451528072357,
      "learning_rate": 1.7832908592463732e-06,
      "log_odds_chosen": 0.982843279838562,
      "log_odds_ratio": -0.3265047073364258,
      "logits/chosen": -0.5769224762916565,
      "logits/rejected": -1.1763486862182617,
      "logps/chosen": -1.4734010696411133,
      "logps/rejected": -2.2912302017211914,
      "loss": 1.5189,
      "nll_loss": 1.4862951040267944,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14734011888504028,
      "rewards/margins": 0.08178289234638214,
      "rewards/rejected": -0.22912301123142242,
      "step": 739
    },
    {
      "epoch": 1.16548463356974,
      "grad_norm": 0.21833467483520508,
      "learning_rate": 1.7778269499023908e-06,
      "log_odds_chosen": 1.0863176584243774,
      "log_odds_ratio": -0.29996857047080994,
      "logits/chosen": -0.4732495844364166,
      "logits/rejected": -1.181039810180664,
      "logps/chosen": -1.5098434686660767,
      "logps/rejected": -2.439993381500244,
      "loss": 1.5424,
      "nll_loss": 1.5123927593231201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15098436176776886,
      "rewards/margins": 0.09301498532295227,
      "rewards/rejected": -0.24399936199188232,
      "step": 740
    },
    {
      "epoch": 1.1670606776989756,
      "grad_norm": 0.23689228296279907,
      "learning_rate": 1.7723647189696843e-06,
      "log_odds_chosen": 1.017876148223877,
      "log_odds_ratio": -0.31690657138824463,
      "logits/chosen": -0.4602625072002411,
      "logits/rejected": -0.9403397440910339,
      "logps/chosen": -1.396916151046753,
      "logps/rejected": -2.2411394119262695,
      "loss": 1.4526,
      "nll_loss": 1.4209301471710205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13969160616397858,
      "rewards/margins": 0.08442234992980957,
      "rewards/rejected": -0.22411397099494934,
      "step": 741
    },
    {
      "epoch": 1.1686367218282112,
      "grad_norm": 0.21541060507297516,
      "learning_rate": 1.7669042077127982e-06,
      "log_odds_chosen": 1.1569600105285645,
      "log_odds_ratio": -0.28846845030784607,
      "logits/chosen": -0.44710707664489746,
      "logits/rejected": -1.1649326086044312,
      "logps/chosen": -1.4673033952713013,
      "logps/rejected": -2.4524731636047363,
      "loss": 1.4956,
      "nll_loss": 1.4667805433273315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14673034846782684,
      "rewards/margins": 0.09851697832345963,
      "rewards/rejected": -0.24524731934070587,
      "step": 742
    },
    {
      "epoch": 1.1702127659574468,
      "grad_norm": 0.1897234320640564,
      "learning_rate": 1.7614454573832886e-06,
      "log_odds_chosen": 1.200345516204834,
      "log_odds_ratio": -0.26881635189056396,
      "logits/chosen": -0.5847516655921936,
      "logits/rejected": -1.1564751863479614,
      "logps/chosen": -1.347001552581787,
      "logps/rejected": -2.3438925743103027,
      "loss": 1.3994,
      "nll_loss": 1.3725018501281738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13470014929771423,
      "rewards/margins": 0.09968913346529007,
      "rewards/rejected": -0.2343892604112625,
      "step": 743
    },
    {
      "epoch": 1.1717888100866825,
      "grad_norm": 0.22839771211147308,
      "learning_rate": 1.7559885092194058e-06,
      "log_odds_chosen": 0.8739182353019714,
      "log_odds_ratio": -0.3523673117160797,
      "logits/chosen": -0.4338620603084564,
      "logits/rejected": -1.1961348056793213,
      "logps/chosen": -1.4970002174377441,
      "logps/rejected": -2.2313930988311768,
      "loss": 1.5439,
      "nll_loss": 1.5086549520492554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14970003068447113,
      "rewards/margins": 0.07343930006027222,
      "rewards/rejected": -0.22313933074474335,
      "step": 744
    },
    {
      "epoch": 1.173364854215918,
      "grad_norm": 0.19497662782669067,
      "learning_rate": 1.7505334044457871e-06,
      "log_odds_chosen": 1.0764094591140747,
      "log_odds_ratio": -0.2964109182357788,
      "logits/chosen": -0.46895459294319153,
      "logits/rejected": -1.2059799432754517,
      "logps/chosen": -1.3835914134979248,
      "logps/rejected": -2.2742092609405518,
      "loss": 1.4197,
      "nll_loss": 1.390027642250061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13835912942886353,
      "rewards/margins": 0.08906181156635284,
      "rewards/rejected": -0.22742094099521637,
      "step": 745
    },
    {
      "epoch": 1.1749408983451537,
      "grad_norm": 0.19152005016803741,
      "learning_rate": 1.7450801842731443e-06,
      "log_odds_chosen": 1.109420657157898,
      "log_odds_ratio": -0.288770854473114,
      "logits/chosen": -0.3884856104850769,
      "logits/rejected": -1.0914006233215332,
      "logps/chosen": -1.491636872291565,
      "logps/rejected": -2.4389634132385254,
      "loss": 1.5245,
      "nll_loss": 1.4956486225128174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14916369318962097,
      "rewards/margins": 0.09473264962434769,
      "rewards/rejected": -0.24389633536338806,
      "step": 746
    },
    {
      "epoch": 1.1765169424743893,
      "grad_norm": 0.20200739800930023,
      "learning_rate": 1.7396288898979497e-06,
      "log_odds_chosen": 1.2157838344573975,
      "log_odds_ratio": -0.2654762864112854,
      "logits/chosen": -0.4906774163246155,
      "logits/rejected": -1.1920275688171387,
      "logps/chosen": -1.3838553428649902,
      "logps/rejected": -2.406334638595581,
      "loss": 1.4194,
      "nll_loss": 1.3928059339523315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1383855640888214,
      "rewards/margins": 0.1022479385137558,
      "rewards/rejected": -0.24063348770141602,
      "step": 747
    },
    {
      "epoch": 1.178092986603625,
      "grad_norm": 0.20935089886188507,
      "learning_rate": 1.7341795625021292e-06,
      "log_odds_chosen": 1.1848759651184082,
      "log_odds_ratio": -0.2726721167564392,
      "logits/chosen": -0.47776636481285095,
      "logits/rejected": -1.3447891473770142,
      "logps/chosen": -1.4217549562454224,
      "logps/rejected": -2.423642873764038,
      "loss": 1.4639,
      "nll_loss": 1.436589241027832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14217551052570343,
      "rewards/margins": 0.10018878430128098,
      "rewards/rejected": -0.2423642873764038,
      "step": 748
    },
    {
      "epoch": 1.1796690307328606,
      "grad_norm": 0.20518095791339874,
      "learning_rate": 1.7287322432527485e-06,
      "log_odds_chosen": 1.0967676639556885,
      "log_odds_ratio": -0.29402458667755127,
      "logits/chosen": -0.5059671998023987,
      "logits/rejected": -1.2210016250610352,
      "logps/chosen": -1.5061734914779663,
      "logps/rejected": -2.44329571723938,
      "loss": 1.5351,
      "nll_loss": 1.5057346820831299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15061736106872559,
      "rewards/margins": 0.09371224790811539,
      "rewards/rejected": -0.24432958662509918,
      "step": 749
    },
    {
      "epoch": 1.1812450748620962,
      "grad_norm": 0.20950697362422943,
      "learning_rate": 1.7232869733017038e-06,
      "log_odds_chosen": 1.0387558937072754,
      "log_odds_ratio": -0.31163373589515686,
      "logits/chosen": -0.48433902859687805,
      "logits/rejected": -1.212583303451538,
      "logps/chosen": -1.4577950239181519,
      "logps/rejected": -2.3341586589813232,
      "loss": 1.4901,
      "nll_loss": 1.4589087963104248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14577950537204742,
      "rewards/margins": 0.08763638138771057,
      "rewards/rejected": -0.2334158718585968,
      "step": 750
    },
    {
      "epoch": 1.1828211189913318,
      "grad_norm": 0.21631582081317902,
      "learning_rate": 1.7178437937854065e-06,
      "log_odds_chosen": 1.3533402681350708,
      "log_odds_ratio": -0.23426946997642517,
      "logits/chosen": -0.5732666850090027,
      "logits/rejected": -1.3766391277313232,
      "logps/chosen": -1.4981077909469604,
      "logps/rejected": -2.6693079471588135,
      "loss": 1.5234,
      "nll_loss": 1.499996542930603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1498107761144638,
      "rewards/margins": 0.11712003499269485,
      "rewards/rejected": -0.26693081855773926,
      "step": 751
    },
    {
      "epoch": 1.1843971631205674,
      "grad_norm": 0.23669928312301636,
      "learning_rate": 1.7124027458244794e-06,
      "log_odds_chosen": 1.1570227146148682,
      "log_odds_ratio": -0.2833133935928345,
      "logits/chosen": -0.4643342196941376,
      "logits/rejected": -1.042296290397644,
      "logps/chosen": -1.4195529222488403,
      "logps/rejected": -2.399897813796997,
      "loss": 1.4656,
      "nll_loss": 1.4372928142547607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14195531606674194,
      "rewards/margins": 0.09803447872400284,
      "rewards/rejected": -0.239989772439003,
      "step": 752
    },
    {
      "epoch": 1.185973207249803,
      "grad_norm": 0.1962566375732422,
      "learning_rate": 1.7069638705234407e-06,
      "log_odds_chosen": 0.8747801780700684,
      "log_odds_ratio": -0.35416561365127563,
      "logits/chosen": -0.3614555299282074,
      "logits/rejected": -1.0183345079421997,
      "logps/chosen": -1.4903472661972046,
      "logps/rejected": -2.225078821182251,
      "loss": 1.5378,
      "nll_loss": 1.5023765563964844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14903472363948822,
      "rewards/margins": 0.07347312569618225,
      "rewards/rejected": -0.22250784933567047,
      "step": 753
    },
    {
      "epoch": 1.1875492513790387,
      "grad_norm": 0.20509277284145355,
      "learning_rate": 1.7015272089703954e-06,
      "log_odds_chosen": 1.076144814491272,
      "log_odds_ratio": -0.3019227683544159,
      "logits/chosen": -0.5210514664649963,
      "logits/rejected": -1.2102211713790894,
      "logps/chosen": -1.4598164558410645,
      "logps/rejected": -2.3627476692199707,
      "loss": 1.4867,
      "nll_loss": 1.4565184116363525,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14598165452480316,
      "rewards/margins": 0.09029310941696167,
      "rewards/rejected": -0.23627477884292603,
      "step": 754
    },
    {
      "epoch": 1.1891252955082743,
      "grad_norm": 0.20275671780109406,
      "learning_rate": 1.6960928022367261e-06,
      "log_odds_chosen": 1.0239466428756714,
      "log_odds_ratio": -0.31612369418144226,
      "logits/chosen": -0.5038920044898987,
      "logits/rejected": -1.1239041090011597,
      "logps/chosen": -1.4878877401351929,
      "logps/rejected": -2.3608345985412598,
      "loss": 1.5215,
      "nll_loss": 1.4899373054504395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14878879487514496,
      "rewards/margins": 0.08729467540979385,
      "rewards/rejected": -0.23608346283435822,
      "step": 755
    },
    {
      "epoch": 1.19070133963751,
      "grad_norm": 0.2775387763977051,
      "learning_rate": 1.6906606913767776e-06,
      "log_odds_chosen": 1.0123605728149414,
      "log_odds_ratio": -0.31833451986312866,
      "logits/chosen": -0.5189042091369629,
      "logits/rejected": -1.2062630653381348,
      "logps/chosen": -1.3932063579559326,
      "logps/rejected": -2.235832691192627,
      "loss": 1.4373,
      "nll_loss": 1.4054844379425049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13932064175605774,
      "rewards/margins": 0.08426263928413391,
      "rewards/rejected": -0.22358328104019165,
      "step": 756
    },
    {
      "epoch": 1.1922773837667455,
      "grad_norm": 0.21399083733558655,
      "learning_rate": 1.6852309174275543e-06,
      "log_odds_chosen": 1.0481847524642944,
      "log_odds_ratio": -0.3066259026527405,
      "logits/chosen": -0.6260600686073303,
      "logits/rejected": -1.1545990705490112,
      "logps/chosen": -1.3172390460968018,
      "logps/rejected": -2.173739194869995,
      "loss": 1.3593,
      "nll_loss": 1.3286503553390503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13172391057014465,
      "rewards/margins": 0.0856500193476677,
      "rewards/rejected": -0.21737393736839294,
      "step": 757
    },
    {
      "epoch": 1.1938534278959811,
      "grad_norm": 0.20945127308368683,
      "learning_rate": 1.6798035214084047e-06,
      "log_odds_chosen": 1.0964981317520142,
      "log_odds_ratio": -0.29487645626068115,
      "logits/chosen": -0.5626031160354614,
      "logits/rejected": -1.301207184791565,
      "logps/chosen": -1.4191815853118896,
      "logps/rejected": -2.341688394546509,
      "loss": 1.4616,
      "nll_loss": 1.4320943355560303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419181525707245,
      "rewards/margins": 0.09225069731473923,
      "rewards/rejected": -0.23416884243488312,
      "step": 758
    },
    {
      "epoch": 1.1954294720252168,
      "grad_norm": 0.21016307175159454,
      "learning_rate": 1.674378544320714e-06,
      "log_odds_chosen": 1.373252511024475,
      "log_odds_ratio": -0.2384091317653656,
      "logits/chosen": -0.47239047288894653,
      "logits/rejected": -1.2062098979949951,
      "logps/chosen": -1.3751307725906372,
      "logps/rejected": -2.534651041030884,
      "loss": 1.4179,
      "nll_loss": 1.3941071033477783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13751307129859924,
      "rewards/margins": 0.11595205217599869,
      "rewards/rejected": -0.25346511602401733,
      "step": 759
    },
    {
      "epoch": 1.1970055161544524,
      "grad_norm": 0.20362719893455505,
      "learning_rate": 1.6689560271475922e-06,
      "log_odds_chosen": 1.2336678504943848,
      "log_odds_ratio": -0.2593075633049011,
      "logits/chosen": -0.5030820965766907,
      "logits/rejected": -1.2119706869125366,
      "logps/chosen": -1.3946470022201538,
      "logps/rejected": -2.4335319995880127,
      "loss": 1.4286,
      "nll_loss": 1.4026226997375488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13946469128131866,
      "rewards/margins": 0.10388850420713425,
      "rewards/rejected": -0.2433532178401947,
      "step": 760
    },
    {
      "epoch": 1.198581560283688,
      "grad_norm": 0.1893111616373062,
      "learning_rate": 1.6635360108535665e-06,
      "log_odds_chosen": 1.1931270360946655,
      "log_odds_ratio": -0.27546483278274536,
      "logits/chosen": -0.5286849737167358,
      "logits/rejected": -1.1763360500335693,
      "logps/chosen": -1.345786213874817,
      "logps/rejected": -2.34234356880188,
      "loss": 1.3977,
      "nll_loss": 1.3701961040496826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1345786303281784,
      "rewards/margins": 0.09965574741363525,
      "rewards/rejected": -0.23423437774181366,
      "step": 761
    },
    {
      "epoch": 1.2001576044129236,
      "grad_norm": 0.20345623791217804,
      "learning_rate": 1.6581185363842717e-06,
      "log_odds_chosen": 1.195793628692627,
      "log_odds_ratio": -0.27840933203697205,
      "logits/chosen": -0.5871078968048096,
      "logits/rejected": -1.2172828912734985,
      "logps/chosen": -1.3926793336868286,
      "logps/rejected": -2.4013118743896484,
      "loss": 1.422,
      "nll_loss": 1.394186019897461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1392679363489151,
      "rewards/margins": 0.1008632630109787,
      "rewards/rejected": -0.2401311844587326,
      "step": 762
    },
    {
      "epoch": 1.2017336485421592,
      "grad_norm": 0.2188248485326767,
      "learning_rate": 1.6527036446661393e-06,
      "log_odds_chosen": 1.1231153011322021,
      "log_odds_ratio": -0.2910814881324768,
      "logits/chosen": -0.5094649195671082,
      "logits/rejected": -1.2092803716659546,
      "logps/chosen": -1.4155865907669067,
      "logps/rejected": -2.3610305786132812,
      "loss": 1.4509,
      "nll_loss": 1.421754240989685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1415586620569229,
      "rewards/margins": 0.09454439580440521,
      "rewards/rejected": -0.23610305786132812,
      "step": 763
    },
    {
      "epoch": 1.2033096926713949,
      "grad_norm": 0.21531014144420624,
      "learning_rate": 1.6472913766060901e-06,
      "log_odds_chosen": 1.2657768726348877,
      "log_odds_ratio": -0.25664135813713074,
      "logits/chosen": -0.6371492743492126,
      "logits/rejected": -1.1650924682617188,
      "logps/chosen": -1.3501787185668945,
      "logps/rejected": -2.4026849269866943,
      "loss": 1.3931,
      "nll_loss": 1.3674345016479492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13501787185668945,
      "rewards/margins": 0.10525061190128326,
      "rewards/rejected": -0.24026848375797272,
      "step": 764
    },
    {
      "epoch": 1.2048857368006305,
      "grad_norm": 0.20706751942634583,
      "learning_rate": 1.6418817730912252e-06,
      "log_odds_chosen": 1.0244804620742798,
      "log_odds_ratio": -0.32321858406066895,
      "logits/chosen": -0.5738548040390015,
      "logits/rejected": -0.9733390808105469,
      "logps/chosen": -1.4535335302352905,
      "logps/rejected": -2.31813383102417,
      "loss": 1.4905,
      "nll_loss": 1.4582172632217407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14535334706306458,
      "rewards/margins": 0.08646003901958466,
      "rewards/rejected": -0.23181338608264923,
      "step": 765
    },
    {
      "epoch": 1.2064617809298661,
      "grad_norm": 0.20738485455513,
      "learning_rate": 1.6364748749885133e-06,
      "log_odds_chosen": 0.8729405403137207,
      "log_odds_ratio": -0.35316628217697144,
      "logits/chosen": -0.5492048859596252,
      "logits/rejected": -1.0449838638305664,
      "logps/chosen": -1.412824273109436,
      "logps/rejected": -2.135004758834839,
      "loss": 1.4509,
      "nll_loss": 1.4155434370040894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14128242433071136,
      "rewards/margins": 0.07221804559230804,
      "rewards/rejected": -0.2135004848241806,
      "step": 766
    },
    {
      "epoch": 1.2080378250591017,
      "grad_norm": 0.22474469244480133,
      "learning_rate": 1.6310707231444883e-06,
      "log_odds_chosen": 1.120745301246643,
      "log_odds_ratio": -0.2919396460056305,
      "logits/chosen": -0.5536065697669983,
      "logits/rejected": -1.0730319023132324,
      "logps/chosen": -1.4180700778961182,
      "logps/rejected": -2.3632853031158447,
      "loss": 1.4496,
      "nll_loss": 1.4203715324401855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14180700480937958,
      "rewards/margins": 0.09452151507139206,
      "rewards/rejected": -0.23632854223251343,
      "step": 767
    },
    {
      "epoch": 1.2096138691883374,
      "grad_norm": 0.21062271296977997,
      "learning_rate": 1.625669358384936e-06,
      "log_odds_chosen": 0.8960245847702026,
      "log_odds_ratio": -0.356604665517807,
      "logits/chosen": -0.45744889974594116,
      "logits/rejected": -0.9470247030258179,
      "logps/chosen": -1.4859485626220703,
      "logps/rejected": -2.240083694458008,
      "loss": 1.5261,
      "nll_loss": 1.4904659986495972,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14859488606452942,
      "rewards/margins": 0.07541351020336151,
      "rewards/rejected": -0.22400839626789093,
      "step": 768
    },
    {
      "epoch": 1.211189913317573,
      "grad_norm": 0.2158898413181305,
      "learning_rate": 1.620270821514587e-06,
      "log_odds_chosen": 1.2348099946975708,
      "log_odds_ratio": -0.26864558458328247,
      "logits/chosen": -0.4252718985080719,
      "logits/rejected": -1.0990837812423706,
      "logps/chosen": -1.5031522512435913,
      "logps/rejected": -2.572052001953125,
      "loss": 1.5245,
      "nll_loss": 1.4976516962051392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15031522512435913,
      "rewards/margins": 0.106890007853508,
      "rewards/rejected": -0.25720521807670593,
      "step": 769
    },
    {
      "epoch": 1.2127659574468086,
      "grad_norm": 0.22295641899108887,
      "learning_rate": 1.6148751533168104e-06,
      "log_odds_chosen": 1.422515869140625,
      "log_odds_ratio": -0.23112696409225464,
      "logits/chosen": -0.5481671094894409,
      "logits/rejected": -1.326934814453125,
      "logps/chosen": -1.4720447063446045,
      "logps/rejected": -2.7048351764678955,
      "loss": 1.4892,
      "nll_loss": 1.4660604000091553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1472044736146927,
      "rewards/margins": 0.12327902764081955,
      "rewards/rejected": -0.27048349380493164,
      "step": 770
    },
    {
      "epoch": 1.2143420015760442,
      "grad_norm": 0.2067849189043045,
      "learning_rate": 1.6094823945532996e-06,
      "log_odds_chosen": 1.069305181503296,
      "log_odds_ratio": -0.3097341060638428,
      "logits/chosen": -0.5835323333740234,
      "logits/rejected": -1.17497718334198,
      "logps/chosen": -1.4260441064834595,
      "logps/rejected": -2.3290538787841797,
      "loss": 1.4547,
      "nll_loss": 1.4237654209136963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14260441064834595,
      "rewards/margins": 0.09030096977949142,
      "rewards/rejected": -0.23290538787841797,
      "step": 771
    },
    {
      "epoch": 1.2159180457052798,
      "grad_norm": 0.23173509538173676,
      "learning_rate": 1.6040925859637728e-06,
      "log_odds_chosen": 1.1006717681884766,
      "log_odds_ratio": -0.2988271713256836,
      "logits/chosen": -0.5543443560600281,
      "logits/rejected": -1.1042602062225342,
      "logps/chosen": -1.5097880363464355,
      "logps/rejected": -2.450514793395996,
      "loss": 1.5391,
      "nll_loss": 1.5092402696609497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15097880363464355,
      "rewards/margins": 0.09407266229391098,
      "rewards/rejected": -0.24505145847797394,
      "step": 772
    },
    {
      "epoch": 1.2174940898345155,
      "grad_norm": 0.2444857805967331,
      "learning_rate": 1.5987057682656596e-06,
      "log_odds_chosen": 1.037376046180725,
      "log_odds_ratio": -0.3062412738800049,
      "logits/chosen": -0.46560242772102356,
      "logits/rejected": -1.0235753059387207,
      "logps/chosen": -1.547714114189148,
      "logps/rejected": -2.438072681427002,
      "loss": 1.5828,
      "nll_loss": 1.5521423816680908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15477140247821808,
      "rewards/margins": 0.08903585374355316,
      "rewards/rejected": -0.24380725622177124,
      "step": 773
    },
    {
      "epoch": 1.219070133963751,
      "grad_norm": 0.23813925683498383,
      "learning_rate": 1.593321982153795e-06,
      "log_odds_chosen": 1.0737147331237793,
      "log_odds_ratio": -0.29867035150527954,
      "logits/chosen": -0.5678068995475769,
      "logits/rejected": -1.246809482574463,
      "logps/chosen": -1.5689599514007568,
      "logps/rejected": -2.4946978092193604,
      "loss": 1.588,
      "nll_loss": 1.5581533908843994,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15689599514007568,
      "rewards/margins": 0.09257378429174423,
      "rewards/rejected": -0.24946977198123932,
      "step": 774
    },
    {
      "epoch": 1.2206461780929867,
      "grad_norm": 0.22847957909107208,
      "learning_rate": 1.5879412683001106e-06,
      "log_odds_chosen": 0.9964001774787903,
      "log_odds_ratio": -0.3199702203273773,
      "logits/chosen": -0.5840435028076172,
      "logits/rejected": -0.9981973171234131,
      "logps/chosen": -1.386216163635254,
      "logps/rejected": -2.212662696838379,
      "loss": 1.4376,
      "nll_loss": 1.4055652618408203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13862162828445435,
      "rewards/margins": 0.08264464139938354,
      "rewards/rejected": -0.2212662547826767,
      "step": 775
    },
    {
      "epoch": 1.2222222222222223,
      "grad_norm": 0.21677440404891968,
      "learning_rate": 1.5825636673533298e-06,
      "log_odds_chosen": 1.4123928546905518,
      "log_odds_ratio": -0.2274841070175171,
      "logits/chosen": -0.6195847988128662,
      "logits/rejected": -1.400698184967041,
      "logps/chosen": -1.4514834880828857,
      "logps/rejected": -2.6688995361328125,
      "loss": 1.4626,
      "nll_loss": 1.439857840538025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14514833688735962,
      "rewards/margins": 0.1217416301369667,
      "rewards/rejected": -0.2668899595737457,
      "step": 776
    },
    {
      "epoch": 1.2237982663514577,
      "grad_norm": 0.22444766759872437,
      "learning_rate": 1.5771892199386598e-06,
      "log_odds_chosen": 1.0706374645233154,
      "log_odds_ratio": -0.3026072084903717,
      "logits/chosen": -0.5365505218505859,
      "logits/rejected": -1.0839470624923706,
      "logps/chosen": -1.470859408378601,
      "logps/rejected": -2.3822879791259766,
      "loss": 1.5179,
      "nll_loss": 1.4876376390457153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14708594977855682,
      "rewards/margins": 0.09114284813404083,
      "rewards/rejected": -0.23822878301143646,
      "step": 777
    },
    {
      "epoch": 1.2253743104806936,
      "grad_norm": 0.19800859689712524,
      "learning_rate": 1.5718179666574834e-06,
      "log_odds_chosen": 1.3811945915222168,
      "log_odds_ratio": -0.23333391547203064,
      "logits/chosen": -0.45926934480667114,
      "logits/rejected": -1.278073787689209,
      "logps/chosen": -1.4371553659439087,
      "logps/rejected": -2.623957395553589,
      "loss": 1.4734,
      "nll_loss": 1.4500432014465332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437155306339264,
      "rewards/margins": 0.1186801940202713,
      "rewards/rejected": -0.2623957395553589,
      "step": 778
    },
    {
      "epoch": 1.226950354609929,
      "grad_norm": 0.21464787423610687,
      "learning_rate": 1.5664499480870539e-06,
      "log_odds_chosen": 1.343256950378418,
      "log_odds_ratio": -0.238076314330101,
      "logits/chosen": -0.49939605593681335,
      "logits/rejected": -1.2993298768997192,
      "logps/chosen": -1.4640394449234009,
      "logps/rejected": -2.619065046310425,
      "loss": 1.5042,
      "nll_loss": 1.4803446531295776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1464039534330368,
      "rewards/margins": 0.11550256609916687,
      "rewards/rejected": -0.2619065046310425,
      "step": 779
    },
    {
      "epoch": 1.2285263987391648,
      "grad_norm": 0.2220604568719864,
      "learning_rate": 1.5610852047801875e-06,
      "log_odds_chosen": 0.9813396334648132,
      "log_odds_ratio": -0.3234339952468872,
      "logits/chosen": -0.5806028842926025,
      "logits/rejected": -1.0990912914276123,
      "logps/chosen": -1.3953056335449219,
      "logps/rejected": -2.211315393447876,
      "loss": 1.4242,
      "nll_loss": 1.3918310403823853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13953056931495667,
      "rewards/margins": 0.08160099387168884,
      "rewards/rejected": -0.2211315631866455,
      "step": 780
    },
    {
      "epoch": 1.2301024428684002,
      "grad_norm": 0.21280692517757416,
      "learning_rate": 1.5557237772649567e-06,
      "log_odds_chosen": 0.9394214153289795,
      "log_odds_ratio": -0.3346244990825653,
      "logits/chosen": -0.4719075560569763,
      "logits/rejected": -1.2131590843200684,
      "logps/chosen": -1.4758708477020264,
      "logps/rejected": -2.2647206783294678,
      "loss": 1.5144,
      "nll_loss": 1.4809293746948242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14758709073066711,
      "rewards/margins": 0.07888500392436981,
      "rewards/rejected": -0.22647210955619812,
      "step": 781
    },
    {
      "epoch": 1.231678486997636,
      "grad_norm": 0.19902761280536652,
      "learning_rate": 1.5503657060443866e-06,
      "log_odds_chosen": 1.0476891994476318,
      "log_odds_ratio": -0.30895373225212097,
      "logits/chosen": -0.512697160243988,
      "logits/rejected": -1.1837438344955444,
      "logps/chosen": -1.3997454643249512,
      "logps/rejected": -2.2706665992736816,
      "loss": 1.4494,
      "nll_loss": 1.4185155630111694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13997454941272736,
      "rewards/margins": 0.08709211647510529,
      "rewards/rejected": -0.22706666588783264,
      "step": 782
    },
    {
      "epoch": 1.2332545311268714,
      "grad_norm": 0.20308339595794678,
      "learning_rate": 1.5450110315961457e-06,
      "log_odds_chosen": 1.2954864501953125,
      "log_odds_ratio": -0.2562984526157379,
      "logits/chosen": -0.5449813604354858,
      "logits/rejected": -1.2691328525543213,
      "logps/chosen": -1.43440580368042,
      "logps/rejected": -2.5437111854553223,
      "loss": 1.4794,
      "nll_loss": 1.453803300857544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14344057440757751,
      "rewards/margins": 0.11093056201934814,
      "rewards/rejected": -0.25437113642692566,
      "step": 783
    },
    {
      "epoch": 1.2348305752561073,
      "grad_norm": 0.20815418660640717,
      "learning_rate": 1.539659794372243e-06,
      "log_odds_chosen": 1.2467131614685059,
      "log_odds_ratio": -0.27821114659309387,
      "logits/chosen": -0.5841869115829468,
      "logits/rejected": -1.2688829898834229,
      "logps/chosen": -1.4486753940582275,
      "logps/rejected": -2.517613172531128,
      "loss": 1.4836,
      "nll_loss": 1.4558051824569702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14486753940582275,
      "rewards/margins": 0.10689377039670944,
      "rewards/rejected": -0.2517613172531128,
      "step": 784
    },
    {
      "epoch": 1.2364066193853427,
      "grad_norm": 0.21452639997005463,
      "learning_rate": 1.5343120347987172e-06,
      "log_odds_chosen": 1.2390491962432861,
      "log_odds_ratio": -0.25820380449295044,
      "logits/chosen": -0.5369148850440979,
      "logits/rejected": -1.2798552513122559,
      "logps/chosen": -1.4309568405151367,
      "logps/rejected": -2.4839766025543213,
      "loss": 1.4814,
      "nll_loss": 1.4556230306625366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1430956870317459,
      "rewards/margins": 0.10530199110507965,
      "rewards/rejected": -0.24839766323566437,
      "step": 785
    },
    {
      "epoch": 1.2379826635145785,
      "grad_norm": 0.20278997719287872,
      "learning_rate": 1.5289677932753398e-06,
      "log_odds_chosen": 0.9853284358978271,
      "log_odds_ratio": -0.3262062668800354,
      "logits/chosen": -0.5501490831375122,
      "logits/rejected": -1.1359549760818481,
      "logps/chosen": -1.4639358520507812,
      "logps/rejected": -2.2930803298950195,
      "loss": 1.5017,
      "nll_loss": 1.4690982103347778,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1463935673236847,
      "rewards/margins": 0.08291444182395935,
      "rewards/rejected": -0.22930802404880524,
      "step": 786
    },
    {
      "epoch": 1.239558707643814,
      "grad_norm": 0.19819343090057373,
      "learning_rate": 1.5236271101753017e-06,
      "log_odds_chosen": 1.1965906620025635,
      "log_odds_ratio": -0.265939325094223,
      "logits/chosen": -0.5788232684135437,
      "logits/rejected": -1.2083020210266113,
      "logps/chosen": -1.4637471437454224,
      "logps/rejected": -2.4821419715881348,
      "loss": 1.4846,
      "nll_loss": 1.4580525159835815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14637473225593567,
      "rewards/margins": 0.10183949023485184,
      "rewards/rejected": -0.2482142150402069,
      "step": 787
    },
    {
      "epoch": 1.2411347517730495,
      "grad_norm": 0.21007592976093292,
      "learning_rate": 1.5182900258449135e-06,
      "log_odds_chosen": 1.3082517385482788,
      "log_odds_ratio": -0.2442568838596344,
      "logits/chosen": -0.5462801456451416,
      "logits/rejected": -1.2828822135925293,
      "logps/chosen": -1.4453314542770386,
      "logps/rejected": -2.5659899711608887,
      "loss": 1.4795,
      "nll_loss": 1.455052137374878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14453314244747162,
      "rewards/margins": 0.1120658740401268,
      "rewards/rejected": -0.2565990090370178,
      "step": 788
    },
    {
      "epoch": 1.2427107959022852,
      "grad_norm": 0.21201248466968536,
      "learning_rate": 1.5129565806032986e-06,
      "log_odds_chosen": 1.2567715644836426,
      "log_odds_ratio": -0.26283809542655945,
      "logits/chosen": -0.556973397731781,
      "logits/rejected": -1.1688377857208252,
      "logps/chosen": -1.331041693687439,
      "logps/rejected": -2.381495714187622,
      "loss": 1.3832,
      "nll_loss": 1.3569332361221313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13310417532920837,
      "rewards/margins": 0.1050453931093216,
      "rewards/rejected": -0.23814958333969116,
      "step": 789
    },
    {
      "epoch": 1.2442868400315208,
      "grad_norm": 0.24223382771015167,
      "learning_rate": 1.507626814742087e-06,
      "log_odds_chosen": 1.0849443674087524,
      "log_odds_ratio": -0.2990293502807617,
      "logits/chosen": -0.5767560601234436,
      "logits/rejected": -1.0875062942504883,
      "logps/chosen": -1.516034722328186,
      "logps/rejected": -2.442598581314087,
      "loss": 1.5562,
      "nll_loss": 1.5263423919677734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15160347521305084,
      "rewards/margins": 0.09265640377998352,
      "rewards/rejected": -0.24425986409187317,
      "step": 790
    },
    {
      "epoch": 1.2458628841607564,
      "grad_norm": 0.20978893339633942,
      "learning_rate": 1.502300768525115e-06,
      "log_odds_chosen": 1.1761633157730103,
      "log_odds_ratio": -0.29160240292549133,
      "logits/chosen": -0.5480546951293945,
      "logits/rejected": -1.253957986831665,
      "logps/chosen": -1.5305815935134888,
      "logps/rejected": -2.5474436283111572,
      "loss": 1.5369,
      "nll_loss": 1.50777006149292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15305818617343903,
      "rewards/margins": 0.10168616473674774,
      "rewards/rejected": -0.25474435091018677,
      "step": 791
    },
    {
      "epoch": 1.247438928289992,
      "grad_norm": 0.2149065136909485,
      "learning_rate": 1.4969784821881177e-06,
      "log_odds_chosen": 1.0186469554901123,
      "log_odds_ratio": -0.3178043067455292,
      "logits/chosen": -0.5656739473342896,
      "logits/rejected": -1.0273687839508057,
      "logps/chosen": -1.3979710340499878,
      "logps/rejected": -2.244584798812866,
      "loss": 1.4568,
      "nll_loss": 1.4250069856643677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13979710638523102,
      "rewards/margins": 0.08466139435768127,
      "rewards/rejected": -0.2244585156440735,
      "step": 792
    },
    {
      "epoch": 1.2490149724192277,
      "grad_norm": 0.21415865421295166,
      "learning_rate": 1.4916599959384262e-06,
      "log_odds_chosen": 0.9932125806808472,
      "log_odds_ratio": -0.3196934759616852,
      "logits/chosen": -0.573021411895752,
      "logits/rejected": -1.244803547859192,
      "logps/chosen": -1.4770406484603882,
      "logps/rejected": -2.3149125576019287,
      "loss": 1.5079,
      "nll_loss": 1.475886344909668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14770406484603882,
      "rewards/margins": 0.08378720283508301,
      "rewards/rejected": -0.23149126768112183,
      "step": 793
    },
    {
      "epoch": 1.2505910165484633,
      "grad_norm": 0.21877042949199677,
      "learning_rate": 1.4863453499546643e-06,
      "log_odds_chosen": 1.2537623643875122,
      "log_odds_ratio": -0.2612696886062622,
      "logits/chosen": -0.5212925672531128,
      "logits/rejected": -1.1985936164855957,
      "logps/chosen": -1.4091440439224243,
      "logps/rejected": -2.4729738235473633,
      "loss": 1.4444,
      "nll_loss": 1.4182521104812622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1409144103527069,
      "rewards/margins": 0.10638298839330673,
      "rewards/rejected": -0.24729740619659424,
      "step": 794
    },
    {
      "epoch": 1.252167060677699,
      "grad_norm": 0.21476829051971436,
      "learning_rate": 1.4810345843864427e-06,
      "log_odds_chosen": 1.1288681030273438,
      "log_odds_ratio": -0.2896095812320709,
      "logits/chosen": -0.5985989570617676,
      "logits/rejected": -1.0858031511306763,
      "logps/chosen": -1.4781461954116821,
      "logps/rejected": -2.4372060298919678,
      "loss": 1.5148,
      "nll_loss": 1.4857908487319946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14781461656093597,
      "rewards/margins": 0.09590598195791245,
      "rewards/rejected": -0.24372060596942902,
      "step": 795
    },
    {
      "epoch": 1.2537431048069345,
      "grad_norm": 0.2166922688484192,
      "learning_rate": 1.4757277393540598e-06,
      "log_odds_chosen": 1.1573015451431274,
      "log_odds_ratio": -0.2887554168701172,
      "logits/chosen": -0.5447591543197632,
      "logits/rejected": -0.9925971031188965,
      "logps/chosen": -1.4548439979553223,
      "logps/rejected": -2.441218614578247,
      "loss": 1.5007,
      "nll_loss": 1.4717755317687988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14548440277576447,
      "rewards/margins": 0.09863746166229248,
      "rewards/rejected": -0.24412187933921814,
      "step": 796
    },
    {
      "epoch": 1.2553191489361701,
      "grad_norm": 0.21161238849163055,
      "learning_rate": 1.4704248549481946e-06,
      "log_odds_chosen": 1.032838225364685,
      "log_odds_ratio": -0.31353726983070374,
      "logits/chosen": -0.563746988773346,
      "logits/rejected": -1.3283902406692505,
      "logps/chosen": -1.521043300628662,
      "logps/rejected": -2.40523624420166,
      "loss": 1.5501,
      "nll_loss": 1.518709659576416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15210433304309845,
      "rewards/margins": 0.08841929584741592,
      "rewards/rejected": -0.24052362143993378,
      "step": 797
    },
    {
      "epoch": 1.2568951930654058,
      "grad_norm": 0.20102782547473907,
      "learning_rate": 1.4651259712296063e-06,
      "log_odds_chosen": 1.2881131172180176,
      "log_odds_ratio": -0.24913185834884644,
      "logits/chosen": -0.4815088212490082,
      "logits/rejected": -1.1829988956451416,
      "logps/chosen": -1.4943760633468628,
      "logps/rejected": -2.603912591934204,
      "loss": 1.5203,
      "nll_loss": 1.4953885078430176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14943762123584747,
      "rewards/margins": 0.1109536662697792,
      "rewards/rejected": -0.2603912651538849,
      "step": 798
    },
    {
      "epoch": 1.2584712371946414,
      "grad_norm": 0.20646706223487854,
      "learning_rate": 1.45983112822883e-06,
      "log_odds_chosen": 1.087032437324524,
      "log_odds_ratio": -0.2961515188217163,
      "logits/chosen": -0.4692777991294861,
      "logits/rejected": -1.0825268030166626,
      "logps/chosen": -1.4173860549926758,
      "logps/rejected": -2.331252336502075,
      "loss": 1.458,
      "nll_loss": 1.428360939025879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14173860847949982,
      "rewards/margins": 0.09138663858175278,
      "rewards/rejected": -0.2331252545118332,
      "step": 799
    },
    {
      "epoch": 1.260047281323877,
      "grad_norm": 0.22476577758789062,
      "learning_rate": 1.4545403659458756e-06,
      "log_odds_chosen": 1.1412038803100586,
      "log_odds_ratio": -0.28584128618240356,
      "logits/chosen": -0.5733813643455505,
      "logits/rejected": -1.0709235668182373,
      "logps/chosen": -1.4297798871994019,
      "logps/rejected": -2.3951807022094727,
      "loss": 1.4689,
      "nll_loss": 1.4403157234191895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1429779976606369,
      "rewards/margins": 0.09654007852077484,
      "rewards/rejected": -0.23951807618141174,
      "step": 800
    },
    {
      "epoch": 1.2616233254531126,
      "grad_norm": 0.21060988306999207,
      "learning_rate": 1.4492537243499253e-06,
      "log_odds_chosen": 1.2021305561065674,
      "log_odds_ratio": -0.26847487688064575,
      "logits/chosen": -0.5819799304008484,
      "logits/rejected": -1.1289535760879517,
      "logps/chosen": -1.4191426038742065,
      "logps/rejected": -2.429500102996826,
      "loss": 1.4414,
      "nll_loss": 1.4145054817199707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419142633676529,
      "rewards/margins": 0.1010357216000557,
      "rewards/rejected": -0.24294999241828918,
      "step": 801
    },
    {
      "epoch": 1.2631993695823482,
      "grad_norm": 0.20302851498126984,
      "learning_rate": 1.443971243379031e-06,
      "log_odds_chosen": 1.2078783512115479,
      "log_odds_ratio": -0.27533021569252014,
      "logits/chosen": -0.48817336559295654,
      "logits/rejected": -1.1431310176849365,
      "logps/chosen": -1.4538004398345947,
      "logps/rejected": -2.480804681777954,
      "loss": 1.4824,
      "nll_loss": 1.4548829793930054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14538003504276276,
      "rewards/margins": 0.10270043462514877,
      "rewards/rejected": -0.24808049201965332,
      "step": 802
    },
    {
      "epoch": 1.2647754137115839,
      "grad_norm": 0.21020014584064484,
      "learning_rate": 1.4386929629398144e-06,
      "log_odds_chosen": 1.3316316604614258,
      "log_odds_ratio": -0.23976022005081177,
      "logits/chosen": -0.548709511756897,
      "logits/rejected": -1.29503333568573,
      "logps/chosen": -1.4874266386032104,
      "logps/rejected": -2.6369125843048096,
      "loss": 1.5128,
      "nll_loss": 1.4887990951538086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1487426608800888,
      "rewards/margins": 0.11494859308004379,
      "rewards/rejected": -0.263691246509552,
      "step": 803
    },
    {
      "epoch": 1.2663514578408195,
      "grad_norm": 0.20007310807704926,
      "learning_rate": 1.4334189229071614e-06,
      "log_odds_chosen": 1.1232373714447021,
      "log_odds_ratio": -0.29498857259750366,
      "logits/chosen": -0.5688536763191223,
      "logits/rejected": -1.1018214225769043,
      "logps/chosen": -1.3822721242904663,
      "logps/rejected": -2.3245296478271484,
      "loss": 1.4177,
      "nll_loss": 1.3881547451019287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13822722434997559,
      "rewards/margins": 0.09422574192285538,
      "rewards/rejected": -0.23245295882225037,
      "step": 804
    },
    {
      "epoch": 1.267927501970055,
      "grad_norm": 0.23340581357479095,
      "learning_rate": 1.4281491631239263e-06,
      "log_odds_chosen": 0.962167501449585,
      "log_odds_ratio": -0.3313472867012024,
      "logits/chosen": -0.601787269115448,
      "logits/rejected": -1.1337599754333496,
      "logps/chosen": -1.5169259309768677,
      "logps/rejected": -2.3363037109375,
      "loss": 1.5379,
      "nll_loss": 1.504810094833374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15169258415699005,
      "rewards/margins": 0.08193778246641159,
      "rewards/rejected": -0.23363037407398224,
      "step": 805
    },
    {
      "epoch": 1.2695035460992907,
      "grad_norm": 0.2152467519044876,
      "learning_rate": 1.4228837234006272e-06,
      "log_odds_chosen": 1.2895896434783936,
      "log_odds_ratio": -0.2514878809452057,
      "logits/chosen": -0.6019549369812012,
      "logits/rejected": -1.1357216835021973,
      "logps/chosen": -1.3879879713058472,
      "logps/rejected": -2.482874870300293,
      "loss": 1.4449,
      "nll_loss": 1.4197907447814941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1387988030910492,
      "rewards/margins": 0.10948868840932846,
      "rewards/rejected": -0.24828746914863586,
      "step": 806
    },
    {
      "epoch": 1.2710795902285263,
      "grad_norm": 0.212411567568779,
      "learning_rate": 1.4176226435151462e-06,
      "log_odds_chosen": 1.1229116916656494,
      "log_odds_ratio": -0.29439884424209595,
      "logits/chosen": -0.5744314193725586,
      "logits/rejected": -1.3252172470092773,
      "logps/chosen": -1.4892692565917969,
      "logps/rejected": -2.4520978927612305,
      "loss": 1.5057,
      "nll_loss": 1.4762930870056152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14892691373825073,
      "rewards/margins": 0.09628288447856903,
      "rewards/rejected": -0.24520979821681976,
      "step": 807
    },
    {
      "epoch": 1.272655634357762,
      "grad_norm": 0.20049603283405304,
      "learning_rate": 1.4123659632124298e-06,
      "log_odds_chosen": 1.2489851713180542,
      "log_odds_ratio": -0.25868484377861023,
      "logits/chosen": -0.5995192527770996,
      "logits/rejected": -1.2314378023147583,
      "logps/chosen": -1.4515783786773682,
      "logps/rejected": -2.518193483352661,
      "loss": 1.4879,
      "nll_loss": 1.462070107460022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1451578438282013,
      "rewards/margins": 0.10666151344776154,
      "rewards/rejected": -0.25181934237480164,
      "step": 808
    },
    {
      "epoch": 1.2742316784869976,
      "grad_norm": 0.20925989747047424,
      "learning_rate": 1.4071137222041852e-06,
      "log_odds_chosen": 1.2564417123794556,
      "log_odds_ratio": -0.258678138256073,
      "logits/chosen": -0.48692595958709717,
      "logits/rejected": -1.3304721117019653,
      "logps/chosen": -1.49967360496521,
      "logps/rejected": -2.581350803375244,
      "loss": 1.5262,
      "nll_loss": 1.5003352165222168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14996737241744995,
      "rewards/margins": 0.10816771537065506,
      "rewards/rejected": -0.2581350803375244,
      "step": 809
    },
    {
      "epoch": 1.2758077226162332,
      "grad_norm": 0.20463845133781433,
      "learning_rate": 1.4018659601685857e-06,
      "log_odds_chosen": 1.1122080087661743,
      "log_odds_ratio": -0.2908812165260315,
      "logits/chosen": -0.5133985280990601,
      "logits/rejected": -1.217057466506958,
      "logps/chosen": -1.4896631240844727,
      "logps/rejected": -2.440269708633423,
      "loss": 1.5158,
      "nll_loss": 1.486702561378479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14896632730960846,
      "rewards/margins": 0.09506067633628845,
      "rewards/rejected": -0.24402698874473572,
      "step": 810
    },
    {
      "epoch": 1.2773837667454688,
      "grad_norm": 0.24326170980930328,
      "learning_rate": 1.3966227167499667e-06,
      "log_odds_chosen": 1.093398928642273,
      "log_odds_ratio": -0.29335302114486694,
      "logits/chosen": -0.595731794834137,
      "logits/rejected": -1.1054972410202026,
      "logps/chosen": -1.529282569885254,
      "logps/rejected": -2.465944290161133,
      "loss": 1.5519,
      "nll_loss": 1.5225212574005127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15292824804782867,
      "rewards/margins": 0.0936662033200264,
      "rewards/rejected": -0.24659444391727448,
      "step": 811
    },
    {
      "epoch": 1.2789598108747045,
      "grad_norm": 0.20511634647846222,
      "learning_rate": 1.3913840315585277e-06,
      "log_odds_chosen": 1.046661615371704,
      "log_odds_ratio": -0.30168384313583374,
      "logits/chosen": -0.5462578535079956,
      "logits/rejected": -1.231189489364624,
      "logps/chosen": -1.4538002014160156,
      "logps/rejected": -2.3359975814819336,
      "loss": 1.4902,
      "nll_loss": 1.4600262641906738,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14538003504276276,
      "rewards/margins": 0.08821971714496613,
      "rewards/rejected": -0.2335997223854065,
      "step": 812
    },
    {
      "epoch": 1.28053585500394,
      "grad_norm": 0.23104678094387054,
      "learning_rate": 1.3861499441700337e-06,
      "log_odds_chosen": 1.3278257846832275,
      "log_odds_ratio": -0.24389421939849854,
      "logits/chosen": -0.6784459948539734,
      "logits/rejected": -1.2904289960861206,
      "logps/chosen": -1.5383292436599731,
      "logps/rejected": -2.6958930492401123,
      "loss": 1.5637,
      "nll_loss": 1.5392690896987915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15383292734622955,
      "rewards/margins": 0.11575637012720108,
      "rewards/rejected": -0.26958930492401123,
      "step": 813
    },
    {
      "epoch": 1.2821118991331757,
      "grad_norm": 0.22023062407970428,
      "learning_rate": 1.3809204941255144e-06,
      "log_odds_chosen": 1.3169293403625488,
      "log_odds_ratio": -0.24366596341133118,
      "logits/chosen": -0.6053946018218994,
      "logits/rejected": -1.286303997039795,
      "logps/chosen": -1.4223405122756958,
      "logps/rejected": -2.544611930847168,
      "loss": 1.4562,
      "nll_loss": 1.4318702220916748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14223404228687286,
      "rewards/margins": 0.11222714185714722,
      "rewards/rejected": -0.2544611990451813,
      "step": 814
    },
    {
      "epoch": 1.2836879432624113,
      "grad_norm": 0.20418378710746765,
      "learning_rate": 1.3756957209309667e-06,
      "log_odds_chosen": 1.3881272077560425,
      "log_odds_ratio": -0.23305433988571167,
      "logits/chosen": -0.5927729606628418,
      "logits/rejected": -1.3652499914169312,
      "logps/chosen": -1.4632982015609741,
      "logps/rejected": -2.6590166091918945,
      "loss": 1.4966,
      "nll_loss": 1.4733445644378662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1463298201560974,
      "rewards/margins": 0.11957187950611115,
      "rewards/rejected": -0.26590168476104736,
      "step": 815
    },
    {
      "epoch": 1.285263987391647,
      "grad_norm": 0.24232569336891174,
      "learning_rate": 1.3704756640570575e-06,
      "log_odds_chosen": 1.4710278511047363,
      "log_odds_ratio": -0.2260030061006546,
      "logits/chosen": -0.6301259994506836,
      "logits/rejected": -1.2269721031188965,
      "logps/chosen": -1.429699420928955,
      "logps/rejected": -2.697316884994507,
      "loss": 1.4622,
      "nll_loss": 1.4395986795425415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14296993613243103,
      "rewards/margins": 0.1267617642879486,
      "rewards/rejected": -0.26973170042037964,
      "step": 816
    },
    {
      "epoch": 1.2868400315208826,
      "grad_norm": 0.21158339083194733,
      "learning_rate": 1.3652603629388224e-06,
      "log_odds_chosen": 1.2720859050750732,
      "log_odds_ratio": -0.2689078152179718,
      "logits/chosen": -0.5594010949134827,
      "logits/rejected": -1.2308145761489868,
      "logps/chosen": -1.3984894752502441,
      "logps/rejected": -2.4730050563812256,
      "loss": 1.4286,
      "nll_loss": 1.401741623878479,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13984894752502441,
      "rewards/margins": 0.10745155811309814,
      "rewards/rejected": -0.24730050563812256,
      "step": 817
    },
    {
      "epoch": 1.2884160756501182,
      "grad_norm": 0.22694048285484314,
      "learning_rate": 1.3600498569753715e-06,
      "log_odds_chosen": 1.0944557189941406,
      "log_odds_ratio": -0.2901502549648285,
      "logits/chosen": -0.4181465804576874,
      "logits/rejected": -1.0776933431625366,
      "logps/chosen": -1.4491682052612305,
      "logps/rejected": -2.3710029125213623,
      "loss": 1.4898,
      "nll_loss": 1.4607493877410889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.144916832447052,
      "rewards/margins": 0.09218346327543259,
      "rewards/rejected": -0.23710030317306519,
      "step": 818
    },
    {
      "epoch": 1.2899921197793538,
      "grad_norm": 0.21990421414375305,
      "learning_rate": 1.3548441855295872e-06,
      "log_odds_chosen": 1.1117238998413086,
      "log_odds_ratio": -0.28862881660461426,
      "logits/chosen": -0.5629165172576904,
      "logits/rejected": -1.3337757587432861,
      "logps/chosen": -1.5235588550567627,
      "logps/rejected": -2.476930856704712,
      "loss": 1.5525,
      "nll_loss": 1.5236594676971436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15235589444637299,
      "rewards/margins": 0.09533717483282089,
      "rewards/rejected": -0.24769306182861328,
      "step": 819
    },
    {
      "epoch": 1.2915681639085894,
      "grad_norm": 0.22807610034942627,
      "learning_rate": 1.3496433879278315e-06,
      "log_odds_chosen": 1.1873540878295898,
      "log_odds_ratio": -0.28461799025535583,
      "logits/chosen": -0.5434601306915283,
      "logits/rejected": -1.245071291923523,
      "logps/chosen": -1.5645902156829834,
      "logps/rejected": -2.598994016647339,
      "loss": 1.5815,
      "nll_loss": 1.5530296564102173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1564590185880661,
      "rewards/margins": 0.10344040393829346,
      "rewards/rejected": -0.25989940762519836,
      "step": 820
    },
    {
      "epoch": 1.293144208037825,
      "grad_norm": 0.22706064581871033,
      "learning_rate": 1.3444475034596463e-06,
      "log_odds_chosen": 1.3518409729003906,
      "log_odds_ratio": -0.23564840853214264,
      "logits/chosen": -0.6026198863983154,
      "logits/rejected": -1.2712290287017822,
      "logps/chosen": -1.4420205354690552,
      "logps/rejected": -2.6035850048065186,
      "loss": 1.469,
      "nll_loss": 1.445432424545288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1442020684480667,
      "rewards/margins": 0.1161564290523529,
      "rewards/rejected": -0.2603584825992584,
      "step": 821
    },
    {
      "epoch": 1.2947202521670607,
      "grad_norm": 0.2116324007511139,
      "learning_rate": 1.3392565713774575e-06,
      "log_odds_chosen": 1.4289965629577637,
      "log_odds_ratio": -0.21957829594612122,
      "logits/chosen": -0.6289047002792358,
      "logits/rejected": -1.3347207307815552,
      "logps/chosen": -1.4517041444778442,
      "logps/rejected": -2.683992385864258,
      "loss": 1.4899,
      "nll_loss": 1.467977523803711,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1451704204082489,
      "rewards/margins": 0.12322881817817688,
      "rewards/rejected": -0.2683992385864258,
      "step": 822
    },
    {
      "epoch": 1.2962962962962963,
      "grad_norm": 0.2263702005147934,
      "learning_rate": 1.3340706308962763e-06,
      "log_odds_chosen": 1.1946312189102173,
      "log_odds_ratio": -0.27804818749427795,
      "logits/chosen": -0.5862393379211426,
      "logits/rejected": -1.0531774759292603,
      "logps/chosen": -1.417628288269043,
      "logps/rejected": -2.4260494709014893,
      "loss": 1.4552,
      "nll_loss": 1.4273781776428223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14176280796527863,
      "rewards/margins": 0.10084213316440582,
      "rewards/rejected": -0.24260495603084564,
      "step": 823
    },
    {
      "epoch": 1.297872340425532,
      "grad_norm": 0.22301039099693298,
      "learning_rate": 1.3288897211934066e-06,
      "log_odds_chosen": 1.1970597505569458,
      "log_odds_ratio": -0.2772637903690338,
      "logits/chosen": -0.5759676098823547,
      "logits/rejected": -0.9729692935943604,
      "logps/chosen": -1.4178369045257568,
      "logps/rejected": -2.4307847023010254,
      "loss": 1.4478,
      "nll_loss": 1.4201104640960693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1417836844921112,
      "rewards/margins": 0.10129477828741074,
      "rewards/rejected": -0.24307844042778015,
      "step": 824
    },
    {
      "epoch": 1.2994483845547675,
      "grad_norm": 0.20512492954730988,
      "learning_rate": 1.323713881408147e-06,
      "log_odds_chosen": 1.0681990385055542,
      "log_odds_ratio": -0.30326151847839355,
      "logits/chosen": -0.5601771473884583,
      "logits/rejected": -1.249066710472107,
      "logps/chosen": -1.4691258668899536,
      "logps/rejected": -2.3761954307556152,
      "loss": 1.4973,
      "nll_loss": 1.4669374227523804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1469125896692276,
      "rewards/margins": 0.09070694446563721,
      "rewards/rejected": -0.2376195341348648,
      "step": 825
    },
    {
      "epoch": 1.3010244286840031,
      "grad_norm": 0.20110926032066345,
      "learning_rate": 1.3185431506414943e-06,
      "log_odds_chosen": 1.210633635520935,
      "log_odds_ratio": -0.268954336643219,
      "logits/chosen": -0.5951650738716125,
      "logits/rejected": -1.297168493270874,
      "logps/chosen": -1.3609910011291504,
      "logps/rejected": -2.3773062229156494,
      "loss": 1.3962,
      "nll_loss": 1.3692840337753296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13609910011291504,
      "rewards/margins": 0.1016315370798111,
      "rewards/rejected": -0.23773062229156494,
      "step": 826
    },
    {
      "epoch": 1.3026004728132388,
      "grad_norm": 0.22214418649673462,
      "learning_rate": 1.313377567955851e-06,
      "log_odds_chosen": 1.0050333738327026,
      "log_odds_ratio": -0.32804763317108154,
      "logits/chosen": -0.5676519274711609,
      "logits/rejected": -1.0883657932281494,
      "logps/chosen": -1.4907535314559937,
      "logps/rejected": -2.346625566482544,
      "loss": 1.5171,
      "nll_loss": 1.48434317111969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14907535910606384,
      "rewards/margins": 0.08558718860149384,
      "rewards/rejected": -0.23466253280639648,
      "step": 827
    },
    {
      "epoch": 1.3041765169424744,
      "grad_norm": 0.2209463268518448,
      "learning_rate": 1.3082171723747257e-06,
      "log_odds_chosen": 1.0857608318328857,
      "log_odds_ratio": -0.30399033427238464,
      "logits/chosen": -0.6888466477394104,
      "logits/rejected": -1.1576621532440186,
      "logps/chosen": -1.4456183910369873,
      "logps/rejected": -2.3623292446136475,
      "loss": 1.4783,
      "nll_loss": 1.4479012489318848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14456184208393097,
      "rewards/margins": 0.09167105704545975,
      "rewards/rejected": -0.2362329214811325,
      "step": 828
    },
    {
      "epoch": 1.30575256107171,
      "grad_norm": 0.24324966967105865,
      "learning_rate": 1.3030620028824424e-06,
      "log_odds_chosen": 1.057613492012024,
      "log_odds_ratio": -0.3044401705265045,
      "logits/chosen": -0.5883345603942871,
      "logits/rejected": -1.1093482971191406,
      "logps/chosen": -1.4602357149124146,
      "logps/rejected": -2.350797653198242,
      "loss": 1.4986,
      "nll_loss": 1.468125581741333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14602357149124146,
      "rewards/margins": 0.08905617892742157,
      "rewards/rejected": -0.23507975041866302,
      "step": 829
    },
    {
      "epoch": 1.3073286052009456,
      "grad_norm": 0.22348783910274506,
      "learning_rate": 1.2979120984238449e-06,
      "log_odds_chosen": 1.1074544191360474,
      "log_odds_ratio": -0.2927136719226837,
      "logits/chosen": -0.6068792939186096,
      "logits/rejected": -1.1124392747879028,
      "logps/chosen": -1.5066428184509277,
      "logps/rejected": -2.452918529510498,
      "loss": 1.5358,
      "nll_loss": 1.5065687894821167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1506642997264862,
      "rewards/margins": 0.09462757408618927,
      "rewards/rejected": -0.24529187381267548,
      "step": 830
    },
    {
      "epoch": 1.3089046493301812,
      "grad_norm": 0.22100266814231873,
      "learning_rate": 1.2927674979040009e-06,
      "log_odds_chosen": 1.140690565109253,
      "log_odds_ratio": -0.2816528379917145,
      "logits/chosen": -0.49456876516342163,
      "logits/rejected": -1.06095290184021,
      "logps/chosen": -1.3918594121932983,
      "logps/rejected": -2.3457107543945312,
      "loss": 1.4448,
      "nll_loss": 1.4166531562805176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13918595016002655,
      "rewards/margins": 0.09538515657186508,
      "rewards/rejected": -0.23457109928131104,
      "step": 831
    },
    {
      "epoch": 1.3104806934594169,
      "grad_norm": 0.19394604861736298,
      "learning_rate": 1.2876282401879106e-06,
      "log_odds_chosen": 1.3044791221618652,
      "log_odds_ratio": -0.24427001178264618,
      "logits/chosen": -0.5360194444656372,
      "logits/rejected": -1.1644905805587769,
      "logps/chosen": -1.4237332344055176,
      "logps/rejected": -2.536370277404785,
      "loss": 1.4615,
      "nll_loss": 1.4370614290237427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14237332344055176,
      "rewards/margins": 0.1112636849284172,
      "rewards/rejected": -0.25363701581954956,
      "step": 832
    },
    {
      "epoch": 1.3120567375886525,
      "grad_norm": 0.21855831146240234,
      "learning_rate": 1.2824943641002115e-06,
      "log_odds_chosen": 1.3061401844024658,
      "log_odds_ratio": -0.24521617591381073,
      "logits/chosen": -0.47566258907318115,
      "logits/rejected": -1.286742091178894,
      "logps/chosen": -1.4216862916946411,
      "logps/rejected": -2.5359673500061035,
      "loss": 1.4581,
      "nll_loss": 1.4335308074951172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14216862618923187,
      "rewards/margins": 0.11142810434103012,
      "rewards/rejected": -0.2535967230796814,
      "step": 833
    },
    {
      "epoch": 1.313632781717888,
      "grad_norm": 0.24970686435699463,
      "learning_rate": 1.2773659084248845e-06,
      "log_odds_chosen": 1.464369773864746,
      "log_odds_ratio": -0.2319881021976471,
      "logits/chosen": -0.6934940218925476,
      "logits/rejected": -1.3603639602661133,
      "logps/chosen": -1.3739126920700073,
      "logps/rejected": -2.630636215209961,
      "loss": 1.4053,
      "nll_loss": 1.382062315940857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13739126920700073,
      "rewards/margins": 0.1256723701953888,
      "rewards/rejected": -0.2630636394023895,
      "step": 834
    },
    {
      "epoch": 1.3152088258471237,
      "grad_norm": 0.20905321836471558,
      "learning_rate": 1.2722429119049632e-06,
      "log_odds_chosen": 1.0040830373764038,
      "log_odds_ratio": -0.3281542956829071,
      "logits/chosen": -0.49832919239997864,
      "logits/rejected": -1.0942928791046143,
      "logps/chosen": -1.5023218393325806,
      "logps/rejected": -2.358431339263916,
      "loss": 1.5414,
      "nll_loss": 1.50859534740448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15023218095302582,
      "rewards/margins": 0.08561095595359802,
      "rewards/rejected": -0.23584315180778503,
      "step": 835
    },
    {
      "epoch": 1.3167848699763594,
      "grad_norm": 0.21633084118366241,
      "learning_rate": 1.267125413242239e-06,
      "log_odds_chosen": 1.2335046529769897,
      "log_odds_ratio": -0.2672288119792938,
      "logits/chosen": -0.6043000817298889,
      "logits/rejected": -1.1264828443527222,
      "logps/chosen": -1.3793232440948486,
      "logps/rejected": -2.418633222579956,
      "loss": 1.4248,
      "nll_loss": 1.3980716466903687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13793234527111053,
      "rewards/margins": 0.1039309874176979,
      "rewards/rejected": -0.24186332523822784,
      "step": 836
    },
    {
      "epoch": 1.318360914105595,
      "grad_norm": 0.2366338074207306,
      "learning_rate": 1.2620134510969719e-06,
      "log_odds_chosen": 1.1477605104446411,
      "log_odds_ratio": -0.2881053686141968,
      "logits/chosen": -0.6598471403121948,
      "logits/rejected": -1.037442684173584,
      "logps/chosen": -1.3703231811523438,
      "logps/rejected": -2.3316545486450195,
      "loss": 1.4169,
      "nll_loss": 1.3880966901779175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13703233003616333,
      "rewards/margins": 0.09613313525915146,
      "rewards/rejected": -0.2331654578447342,
      "step": 837
    },
    {
      "epoch": 1.3199369582348306,
      "grad_norm": 0.24140475690364838,
      "learning_rate": 1.2569070640875912e-06,
      "log_odds_chosen": 1.4002710580825806,
      "log_odds_ratio": -0.25709980726242065,
      "logits/chosen": -0.5062721967697144,
      "logits/rejected": -1.1856307983398438,
      "logps/chosen": -1.3519026041030884,
      "logps/rejected": -2.510430335998535,
      "loss": 1.4002,
      "nll_loss": 1.374483585357666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13519026339054108,
      "rewards/margins": 0.11585275828838348,
      "rewards/rejected": -0.25104305148124695,
      "step": 838
    },
    {
      "epoch": 1.3215130023640662,
      "grad_norm": 0.23529188334941864,
      "learning_rate": 1.2518062907904136e-06,
      "log_odds_chosen": 1.3921624422073364,
      "log_odds_ratio": -0.239127978682518,
      "logits/chosen": -0.6545761227607727,
      "logits/rejected": -1.2730376720428467,
      "logps/chosen": -1.419264316558838,
      "logps/rejected": -2.61517596244812,
      "loss": 1.4544,
      "nll_loss": 1.4305319786071777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14192643761634827,
      "rewards/margins": 0.11959117650985718,
      "rewards/rejected": -0.26151761412620544,
      "step": 839
    },
    {
      "epoch": 1.3230890464933018,
      "grad_norm": 0.21813492476940155,
      "learning_rate": 1.2467111697393446e-06,
      "log_odds_chosen": 1.2054600715637207,
      "log_odds_ratio": -0.26900941133499146,
      "logits/chosen": -0.5609135031700134,
      "logits/rejected": -1.3460763692855835,
      "logps/chosen": -1.521897315979004,
      "logps/rejected": -2.5616214275360107,
      "loss": 1.5507,
      "nll_loss": 1.5237724781036377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1521897315979004,
      "rewards/margins": 0.1039724051952362,
      "rewards/rejected": -0.2561621367931366,
      "step": 840
    },
    {
      "epoch": 1.3246650906225375,
      "grad_norm": 0.22650055587291718,
      "learning_rate": 1.2416217394255905e-06,
      "log_odds_chosen": 1.0618315935134888,
      "log_odds_ratio": -0.30074170231819153,
      "logits/chosen": -0.5073498487472534,
      "logits/rejected": -0.9713736772537231,
      "logps/chosen": -1.3592252731323242,
      "logps/rejected": -2.237431764602661,
      "loss": 1.4268,
      "nll_loss": 1.3967235088348389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13592253625392914,
      "rewards/margins": 0.08782064914703369,
      "rewards/rejected": -0.22374318540096283,
      "step": 841
    },
    {
      "epoch": 1.326241134751773,
      "grad_norm": 0.23454034328460693,
      "learning_rate": 1.2365380382973669e-06,
      "log_odds_chosen": 0.9980802536010742,
      "log_odds_ratio": -0.322952002286911,
      "logits/chosen": -0.5843859910964966,
      "logits/rejected": -0.9952538013458252,
      "logps/chosen": -1.5202569961547852,
      "logps/rejected": -2.3726718425750732,
      "loss": 1.5416,
      "nll_loss": 1.5093423128128052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15202569961547852,
      "rewards/margins": 0.08524147421121597,
      "rewards/rejected": -0.23726718127727509,
      "step": 842
    },
    {
      "epoch": 1.3278171788810087,
      "grad_norm": 0.21943433582782745,
      "learning_rate": 1.2314601047596061e-06,
      "log_odds_chosen": 1.416329026222229,
      "log_odds_ratio": -0.2220940887928009,
      "logits/chosen": -0.532533586025238,
      "logits/rejected": -1.2685941457748413,
      "logps/chosen": -1.4418177604675293,
      "logps/rejected": -2.6611852645874023,
      "loss": 1.4614,
      "nll_loss": 1.4391814470291138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14418178796768188,
      "rewards/margins": 0.12193672358989716,
      "rewards/rejected": -0.26611852645874023,
      "step": 843
    },
    {
      "epoch": 1.3293932230102443,
      "grad_norm": 0.20646244287490845,
      "learning_rate": 1.2263879771736713e-06,
      "log_odds_chosen": 1.0643417835235596,
      "log_odds_ratio": -0.30553188920021057,
      "logits/chosen": -0.618087649345398,
      "logits/rejected": -1.2652965784072876,
      "logps/chosen": -1.4606696367263794,
      "logps/rejected": -2.3593645095825195,
      "loss": 1.496,
      "nll_loss": 1.465423583984375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14606696367263794,
      "rewards/margins": 0.08986948430538177,
      "rewards/rejected": -0.2359364628791809,
      "step": 844
    },
    {
      "epoch": 1.33096926713948,
      "grad_norm": 0.2068682163953781,
      "learning_rate": 1.2213216938570642e-06,
      "log_odds_chosen": 1.230912685394287,
      "log_odds_ratio": -0.28044185042381287,
      "logits/chosen": -0.5915220975875854,
      "logits/rejected": -1.1778411865234375,
      "logps/chosen": -1.3839596509933472,
      "logps/rejected": -2.4171080589294434,
      "loss": 1.4331,
      "nll_loss": 1.4050248861312866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13839596509933472,
      "rewards/margins": 0.1033148318529129,
      "rewards/rejected": -0.2417108118534088,
      "step": 845
    },
    {
      "epoch": 1.3325453112687156,
      "grad_norm": 0.21691980957984924,
      "learning_rate": 1.2162612930831354e-06,
      "log_odds_chosen": 1.5139830112457275,
      "log_odds_ratio": -0.21375960111618042,
      "logits/chosen": -0.6485769748687744,
      "logits/rejected": -1.2731252908706665,
      "logps/chosen": -1.2814886569976807,
      "logps/rejected": -2.5519373416900635,
      "loss": 1.3287,
      "nll_loss": 1.3073444366455078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1281488686800003,
      "rewards/margins": 0.12704485654830933,
      "rewards/rejected": -0.2551937401294708,
      "step": 846
    },
    {
      "epoch": 1.3341213553979512,
      "grad_norm": 0.2094448357820511,
      "learning_rate": 1.2112068130807949e-06,
      "log_odds_chosen": 1.5493706464767456,
      "log_odds_ratio": -0.20080287754535675,
      "logits/chosen": -0.5958383083343506,
      "logits/rejected": -1.3628191947937012,
      "logps/chosen": -1.4311600923538208,
      "logps/rejected": -2.7756786346435547,
      "loss": 1.4462,
      "nll_loss": 1.4260706901550293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14311599731445312,
      "rewards/margins": 0.13445188105106354,
      "rewards/rejected": -0.27756786346435547,
      "step": 847
    },
    {
      "epoch": 1.3356973995271868,
      "grad_norm": 0.2263561338186264,
      "learning_rate": 1.206158292034226e-06,
      "log_odds_chosen": 1.2998851537704468,
      "log_odds_ratio": -0.24632519483566284,
      "logits/chosen": -0.6226043701171875,
      "logits/rejected": -1.2622638940811157,
      "logps/chosen": -1.4475815296173096,
      "logps/rejected": -2.5599365234375,
      "loss": 1.4792,
      "nll_loss": 1.4545679092407227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14475813508033752,
      "rewards/margins": 0.11123552173376083,
      "rewards/rejected": -0.25599363446235657,
      "step": 848
    },
    {
      "epoch": 1.3372734436564224,
      "grad_norm": 0.2060093730688095,
      "learning_rate": 1.2011157680825928e-06,
      "log_odds_chosen": 1.391319751739502,
      "log_odds_ratio": -0.2346474677324295,
      "logits/chosen": -0.5227964520454407,
      "logits/rejected": -1.3248584270477295,
      "logps/chosen": -1.377244234085083,
      "logps/rejected": -2.563082695007324,
      "loss": 1.415,
      "nll_loss": 1.3915141820907593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13772442936897278,
      "rewards/margins": 0.11858386546373367,
      "rewards/rejected": -0.25630828738212585,
      "step": 849
    },
    {
      "epoch": 1.338849487785658,
      "grad_norm": 0.234042689204216,
      "learning_rate": 1.1960792793197553e-06,
      "log_odds_chosen": 1.280304193496704,
      "log_odds_ratio": -0.26100626587867737,
      "logits/chosen": -0.6842824220657349,
      "logits/rejected": -1.2155097723007202,
      "logps/chosen": -1.4313563108444214,
      "logps/rejected": -2.523120403289795,
      "loss": 1.4629,
      "nll_loss": 1.436840295791626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14313562214374542,
      "rewards/margins": 0.10917641967535019,
      "rewards/rejected": -0.252312034368515,
      "step": 850
    },
    {
      "epoch": 1.3404255319148937,
      "grad_norm": 0.22375516593456268,
      "learning_rate": 1.1910488637939824e-06,
      "log_odds_chosen": 1.4752148389816284,
      "log_odds_ratio": -0.20883040130138397,
      "logits/chosen": -0.7203405499458313,
      "logits/rejected": -1.3447585105895996,
      "logps/chosen": -1.4743764400482178,
      "logps/rejected": -2.7555229663848877,
      "loss": 1.4825,
      "nll_loss": 1.4616410732269287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14743764698505402,
      "rewards/margins": 0.12811464071273804,
      "rewards/rejected": -0.27555227279663086,
      "step": 851
    },
    {
      "epoch": 1.3420015760441293,
      "grad_norm": 0.21877829730510712,
      "learning_rate": 1.1860245595076582e-06,
      "log_odds_chosen": 1.0403438806533813,
      "log_odds_ratio": -0.307626336812973,
      "logits/chosen": -0.5076892375946045,
      "logits/rejected": -1.2057814598083496,
      "logps/chosen": -1.509682536125183,
      "logps/rejected": -2.3968148231506348,
      "loss": 1.5203,
      "nll_loss": 1.4895113706588745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1509682536125183,
      "rewards/margins": 0.08871322870254517,
      "rewards/rejected": -0.23968146741390228,
      "step": 852
    },
    {
      "epoch": 1.343577620173365,
      "grad_norm": 0.21908845007419586,
      "learning_rate": 1.1810064044170027e-06,
      "log_odds_chosen": 1.250730037689209,
      "log_odds_ratio": -0.2621658742427826,
      "logits/chosen": -0.6624240875244141,
      "logits/rejected": -1.288458228111267,
      "logps/chosen": -1.393250823020935,
      "logps/rejected": -2.4505105018615723,
      "loss": 1.4314,
      "nll_loss": 1.4051614999771118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1393250823020935,
      "rewards/margins": 0.105725958943367,
      "rewards/rejected": -0.2450510561466217,
      "step": 853
    },
    {
      "epoch": 1.3451536643026005,
      "grad_norm": 0.21907569468021393,
      "learning_rate": 1.1759944364317812e-06,
      "log_odds_chosen": 1.2535991668701172,
      "log_odds_ratio": -0.2625347673892975,
      "logits/chosen": -0.6723836660385132,
      "logits/rejected": -1.2248752117156982,
      "logps/chosen": -1.4535833597183228,
      "logps/rejected": -2.5257718563079834,
      "loss": 1.4786,
      "nll_loss": 1.452339768409729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14535833895206451,
      "rewards/margins": 0.10721883922815323,
      "rewards/rejected": -0.25257718563079834,
      "step": 854
    },
    {
      "epoch": 1.3467297084318361,
      "grad_norm": 0.22589510679244995,
      "learning_rate": 1.1709886934150172e-06,
      "log_odds_chosen": 1.1051852703094482,
      "log_odds_ratio": -0.29447680711746216,
      "logits/chosen": -0.5418897867202759,
      "logits/rejected": -1.1283231973648071,
      "logps/chosen": -1.4746602773666382,
      "logps/rejected": -2.410953998565674,
      "loss": 1.5158,
      "nll_loss": 1.486337423324585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1474660336971283,
      "rewards/margins": 0.093629390001297,
      "rewards/rejected": -0.2410954385995865,
      "step": 855
    },
    {
      "epoch": 1.3483057525610718,
      "grad_norm": 0.243242546916008,
      "learning_rate": 1.1659892131827097e-06,
      "log_odds_chosen": 1.2872827053070068,
      "log_odds_ratio": -0.25760525465011597,
      "logits/chosen": -0.6574603319168091,
      "logits/rejected": -1.3787659406661987,
      "logps/chosen": -1.4968070983886719,
      "logps/rejected": -2.6109044551849365,
      "loss": 1.5189,
      "nll_loss": 1.4931496381759644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14968068897724152,
      "rewards/margins": 0.1114097535610199,
      "rewards/rejected": -0.2610904574394226,
      "step": 856
    },
    {
      "epoch": 1.3498817966903074,
      "grad_norm": 0.20769359171390533,
      "learning_rate": 1.1609960335035423e-06,
      "log_odds_chosen": 1.699277639389038,
      "log_odds_ratio": -0.17834420502185822,
      "logits/chosen": -0.6118935346603394,
      "logits/rejected": -1.3896839618682861,
      "logps/chosen": -1.4819233417510986,
      "logps/rejected": -2.9720335006713867,
      "loss": 1.5046,
      "nll_loss": 1.4868056774139404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14819233119487762,
      "rewards/margins": 0.1490110158920288,
      "rewards/rejected": -0.2972033619880676,
      "step": 857
    },
    {
      "epoch": 1.351457840819543,
      "grad_norm": 0.22415241599082947,
      "learning_rate": 1.1560091920986028e-06,
      "log_odds_chosen": 1.3987443447113037,
      "log_odds_ratio": -0.2257547229528427,
      "logits/chosen": -0.6128131747245789,
      "logits/rejected": -1.2887446880340576,
      "logps/chosen": -1.4026092290878296,
      "logps/rejected": -2.596917152404785,
      "loss": 1.4414,
      "nll_loss": 1.4188055992126465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14026093482971191,
      "rewards/margins": 0.11943075805902481,
      "rewards/rejected": -0.2596917152404785,
      "step": 858
    },
    {
      "epoch": 1.3530338849487786,
      "grad_norm": 0.23100236058235168,
      "learning_rate": 1.1510287266410967e-06,
      "log_odds_chosen": 1.091015100479126,
      "log_odds_ratio": -0.296450674533844,
      "logits/chosen": -0.6262733340263367,
      "logits/rejected": -1.134032130241394,
      "logps/chosen": -1.5171427726745605,
      "logps/rejected": -2.453106641769409,
      "loss": 1.5432,
      "nll_loss": 1.5135858058929443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517142802476883,
      "rewards/margins": 0.09359639883041382,
      "rewards/rejected": -0.2453106939792633,
      "step": 859
    },
    {
      "epoch": 1.3546099290780143,
      "grad_norm": 0.21196019649505615,
      "learning_rate": 1.1460546747560616e-06,
      "log_odds_chosen": 1.4643217325210571,
      "log_odds_ratio": -0.21655550599098206,
      "logits/chosen": -0.6504445672035217,
      "logits/rejected": -1.3764665126800537,
      "logps/chosen": -1.4477097988128662,
      "logps/rejected": -2.7154061794281006,
      "loss": 1.4645,
      "nll_loss": 1.4428132772445679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14477097988128662,
      "rewards/margins": 0.12676966190338135,
      "rewards/rejected": -0.27154064178466797,
      "step": 860
    },
    {
      "epoch": 1.3561859732072499,
      "grad_norm": 0.24296846985816956,
      "learning_rate": 1.1410870740200839e-06,
      "log_odds_chosen": 1.2555168867111206,
      "log_odds_ratio": -0.2696991562843323,
      "logits/chosen": -0.6015902161598206,
      "logits/rejected": -1.2834227085113525,
      "logps/chosen": -1.4975497722625732,
      "logps/rejected": -2.583686590194702,
      "loss": 1.5199,
      "nll_loss": 1.492963194847107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14975497126579285,
      "rewards/margins": 0.10861368477344513,
      "rewards/rejected": -0.25836867094039917,
      "step": 861
    },
    {
      "epoch": 1.3577620173364855,
      "grad_norm": 0.21845929324626923,
      "learning_rate": 1.1361259619610138e-06,
      "log_odds_chosen": 1.3727145195007324,
      "log_odds_ratio": -0.23824596405029297,
      "logits/chosen": -0.6252405047416687,
      "logits/rejected": -1.2933534383773804,
      "logps/chosen": -1.4324400424957275,
      "logps/rejected": -2.6054868698120117,
      "loss": 1.4694,
      "nll_loss": 1.4455652236938477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14324399828910828,
      "rewards/margins": 0.11730466783046722,
      "rewards/rejected": -0.2605486512184143,
      "step": 862
    },
    {
      "epoch": 1.3593380614657211,
      "grad_norm": 0.2358924150466919,
      "learning_rate": 1.1311713760576834e-06,
      "log_odds_chosen": 1.348677158355713,
      "log_odds_ratio": -0.243307426571846,
      "logits/chosen": -0.752122163772583,
      "logits/rejected": -1.0987956523895264,
      "logps/chosen": -1.418363094329834,
      "logps/rejected": -2.568345546722412,
      "loss": 1.4341,
      "nll_loss": 1.4097533226013184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14183631539344788,
      "rewards/margins": 0.11499825865030289,
      "rewards/rejected": -0.25683456659317017,
      "step": 863
    },
    {
      "epoch": 1.3609141055949567,
      "grad_norm": 0.2159270942211151,
      "learning_rate": 1.1262233537396228e-06,
      "log_odds_chosen": 1.532590627670288,
      "log_odds_ratio": -0.21828334033489227,
      "logits/chosen": -0.5851295590400696,
      "logits/rejected": -1.2376445531845093,
      "logps/chosen": -1.3959558010101318,
      "logps/rejected": -2.7141928672790527,
      "loss": 1.4127,
      "nll_loss": 1.3908597230911255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13959558308124542,
      "rewards/margins": 0.13182371854782104,
      "rewards/rejected": -0.27141931653022766,
      "step": 864
    },
    {
      "epoch": 1.3624901497241924,
      "grad_norm": 0.2176884561777115,
      "learning_rate": 1.1212819323867778e-06,
      "log_odds_chosen": 1.3009604215621948,
      "log_odds_ratio": -0.2607001066207886,
      "logits/chosen": -0.6513187885284424,
      "logits/rejected": -1.22568678855896,
      "logps/chosen": -1.3695366382598877,
      "logps/rejected": -2.4761996269226074,
      "loss": 1.4193,
      "nll_loss": 1.3931822776794434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13695365190505981,
      "rewards/margins": 0.11066628992557526,
      "rewards/rejected": -0.24761995673179626,
      "step": 865
    },
    {
      "epoch": 1.364066193853428,
      "grad_norm": 0.21860244870185852,
      "learning_rate": 1.1163471493292267e-06,
      "log_odds_chosen": 1.335142731666565,
      "log_odds_ratio": -0.24506187438964844,
      "logits/chosen": -0.6505783796310425,
      "logits/rejected": -1.235350489616394,
      "logps/chosen": -1.4486974477767944,
      "logps/rejected": -2.594944953918457,
      "loss": 1.4729,
      "nll_loss": 1.448359489440918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14486974477767944,
      "rewards/margins": 0.1146247461438179,
      "rewards/rejected": -0.25949448347091675,
      "step": 866
    },
    {
      "epoch": 1.3656422379826636,
      "grad_norm": 0.2037617564201355,
      "learning_rate": 1.1114190418468972e-06,
      "log_odds_chosen": 1.215273380279541,
      "log_odds_ratio": -0.2734428644180298,
      "logits/chosen": -0.5610312223434448,
      "logits/rejected": -1.2139989137649536,
      "logps/chosen": -1.4694149494171143,
      "logps/rejected": -2.5142338275909424,
      "loss": 1.512,
      "nll_loss": 1.4846316576004028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14694149792194366,
      "rewards/margins": 0.10448186844587326,
      "rewards/rejected": -0.25142335891723633,
      "step": 867
    },
    {
      "epoch": 1.367218282111899,
      "grad_norm": 0.21670474112033844,
      "learning_rate": 1.106497647169288e-06,
      "log_odds_chosen": 1.4382001161575317,
      "log_odds_ratio": -0.21835294365882874,
      "logits/chosen": -0.5780256390571594,
      "logits/rejected": -1.2578712701797485,
      "logps/chosen": -1.3970013856887817,
      "logps/rejected": -2.6271862983703613,
      "loss": 1.4187,
      "nll_loss": 1.3968452215194702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13970011472702026,
      "rewards/margins": 0.12301848828792572,
      "rewards/rejected": -0.2627186179161072,
      "step": 868
    },
    {
      "epoch": 1.3687943262411348,
      "grad_norm": 0.22069762647151947,
      "learning_rate": 1.1015830024751854e-06,
      "log_odds_chosen": 1.2497507333755493,
      "log_odds_ratio": -0.2625420391559601,
      "logits/chosen": -0.7378687262535095,
      "logits/rejected": -1.0852506160736084,
      "logps/chosen": -1.3546242713928223,
      "logps/rejected": -2.4059107303619385,
      "loss": 1.3946,
      "nll_loss": 1.368326187133789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1354624480009079,
      "rewards/margins": 0.10512865334749222,
      "rewards/rejected": -0.2405911087989807,
      "step": 869
    },
    {
      "epoch": 1.3703703703703702,
      "grad_norm": 0.21761703491210938,
      "learning_rate": 1.0966751448923834e-06,
      "log_odds_chosen": 1.5589405298233032,
      "log_odds_ratio": -0.199112206697464,
      "logits/chosen": -0.5823766589164734,
      "logits/rejected": -1.1930749416351318,
      "logps/chosen": -1.358464241027832,
      "logps/rejected": -2.6928327083587646,
      "loss": 1.3927,
      "nll_loss": 1.3728300333023071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13584642112255096,
      "rewards/margins": 0.13343684375286102,
      "rewards/rejected": -0.269283264875412,
      "step": 870
    },
    {
      "epoch": 1.371946414499606,
      "grad_norm": 0.2293926179409027,
      "learning_rate": 1.0917741114974007e-06,
      "log_odds_chosen": 1.2105826139450073,
      "log_odds_ratio": -0.26561376452445984,
      "logits/chosen": -0.5363762378692627,
      "logits/rejected": -1.2512714862823486,
      "logps/chosen": -1.4965007305145264,
      "logps/rejected": -2.5332887172698975,
      "loss": 1.5255,
      "nll_loss": 1.4989145994186401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14965006709098816,
      "rewards/margins": 0.10367881506681442,
      "rewards/rejected": -0.2533288896083832,
      "step": 871
    },
    {
      "epoch": 1.3735224586288415,
      "grad_norm": 0.21842007339000702,
      "learning_rate": 1.0868799393152035e-06,
      "log_odds_chosen": 1.5115584135055542,
      "log_odds_ratio": -0.21881306171417236,
      "logits/chosen": -0.6933940649032593,
      "logits/rejected": -1.3086124658584595,
      "logps/chosen": -1.3789467811584473,
      "logps/rejected": -2.6748805046081543,
      "loss": 1.4119,
      "nll_loss": 1.38998544216156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13789469003677368,
      "rewards/margins": 0.12959334254264832,
      "rewards/rejected": -0.267488032579422,
      "step": 872
    },
    {
      "epoch": 1.3750985027580773,
      "grad_norm": 0.2425728440284729,
      "learning_rate": 1.0819926653189271e-06,
      "log_odds_chosen": 1.2734521627426147,
      "log_odds_ratio": -0.258124440908432,
      "logits/chosen": -0.6365923881530762,
      "logits/rejected": -1.1608169078826904,
      "logps/chosen": -1.4639427661895752,
      "logps/rejected": -2.55708909034729,
      "loss": 1.4961,
      "nll_loss": 1.4702869653701782,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.146394282579422,
      "rewards/margins": 0.10931465029716492,
      "rewards/rejected": -0.2557089328765869,
      "step": 873
    },
    {
      "epoch": 1.3766745468873127,
      "grad_norm": 0.2087407410144806,
      "learning_rate": 1.0771123264295895e-06,
      "log_odds_chosen": 1.2941322326660156,
      "log_odds_ratio": -0.25062721967697144,
      "logits/chosen": -0.6451308131217957,
      "logits/rejected": -1.319624423980713,
      "logps/chosen": -1.359085202217102,
      "logps/rejected": -2.4458811283111572,
      "loss": 1.3948,
      "nll_loss": 1.3696986436843872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13590851426124573,
      "rewards/margins": 0.10867959260940552,
      "rewards/rejected": -0.24458810687065125,
      "step": 874
    },
    {
      "epoch": 1.3782505910165486,
      "grad_norm": 0.20348989963531494,
      "learning_rate": 1.0722389595158215e-06,
      "log_odds_chosen": 1.4972018003463745,
      "log_odds_ratio": -0.21852149069309235,
      "logits/chosen": -0.5357131958007812,
      "logits/rejected": -1.2564542293548584,
      "logps/chosen": -1.347593903541565,
      "logps/rejected": -2.6263251304626465,
      "loss": 1.3883,
      "nll_loss": 1.366422414779663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13475941121578217,
      "rewards/margins": 0.12787312269210815,
      "rewards/rejected": -0.2626325190067291,
      "step": 875
    },
    {
      "epoch": 1.379826635145784,
      "grad_norm": 0.20969262719154358,
      "learning_rate": 1.0673726013935827e-06,
      "log_odds_chosen": 1.3771296739578247,
      "log_odds_ratio": -0.25535959005355835,
      "logits/chosen": -0.5636877417564392,
      "logits/rejected": -1.1472883224487305,
      "logps/chosen": -1.3606841564178467,
      "logps/rejected": -2.5294346809387207,
      "loss": 1.4039,
      "nll_loss": 1.378409504890442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1360684037208557,
      "rewards/margins": 0.116875059902668,
      "rewards/rejected": -0.2529434561729431,
      "step": 876
    },
    {
      "epoch": 1.3814026792750198,
      "grad_norm": 0.21677014231681824,
      "learning_rate": 1.0625132888258833e-06,
      "log_odds_chosen": 1.2514504194259644,
      "log_odds_ratio": -0.2650548219680786,
      "logits/chosen": -0.7082834839820862,
      "logits/rejected": -1.2047538757324219,
      "logps/chosen": -1.430253028869629,
      "logps/rejected": -2.5011048316955566,
      "loss": 1.4603,
      "nll_loss": 1.4338442087173462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14302532374858856,
      "rewards/margins": 0.10708514600992203,
      "rewards/rejected": -0.2501104772090912,
      "step": 877
    },
    {
      "epoch": 1.3829787234042552,
      "grad_norm": 0.22289732098579407,
      "learning_rate": 1.057661058522509e-06,
      "log_odds_chosen": 1.3792253732681274,
      "log_odds_ratio": -0.2334081530570984,
      "logits/chosen": -0.6443461775779724,
      "logits/rejected": -1.3520209789276123,
      "logps/chosen": -1.4532337188720703,
      "logps/rejected": -2.6397550106048584,
      "loss": 1.4873,
      "nll_loss": 1.4639172554016113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14532336592674255,
      "rewards/margins": 0.11865213513374329,
      "rewards/rejected": -0.2639755308628082,
      "step": 878
    },
    {
      "epoch": 1.384554767533491,
      "grad_norm": 0.23200878500938416,
      "learning_rate": 1.0528159471397425e-06,
      "log_odds_chosen": 1.25364089012146,
      "log_odds_ratio": -0.2615946829319,
      "logits/chosen": -0.6304550170898438,
      "logits/rejected": -1.3341405391693115,
      "logps/chosen": -1.4004929065704346,
      "logps/rejected": -2.4634275436401367,
      "loss": 1.4412,
      "nll_loss": 1.4150177240371704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1400493085384369,
      "rewards/margins": 0.1062934622168541,
      "rewards/rejected": -0.2463427484035492,
      "step": 879
    },
    {
      "epoch": 1.3861308116627264,
      "grad_norm": 0.21943318843841553,
      "learning_rate": 1.0479779912800868e-06,
      "log_odds_chosen": 1.2120345830917358,
      "log_odds_ratio": -0.26791390776634216,
      "logits/chosen": -0.7424058318138123,
      "logits/rejected": -1.2446584701538086,
      "logps/chosen": -1.3703131675720215,
      "logps/rejected": -2.385462522506714,
      "loss": 1.4123,
      "nll_loss": 1.3855550289154053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13703133165836334,
      "rewards/margins": 0.10151493549346924,
      "rewards/rejected": -0.2385462522506714,
      "step": 880
    },
    {
      "epoch": 1.3877068557919623,
      "grad_norm": 0.21455641090869904,
      "learning_rate": 1.0431472274919863e-06,
      "log_odds_chosen": 1.4327553510665894,
      "log_odds_ratio": -0.22192268073558807,
      "logits/chosen": -0.5550692081451416,
      "logits/rejected": -1.2892367839813232,
      "logps/chosen": -1.3894597291946411,
      "logps/rejected": -2.615027904510498,
      "loss": 1.4207,
      "nll_loss": 1.3985216617584229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13894596695899963,
      "rewards/margins": 0.12255682051181793,
      "rewards/rejected": -0.26150280237197876,
      "step": 881
    },
    {
      "epoch": 1.3892828999211977,
      "grad_norm": 0.1981762945652008,
      "learning_rate": 1.0383236922695543e-06,
      "log_odds_chosen": 1.5495109558105469,
      "log_odds_ratio": -0.2009485512971878,
      "logits/chosen": -0.4859476387500763,
      "logits/rejected": -1.2268177270889282,
      "logps/chosen": -1.230947494506836,
      "logps/rejected": -2.5039620399475098,
      "loss": 1.2847,
      "nll_loss": 1.26456618309021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12309475988149643,
      "rewards/margins": 0.12730145454406738,
      "rewards/rejected": -0.2503962218761444,
      "step": 882
    },
    {
      "epoch": 1.3908589440504335,
      "grad_norm": 0.26448187232017517,
      "learning_rate": 1.0335074220522962e-06,
      "log_odds_chosen": 1.4533723592758179,
      "log_odds_ratio": -0.22624823451042175,
      "logits/chosen": -0.6028724312782288,
      "logits/rejected": -1.0379383563995361,
      "logps/chosen": -1.4635196924209595,
      "logps/rejected": -2.726659059524536,
      "loss": 1.4944,
      "nll_loss": 1.471768856048584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14635197818279266,
      "rewards/margins": 0.1263139396905899,
      "rewards/rejected": -0.27266591787338257,
      "step": 883
    },
    {
      "epoch": 1.392434988179669,
      "grad_norm": 0.2470337599515915,
      "learning_rate": 1.0286984532248326e-06,
      "log_odds_chosen": 1.042252540588379,
      "log_odds_ratio": -0.3133049011230469,
      "logits/chosen": -0.5429861545562744,
      "logits/rejected": -1.2800817489624023,
      "logps/chosen": -1.4960817098617554,
      "logps/rejected": -2.380593776702881,
      "loss": 1.524,
      "nll_loss": 1.4926228523254395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14960817992687225,
      "rewards/margins": 0.08845119178295135,
      "rewards/rejected": -0.2380593717098236,
      "step": 884
    },
    {
      "epoch": 1.3940110323089048,
      "grad_norm": 0.23572620749473572,
      "learning_rate": 1.0238968221166269e-06,
      "log_odds_chosen": 1.212639570236206,
      "log_odds_ratio": -0.282530277967453,
      "logits/chosen": -0.6232253313064575,
      "logits/rejected": -1.2993882894515991,
      "logps/chosen": -1.4530023336410522,
      "logps/rejected": -2.4934468269348145,
      "loss": 1.4931,
      "nll_loss": 1.4648467302322388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1453002244234085,
      "rewards/margins": 0.10404447466135025,
      "rewards/rejected": -0.24934470653533936,
      "step": 885
    },
    {
      "epoch": 1.3955870764381402,
      "grad_norm": 0.22473900020122528,
      "learning_rate": 1.019102565001707e-06,
      "log_odds_chosen": 1.2412980794906616,
      "log_odds_ratio": -0.2679976224899292,
      "logits/chosen": -0.5948019623756409,
      "logits/rejected": -1.0699928998947144,
      "logps/chosen": -1.4866297245025635,
      "logps/rejected": -2.5560004711151123,
      "loss": 1.5145,
      "nll_loss": 1.4876673221588135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486629843711853,
      "rewards/margins": 0.10693708062171936,
      "rewards/rejected": -0.25560006499290466,
      "step": 886
    },
    {
      "epoch": 1.397163120567376,
      "grad_norm": 0.24419210851192474,
      "learning_rate": 1.0143157180983965e-06,
      "log_odds_chosen": 1.2027636766433716,
      "log_odds_ratio": -0.27474772930145264,
      "logits/chosen": -0.6643444895744324,
      "logits/rejected": -1.2303210496902466,
      "logps/chosen": -1.4725544452667236,
      "logps/rejected": -2.502936363220215,
      "loss": 1.4769,
      "nll_loss": 1.449439287185669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14725545048713684,
      "rewards/margins": 0.10303816944360733,
      "rewards/rejected": -0.25029364228248596,
      "step": 887
    },
    {
      "epoch": 1.3987391646966114,
      "grad_norm": 0.2135627120733261,
      "learning_rate": 1.0095363175690375e-06,
      "log_odds_chosen": 1.275749683380127,
      "log_odds_ratio": -0.2568631172180176,
      "logits/chosen": -0.6728612780570984,
      "logits/rejected": -1.0670406818389893,
      "logps/chosen": -1.4618005752563477,
      "logps/rejected": -2.5558342933654785,
      "loss": 1.4781,
      "nll_loss": 1.4524221420288086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14618004858493805,
      "rewards/margins": 0.1094033420085907,
      "rewards/rejected": -0.25558337569236755,
      "step": 888
    },
    {
      "epoch": 1.4003152088258473,
      "grad_norm": 0.21407313644886017,
      "learning_rate": 1.0047643995197178e-06,
      "log_odds_chosen": 1.3403247594833374,
      "log_odds_ratio": -0.2453526258468628,
      "logits/chosen": -0.5810958743095398,
      "logits/rejected": -1.220537781715393,
      "logps/chosen": -1.3764361143112183,
      "logps/rejected": -2.50840163230896,
      "loss": 1.4072,
      "nll_loss": 1.3826637268066406,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13764360547065735,
      "rewards/margins": 0.11319655179977417,
      "rewards/rejected": -0.2508401572704315,
      "step": 889
    },
    {
      "epoch": 1.4018912529550827,
      "grad_norm": 0.2221897691488266,
      "learning_rate": 1.0000000000000004e-06,
      "log_odds_chosen": 1.1772109270095825,
      "log_odds_ratio": -0.27655741572380066,
      "logits/chosen": -0.5688602328300476,
      "logits/rejected": -1.179971694946289,
      "logps/chosen": -1.4261137247085571,
      "logps/rejected": -2.417499542236328,
      "loss": 1.4727,
      "nll_loss": 1.4450145959854126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14261138439178467,
      "rewards/margins": 0.09913858771324158,
      "rewards/rejected": -0.24174995720386505,
      "step": 890
    },
    {
      "epoch": 1.4034672970843185,
      "grad_norm": 0.22176368534564972,
      "learning_rate": 9.952431550026459e-07,
      "log_odds_chosen": 1.328328251838684,
      "log_odds_ratio": -0.24654781818389893,
      "logits/chosen": -0.6992983818054199,
      "logits/rejected": -1.1700458526611328,
      "logps/chosen": -1.4487340450286865,
      "logps/rejected": -2.5920920372009277,
      "loss": 1.4819,
      "nll_loss": 1.4572343826293945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14487342536449432,
      "rewards/margins": 0.11433577537536621,
      "rewards/rejected": -0.25920918583869934,
      "step": 891
    },
    {
      "epoch": 1.405043341213554,
      "grad_norm": 0.23174212872982025,
      "learning_rate": 9.904939004633471e-07,
      "log_odds_chosen": 1.4404070377349854,
      "log_odds_ratio": -0.2158270925283432,
      "logits/chosen": -0.754274308681488,
      "logits/rejected": -1.3216673135757446,
      "logps/chosen": -1.506775140762329,
      "logps/rejected": -2.7591466903686523,
      "loss": 1.5168,
      "nll_loss": 1.4951709508895874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15067751705646515,
      "rewards/margins": 0.12523716688156128,
      "rewards/rejected": -0.2759146988391876,
      "step": 892
    },
    {
      "epoch": 1.4066193853427895,
      "grad_norm": 0.23180878162384033,
      "learning_rate": 9.857522722604536e-07,
      "log_odds_chosen": 1.15290105342865,
      "log_odds_ratio": -0.29690980911254883,
      "logits/chosen": -0.5832675695419312,
      "logits/rejected": -0.9560672044754028,
      "logps/chosen": -1.3804380893707275,
      "logps/rejected": -2.3488149642944336,
      "loss": 1.433,
      "nll_loss": 1.4033195972442627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13804380595684052,
      "rewards/margins": 0.09683768451213837,
      "rewards/rejected": -0.23488149046897888,
      "step": 893
    },
    {
      "epoch": 1.4081954294720251,
      "grad_norm": 0.2307136207818985,
      "learning_rate": 9.81018306214702e-07,
      "log_odds_chosen": 1.2128348350524902,
      "log_odds_ratio": -0.2658560276031494,
      "logits/chosen": -0.6448706984519958,
      "logits/rejected": -1.1317309141159058,
      "logps/chosen": -1.3916270732879639,
      "logps/rejected": -2.409489870071411,
      "loss": 1.4452,
      "nll_loss": 1.418574571609497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13916270434856415,
      "rewards/margins": 0.1017862856388092,
      "rewards/rejected": -0.24094898998737335,
      "step": 894
    },
    {
      "epoch": 1.4097714736012608,
      "grad_norm": 0.23846496641635895,
      "learning_rate": 9.76292038088945e-07,
      "log_odds_chosen": 1.160768747329712,
      "log_odds_ratio": -0.27834388613700867,
      "logits/chosen": -0.5789107084274292,
      "logits/rejected": -1.0358608961105347,
      "logps/chosen": -1.4730356931686401,
      "logps/rejected": -2.4634976387023926,
      "loss": 1.4983,
      "nll_loss": 1.4704917669296265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14730356633663177,
      "rewards/margins": 0.09904619306325912,
      "rewards/rejected": -0.2463497519493103,
      "step": 895
    },
    {
      "epoch": 1.4113475177304964,
      "grad_norm": 0.21409015357494354,
      "learning_rate": 9.715735035878799e-07,
      "log_odds_chosen": 1.4156800508499146,
      "log_odds_ratio": -0.22610259056091309,
      "logits/chosen": -0.6747400164604187,
      "logits/rejected": -1.200189471244812,
      "logps/chosen": -1.3644189834594727,
      "logps/rejected": -2.560804843902588,
      "loss": 1.3945,
      "nll_loss": 1.3718866109848022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1364419013261795,
      "rewards/margins": 0.11963862180709839,
      "rewards/rejected": -0.2560805082321167,
      "step": 896
    },
    {
      "epoch": 1.412923561859732,
      "grad_norm": 0.20026254653930664,
      "learning_rate": 9.668627383577812e-07,
      "log_odds_chosen": 1.2598985433578491,
      "log_odds_ratio": -0.2787105143070221,
      "logits/chosen": -0.6300346255302429,
      "logits/rejected": -1.2784793376922607,
      "logps/chosen": -1.4460864067077637,
      "logps/rejected": -2.530022382736206,
      "loss": 1.4827,
      "nll_loss": 1.4548131227493286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14460863173007965,
      "rewards/margins": 0.108393594622612,
      "rewards/rejected": -0.25300222635269165,
      "step": 897
    },
    {
      "epoch": 1.4144996059889676,
      "grad_norm": 0.2235194444656372,
      "learning_rate": 9.621597779862307e-07,
      "log_odds_chosen": 1.171904444694519,
      "log_odds_ratio": -0.27671828866004944,
      "logits/chosen": -0.7185797691345215,
      "logits/rejected": -1.1822232007980347,
      "logps/chosen": -1.4511301517486572,
      "logps/rejected": -2.445263385772705,
      "loss": 1.4798,
      "nll_loss": 1.4521715641021729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.145113006234169,
      "rewards/margins": 0.09941334277391434,
      "rewards/rejected": -0.24452634155750275,
      "step": 898
    },
    {
      "epoch": 1.4160756501182032,
      "grad_norm": 0.2320316880941391,
      "learning_rate": 9.57464658001848e-07,
      "log_odds_chosen": 1.2082061767578125,
      "log_odds_ratio": -0.2669847011566162,
      "logits/chosen": -0.6269704103469849,
      "logits/rejected": -1.2230405807495117,
      "logps/chosen": -1.4637227058410645,
      "logps/rejected": -2.495922088623047,
      "loss": 1.4934,
      "nll_loss": 1.4666904211044312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14637227356433868,
      "rewards/margins": 0.10321992635726929,
      "rewards/rejected": -0.24959219992160797,
      "step": 899
    },
    {
      "epoch": 1.4176516942474389,
      "grad_norm": 0.23445342481136322,
      "learning_rate": 9.527774138740212e-07,
      "log_odds_chosen": 1.3019967079162598,
      "log_odds_ratio": -0.24625040590763092,
      "logits/chosen": -0.7765132188796997,
      "logits/rejected": -1.2730624675750732,
      "logps/chosen": -1.4163048267364502,
      "logps/rejected": -2.5148894786834717,
      "loss": 1.4552,
      "nll_loss": 1.4305555820465088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14163048565387726,
      "rewards/margins": 0.10985846817493439,
      "rewards/rejected": -0.25148895382881165,
      "step": 900
    },
    {
      "epoch": 1.4192277383766745,
      "grad_norm": 0.22696658968925476,
      "learning_rate": 9.480980810126411e-07,
      "log_odds_chosen": 1.408170461654663,
      "log_odds_ratio": -0.25036320090293884,
      "logits/chosen": -0.7228882312774658,
      "logits/rejected": -1.2028861045837402,
      "logps/chosen": -1.3227603435516357,
      "logps/rejected": -2.523303270339966,
      "loss": 1.3703,
      "nll_loss": 1.3452810049057007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1322760432958603,
      "rewards/margins": 0.12005428969860077,
      "rewards/rejected": -0.25233036279678345,
      "step": 901
    },
    {
      "epoch": 1.42080378250591,
      "grad_norm": 0.23633477091789246,
      "learning_rate": 9.434266947678324e-07,
      "log_odds_chosen": 1.282657265663147,
      "log_odds_ratio": -0.2609420418739319,
      "logits/chosen": -0.7121891975402832,
      "logits/rejected": -1.247926950454712,
      "logps/chosen": -1.4080538749694824,
      "logps/rejected": -2.5020222663879395,
      "loss": 1.4337,
      "nll_loss": 1.4076323509216309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14080539345741272,
      "rewards/margins": 0.10939682275056839,
      "rewards/rejected": -0.2502022385597229,
      "step": 902
    },
    {
      "epoch": 1.4223798266351457,
      "grad_norm": 0.22363537549972534,
      "learning_rate": 9.387632904296872e-07,
      "log_odds_chosen": 1.52983820438385,
      "log_odds_ratio": -0.21243828535079956,
      "logits/chosen": -0.659780740737915,
      "logits/rejected": -1.2915232181549072,
      "logps/chosen": -1.4371575117111206,
      "logps/rejected": -2.762830972671509,
      "loss": 1.4589,
      "nll_loss": 1.4376457929611206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437157690525055,
      "rewards/margins": 0.13256731629371643,
      "rewards/rejected": -0.2762830853462219,
      "step": 903
    },
    {
      "epoch": 1.4239558707643813,
      "grad_norm": 0.21816618740558624,
      "learning_rate": 9.341079032279986e-07,
      "log_odds_chosen": 1.4585187435150146,
      "log_odds_ratio": -0.22713728249073029,
      "logits/chosen": -0.5981857180595398,
      "logits/rejected": -1.1607801914215088,
      "logps/chosen": -1.3972305059432983,
      "logps/rejected": -2.645676612854004,
      "loss": 1.4342,
      "nll_loss": 1.4114580154418945,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1397230476140976,
      "rewards/margins": 0.12484461814165115,
      "rewards/rejected": -0.26456767320632935,
      "step": 904
    },
    {
      "epoch": 1.425531914893617,
      "grad_norm": 0.20383957028388977,
      "learning_rate": 9.294605683319919e-07,
      "log_odds_chosen": 1.4679661989212036,
      "log_odds_ratio": -0.21783822774887085,
      "logits/chosen": -0.7071681618690491,
      "logits/rejected": -1.2394092082977295,
      "logps/chosen": -1.2906256914138794,
      "logps/rejected": -2.522862434387207,
      "loss": 1.3255,
      "nll_loss": 1.303668737411499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12906257808208466,
      "rewards/margins": 0.123223677277565,
      "rewards/rejected": -0.25228625535964966,
      "step": 905
    },
    {
      "epoch": 1.4271079590228526,
      "grad_norm": 0.22947923839092255,
      "learning_rate": 9.248213208500629e-07,
      "log_odds_chosen": 1.2109204530715942,
      "log_odds_ratio": -0.2752280533313751,
      "logits/chosen": -0.751317024230957,
      "logits/rejected": -1.2244060039520264,
      "logps/chosen": -1.4690463542938232,
      "logps/rejected": -2.5047669410705566,
      "loss": 1.4885,
      "nll_loss": 1.460947871208191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14690463244915009,
      "rewards/margins": 0.1035720556974411,
      "rewards/rejected": -0.2504766583442688,
      "step": 906
    },
    {
      "epoch": 1.4286840031520882,
      "grad_norm": 0.2369922697544098,
      "learning_rate": 9.201901958295115e-07,
      "log_odds_chosen": 1.4616670608520508,
      "log_odds_ratio": -0.22588446736335754,
      "logits/chosen": -0.6530401706695557,
      "logits/rejected": -1.1584906578063965,
      "logps/chosen": -1.425114393234253,
      "logps/rejected": -2.6863808631896973,
      "loss": 1.4714,
      "nll_loss": 1.4488575458526611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14251144230365753,
      "rewards/margins": 0.12612661719322205,
      "rewards/rejected": -0.26863807439804077,
      "step": 907
    },
    {
      "epoch": 1.4302600472813238,
      "grad_norm": 0.21130536496639252,
      "learning_rate": 9.155672282562736e-07,
      "log_odds_chosen": 1.300620675086975,
      "log_odds_ratio": -0.25154638290405273,
      "logits/chosen": -0.7621893882751465,
      "logits/rejected": -1.2991900444030762,
      "logps/chosen": -1.4467990398406982,
      "logps/rejected": -2.5623691082000732,
      "loss": 1.4831,
      "nll_loss": 1.4579659700393677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14467990398406982,
      "rewards/margins": 0.1115570142865181,
      "rewards/rejected": -0.2562369108200073,
      "step": 908
    },
    {
      "epoch": 1.4318360914105595,
      "grad_norm": 0.2274758666753769,
      "learning_rate": 9.109524530546622e-07,
      "log_odds_chosen": 1.1850595474243164,
      "log_odds_ratio": -0.2728549838066101,
      "logits/chosen": -0.7266186475753784,
      "logits/rejected": -1.2169804573059082,
      "logps/chosen": -1.4477635622024536,
      "logps/rejected": -2.4506494998931885,
      "loss": 1.4864,
      "nll_loss": 1.4591138362884521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1447763741016388,
      "rewards/margins": 0.10028856247663498,
      "rewards/rejected": -0.24506492912769318,
      "step": 909
    },
    {
      "epoch": 1.433412135539795,
      "grad_norm": 0.2616197466850281,
      "learning_rate": 9.063459050871001e-07,
      "log_odds_chosen": 1.1652344465255737,
      "log_odds_ratio": -0.27971285581588745,
      "logits/chosen": -0.7794743180274963,
      "logits/rejected": -1.0473065376281738,
      "logps/chosen": -1.4129595756530762,
      "logps/rejected": -2.395077705383301,
      "loss": 1.4401,
      "nll_loss": 1.4120924472808838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14129595458507538,
      "rewards/margins": 0.09821180999279022,
      "rewards/rejected": -0.2395077496767044,
      "step": 910
    },
    {
      "epoch": 1.4349881796690307,
      "grad_norm": 0.2275470495223999,
      "learning_rate": 9.017476191538555e-07,
      "log_odds_chosen": 1.414859414100647,
      "log_odds_ratio": -0.2326749563217163,
      "logits/chosen": -0.7399898767471313,
      "logits/rejected": -1.0788378715515137,
      "logps/chosen": -1.3633774518966675,
      "logps/rejected": -2.5597002506256104,
      "loss": 1.408,
      "nll_loss": 1.384740948677063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1363377571105957,
      "rewards/margins": 0.11963231861591339,
      "rewards/rejected": -0.2559700608253479,
      "step": 911
    },
    {
      "epoch": 1.4365642237982663,
      "grad_norm": 0.22230906784534454,
      "learning_rate": 8.971576299927832e-07,
      "log_odds_chosen": 1.4642901420593262,
      "log_odds_ratio": -0.22000297904014587,
      "logits/chosen": -0.6901626586914062,
      "logits/rejected": -1.3889858722686768,
      "logps/chosen": -1.454336404800415,
      "logps/rejected": -2.7216720581054688,
      "loss": 1.4837,
      "nll_loss": 1.4616817235946655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14543363451957703,
      "rewards/margins": 0.12673358619213104,
      "rewards/rejected": -0.2721672058105469,
      "step": 912
    },
    {
      "epoch": 1.438140267927502,
      "grad_norm": 0.2147761881351471,
      "learning_rate": 8.925759722790591e-07,
      "log_odds_chosen": 1.3027318716049194,
      "log_odds_ratio": -0.24704775214195251,
      "logits/chosen": -0.6561098694801331,
      "logits/rejected": -1.2570255994796753,
      "logps/chosen": -1.4217052459716797,
      "logps/rejected": -2.5311551094055176,
      "loss": 1.4559,
      "nll_loss": 1.4312403202056885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14217053353786469,
      "rewards/margins": 0.11094498634338379,
      "rewards/rejected": -0.25311553478240967,
      "step": 913
    },
    {
      "epoch": 1.4397163120567376,
      "grad_norm": 0.24627818167209625,
      "learning_rate": 8.880026806249192e-07,
      "log_odds_chosen": 1.5386884212493896,
      "log_odds_ratio": -0.20066040754318237,
      "logits/chosen": -0.5873969793319702,
      "logits/rejected": -1.3228124380111694,
      "logps/chosen": -1.4800702333450317,
      "logps/rejected": -2.820051670074463,
      "loss": 1.5014,
      "nll_loss": 1.4813209772109985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14800702035427094,
      "rewards/margins": 0.13399815559387207,
      "rewards/rejected": -0.2820051908493042,
      "step": 914
    },
    {
      "epoch": 1.4412923561859732,
      "grad_norm": 0.2522904872894287,
      "learning_rate": 8.834377895793965e-07,
      "log_odds_chosen": 1.2323895692825317,
      "log_odds_ratio": -0.27465054392814636,
      "logits/chosen": -0.6142185926437378,
      "logits/rejected": -1.1593711376190186,
      "logps/chosen": -1.5539109706878662,
      "logps/rejected": -2.6245932579040527,
      "loss": 1.5796,
      "nll_loss": 1.5521807670593262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15539109706878662,
      "rewards/margins": 0.10706820338964462,
      "rewards/rejected": -0.26245933771133423,
      "step": 915
    },
    {
      "epoch": 1.4428684003152088,
      "grad_norm": 0.22280728816986084,
      "learning_rate": 8.78881333628063e-07,
      "log_odds_chosen": 1.2903611660003662,
      "log_odds_ratio": -0.2451159507036209,
      "logits/chosen": -0.6921783685684204,
      "logits/rejected": -1.3785640001296997,
      "logps/chosen": -1.4704359769821167,
      "logps/rejected": -2.5771100521087646,
      "loss": 1.4905,
      "nll_loss": 1.465954303741455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470436006784439,
      "rewards/margins": 0.11066742241382599,
      "rewards/rejected": -0.2577109932899475,
      "step": 916
    },
    {
      "epoch": 1.4444444444444444,
      "grad_norm": 0.2196531593799591,
      "learning_rate": 8.743333471927672e-07,
      "log_odds_chosen": 1.4275996685028076,
      "log_odds_ratio": -0.21908238530158997,
      "logits/chosen": -0.7361686825752258,
      "logits/rejected": -1.3575388193130493,
      "logps/chosen": -1.478491187095642,
      "logps/rejected": -2.715571165084839,
      "loss": 1.5082,
      "nll_loss": 1.4862620830535889,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14784912765026093,
      "rewards/margins": 0.12370800226926804,
      "rewards/rejected": -0.27155715227127075,
      "step": 917
    },
    {
      "epoch": 1.44602048857368,
      "grad_norm": 0.24711570143699646,
      "learning_rate": 8.697938646313742e-07,
      "log_odds_chosen": 1.368897795677185,
      "log_odds_ratio": -0.2284998595714569,
      "logits/chosen": -0.7581544518470764,
      "logits/rejected": -1.1745307445526123,
      "logps/chosen": -1.3835235834121704,
      "logps/rejected": -2.5430870056152344,
      "loss": 1.4128,
      "nll_loss": 1.3899706602096558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13835236430168152,
      "rewards/margins": 0.11595632135868073,
      "rewards/rejected": -0.25430867075920105,
      "step": 918
    },
    {
      "epoch": 1.4475965327029157,
      "grad_norm": 0.2431514710187912,
      "learning_rate": 8.652629202375075e-07,
      "log_odds_chosen": 1.2483316659927368,
      "log_odds_ratio": -0.26442867517471313,
      "logits/chosen": -0.5306358933448792,
      "logits/rejected": -1.2592718601226807,
      "logps/chosen": -1.40236234664917,
      "logps/rejected": -2.45929217338562,
      "loss": 1.4467,
      "nll_loss": 1.4202969074249268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14023622870445251,
      "rewards/margins": 0.10569299757480621,
      "rewards/rejected": -0.24592924118041992,
      "step": 919
    },
    {
      "epoch": 1.4491725768321513,
      "grad_norm": 0.2159259021282196,
      "learning_rate": 8.607405482402861e-07,
      "log_odds_chosen": 1.5411865711212158,
      "log_odds_ratio": -0.21287758648395538,
      "logits/chosen": -0.7407127022743225,
      "logits/rejected": -1.2575466632843018,
      "logps/chosen": -1.2843648195266724,
      "logps/rejected": -2.5874857902526855,
      "loss": 1.3248,
      "nll_loss": 1.3034793138504028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12843649089336395,
      "rewards/margins": 0.13031210005283356,
      "rewards/rejected": -0.2587485909461975,
      "step": 920
    },
    {
      "epoch": 1.450748620961387,
      "grad_norm": 0.2194749265909195,
      "learning_rate": 8.562267828040712e-07,
      "log_odds_chosen": 1.4350190162658691,
      "log_odds_ratio": -0.22347907721996307,
      "logits/chosen": -0.737739622592926,
      "logits/rejected": -1.4998080730438232,
      "logps/chosen": -1.4111101627349854,
      "logps/rejected": -2.6424412727355957,
      "loss": 1.436,
      "nll_loss": 1.4136382341384888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14111101627349854,
      "rewards/margins": 0.1231331080198288,
      "rewards/rejected": -0.26424410939216614,
      "step": 921
    },
    {
      "epoch": 1.4523246650906225,
      "grad_norm": 0.22178393602371216,
      "learning_rate": 8.517216580282048e-07,
      "log_odds_chosen": 1.2065441608428955,
      "log_odds_ratio": -0.2721346616744995,
      "logits/chosen": -0.5913813710212708,
      "logits/rejected": -1.296689510345459,
      "logps/chosen": -1.4809776544570923,
      "logps/rejected": -2.5122039318084717,
      "loss": 1.5288,
      "nll_loss": 1.5016052722930908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14809775352478027,
      "rewards/margins": 0.10312263667583466,
      "rewards/rejected": -0.2512204051017761,
      "step": 922
    },
    {
      "epoch": 1.4539007092198581,
      "grad_norm": 0.24411170184612274,
      "learning_rate": 8.472252079467535e-07,
      "log_odds_chosen": 1.2505497932434082,
      "log_odds_ratio": -0.2577451467514038,
      "logits/chosen": -0.6834046840667725,
      "logits/rejected": -1.3004528284072876,
      "logps/chosen": -1.4548959732055664,
      "logps/rejected": -2.5227668285369873,
      "loss": 1.4872,
      "nll_loss": 1.4613832235336304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14548960328102112,
      "rewards/margins": 0.10678709298372269,
      "rewards/rejected": -0.2522766888141632,
      "step": 923
    },
    {
      "epoch": 1.4554767533490938,
      "grad_norm": 0.21333174407482147,
      "learning_rate": 8.427374665282488e-07,
      "log_odds_chosen": 1.1033653020858765,
      "log_odds_ratio": -0.2901444435119629,
      "logits/chosen": -0.5654793381690979,
      "logits/rejected": -1.0405590534210205,
      "logps/chosen": -1.3729230165481567,
      "logps/rejected": -2.289926290512085,
      "loss": 1.4174,
      "nll_loss": 1.3884345293045044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1372922956943512,
      "rewards/margins": 0.09170033782720566,
      "rewards/rejected": -0.22899261116981506,
      "step": 924
    },
    {
      "epoch": 1.4570527974783294,
      "grad_norm": 0.28356799483299255,
      "learning_rate": 8.382584676754336e-07,
      "log_odds_chosen": 1.1785967350006104,
      "log_odds_ratio": -0.2742956280708313,
      "logits/chosen": -0.724522590637207,
      "logits/rejected": -1.1836694478988647,
      "logps/chosen": -1.4601523876190186,
      "logps/rejected": -2.4657604694366455,
      "loss": 1.4931,
      "nll_loss": 1.4656444787979126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460152417421341,
      "rewards/margins": 0.10056082904338837,
      "rewards/rejected": -0.24657605588436127,
      "step": 925
    },
    {
      "epoch": 1.458628841607565,
      "grad_norm": 0.21661214530467987,
      "learning_rate": 8.337882452250058e-07,
      "log_odds_chosen": 1.645066261291504,
      "log_odds_ratio": -0.1954662799835205,
      "logits/chosen": -0.7276672720909119,
      "logits/rejected": -1.4512091875076294,
      "logps/chosen": -1.4748291969299316,
      "logps/rejected": -2.9244494438171387,
      "loss": 1.4954,
      "nll_loss": 1.4758051633834839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14748293161392212,
      "rewards/margins": 0.14496199786663055,
      "rewards/rejected": -0.2924448847770691,
      "step": 926
    },
    {
      "epoch": 1.4602048857368006,
      "grad_norm": 0.22286337614059448,
      "learning_rate": 8.293268329473602e-07,
      "log_odds_chosen": 1.6206045150756836,
      "log_odds_ratio": -0.1952732503414154,
      "logits/chosen": -0.7577838897705078,
      "logits/rejected": -1.3013466596603394,
      "logps/chosen": -1.4113264083862305,
      "logps/rejected": -2.820190668106079,
      "loss": 1.4419,
      "nll_loss": 1.4224051237106323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.141132652759552,
      "rewards/margins": 0.14088644087314606,
      "rewards/rejected": -0.28201907873153687,
      "step": 927
    },
    {
      "epoch": 1.4617809298660362,
      "grad_norm": 0.2205461710691452,
      "learning_rate": 8.248742645463367e-07,
      "log_odds_chosen": 1.463126301765442,
      "log_odds_ratio": -0.2200469821691513,
      "logits/chosen": -0.6799696087837219,
      "logits/rejected": -1.2515597343444824,
      "logps/chosen": -1.4793611764907837,
      "logps/rejected": -2.7522425651550293,
      "loss": 1.4921,
      "nll_loss": 1.4701058864593506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1479361206293106,
      "rewards/margins": 0.12728813290596008,
      "rewards/rejected": -0.2752242684364319,
      "step": 928
    },
    {
      "epoch": 1.4633569739952719,
      "grad_norm": 0.22218818962574005,
      "learning_rate": 8.204305736589612e-07,
      "log_odds_chosen": 1.3795361518859863,
      "log_odds_ratio": -0.24047043919563293,
      "logits/chosen": -0.6664568185806274,
      "logits/rejected": -1.1032912731170654,
      "logps/chosen": -1.4322352409362793,
      "logps/rejected": -2.617957353591919,
      "loss": 1.4574,
      "nll_loss": 1.433355450630188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14322352409362793,
      "rewards/margins": 0.11857221275568008,
      "rewards/rejected": -0.2617957592010498,
      "step": 929
    },
    {
      "epoch": 1.4649330181245075,
      "grad_norm": 0.2152458131313324,
      "learning_rate": 8.159957938551966e-07,
      "log_odds_chosen": 1.282020092010498,
      "log_odds_ratio": -0.26019296050071716,
      "logits/chosen": -0.6306040287017822,
      "logits/rejected": -1.2225638628005981,
      "logps/chosen": -1.437345027923584,
      "logps/rejected": -2.5258424282073975,
      "loss": 1.4739,
      "nll_loss": 1.44788658618927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14373449981212616,
      "rewards/margins": 0.10884976387023926,
      "rewards/rejected": -0.2525842785835266,
      "step": 930
    },
    {
      "epoch": 1.4665090622537431,
      "grad_norm": 0.2308555394411087,
      "learning_rate": 8.115699586376865e-07,
      "log_odds_chosen": 1.2568104267120361,
      "log_odds_ratio": -0.25735166668891907,
      "logits/chosen": -0.7380691766738892,
      "logits/rejected": -1.188723087310791,
      "logps/chosen": -1.4800864458084106,
      "logps/rejected": -2.5618178844451904,
      "loss": 1.506,
      "nll_loss": 1.4802873134613037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14800864458084106,
      "rewards/margins": 0.10817313194274902,
      "rewards/rejected": -0.2561817765235901,
      "step": 931
    },
    {
      "epoch": 1.4680851063829787,
      "grad_norm": 0.237010195851326,
      "learning_rate": 8.071531014415018e-07,
      "log_odds_chosen": 1.5165319442749023,
      "log_odds_ratio": -0.2087167650461197,
      "logits/chosen": -0.70307457447052,
      "logits/rejected": -1.24103844165802,
      "logps/chosen": -1.4413257837295532,
      "logps/rejected": -2.7571234703063965,
      "loss": 1.4508,
      "nll_loss": 1.4299489259719849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1441325694322586,
      "rewards/margins": 0.13157980144023895,
      "rewards/rejected": -0.27571237087249756,
      "step": 932
    },
    {
      "epoch": 1.4696611505122144,
      "grad_norm": 0.21725594997406006,
      "learning_rate": 8.027452556338894e-07,
      "log_odds_chosen": 1.4184367656707764,
      "log_odds_ratio": -0.22002457082271576,
      "logits/chosen": -0.659585177898407,
      "logits/rejected": -1.3644376993179321,
      "logps/chosen": -1.444718599319458,
      "logps/rejected": -2.6607296466827393,
      "loss": 1.472,
      "nll_loss": 1.450027585029602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14447186887264252,
      "rewards/margins": 0.12160112708806992,
      "rewards/rejected": -0.26607298851013184,
      "step": 933
    },
    {
      "epoch": 1.47123719464145,
      "grad_norm": 0.2410678118467331,
      "learning_rate": 7.983464545140179e-07,
      "log_odds_chosen": 1.5320338010787964,
      "log_odds_ratio": -0.19976618885993958,
      "logits/chosen": -0.7432952523231506,
      "logits/rejected": -1.2311393022537231,
      "logps/chosen": -1.3846819400787354,
      "logps/rejected": -2.6984381675720215,
      "loss": 1.4042,
      "nll_loss": 1.3842161893844604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1384681910276413,
      "rewards/margins": 0.13137564063072205,
      "rewards/rejected": -0.26984384655952454,
      "step": 934
    },
    {
      "epoch": 1.4728132387706856,
      "grad_norm": 0.2691135108470917,
      "learning_rate": 7.939567313127295e-07,
      "log_odds_chosen": 1.4454156160354614,
      "log_odds_ratio": -0.2154652178287506,
      "logits/chosen": -0.6969341039657593,
      "logits/rejected": -1.1929199695587158,
      "logps/chosen": -1.4495669603347778,
      "logps/rejected": -2.6962738037109375,
      "loss": 1.4794,
      "nll_loss": 1.4578465223312378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14495669305324554,
      "rewards/margins": 0.12467069923877716,
      "rewards/rejected": -0.2696273922920227,
      "step": 935
    },
    {
      "epoch": 1.4743892828999212,
      "grad_norm": 0.22375920414924622,
      "learning_rate": 7.89576119192286e-07,
      "log_odds_chosen": 1.4953581094741821,
      "log_odds_ratio": -0.21105709671974182,
      "logits/chosen": -0.6594355702400208,
      "logits/rejected": -1.241795301437378,
      "logps/chosen": -1.3873887062072754,
      "logps/rejected": -2.668959140777588,
      "loss": 1.4062,
      "nll_loss": 1.3851351737976074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13873887062072754,
      "rewards/margins": 0.12815703451633453,
      "rewards/rejected": -0.2668958902359009,
      "step": 936
    },
    {
      "epoch": 1.4759653270291568,
      "grad_norm": 0.23197168111801147,
      "learning_rate": 7.852046512461201e-07,
      "log_odds_chosen": 1.4727333784103394,
      "log_odds_ratio": -0.21330472826957703,
      "logits/chosen": -0.7134162187576294,
      "logits/rejected": -1.269208550453186,
      "logps/chosen": -1.409096598625183,
      "logps/rejected": -2.6700189113616943,
      "loss": 1.4368,
      "nll_loss": 1.4155021905899048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14090964198112488,
      "rewards/margins": 0.12609222531318665,
      "rewards/rejected": -0.2670018970966339,
      "step": 937
    },
    {
      "epoch": 1.4775413711583925,
      "grad_norm": 0.21121343970298767,
      "learning_rate": 7.808423604985843e-07,
      "log_odds_chosen": 1.507707118988037,
      "log_odds_ratio": -0.21521711349487305,
      "logits/chosen": -0.6082285642623901,
      "logits/rejected": -1.1762725114822388,
      "logps/chosen": -1.401310682296753,
      "logps/rejected": -2.695380687713623,
      "loss": 1.4557,
      "nll_loss": 1.4341330528259277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14013107120990753,
      "rewards/margins": 0.12940700352191925,
      "rewards/rejected": -0.2695380449295044,
      "step": 938
    },
    {
      "epoch": 1.479117415287628,
      "grad_norm": 0.22284933924674988,
      "learning_rate": 7.764892799047005e-07,
      "log_odds_chosen": 1.5386701822280884,
      "log_odds_ratio": -0.20569512248039246,
      "logits/chosen": -0.668232262134552,
      "logits/rejected": -1.3395743370056152,
      "logps/chosen": -1.445014476776123,
      "logps/rejected": -2.7790582180023193,
      "loss": 1.4715,
      "nll_loss": 1.4508986473083496,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1445014625787735,
      "rewards/margins": 0.13340437412261963,
      "rewards/rejected": -0.2779058516025543,
      "step": 939
    },
    {
      "epoch": 1.4806934594168637,
      "grad_norm": 0.22371213138103485,
      "learning_rate": 7.721454423499143e-07,
      "log_odds_chosen": 1.5315446853637695,
      "log_odds_ratio": -0.2103346288204193,
      "logits/chosen": -0.7251273393630981,
      "logits/rejected": -1.4145408868789673,
      "logps/chosen": -1.4225165843963623,
      "logps/rejected": -2.7425949573516846,
      "loss": 1.4498,
      "nll_loss": 1.4287962913513184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.142251655459404,
      "rewards/margins": 0.1320078819990158,
      "rewards/rejected": -0.2742595076560974,
      "step": 940
    },
    {
      "epoch": 1.4822695035460993,
      "grad_norm": 0.24764937162399292,
      "learning_rate": 7.678108806498441e-07,
      "log_odds_chosen": 1.542768955230713,
      "log_odds_ratio": -0.21060390770435333,
      "logits/chosen": -0.649777889251709,
      "logits/rejected": -1.203359603881836,
      "logps/chosen": -1.396357536315918,
      "logps/rejected": -2.729748249053955,
      "loss": 1.4335,
      "nll_loss": 1.4123975038528442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13963574171066284,
      "rewards/margins": 0.133339062333107,
      "rewards/rejected": -0.27297481894493103,
      "step": 941
    },
    {
      "epoch": 1.483845547675335,
      "grad_norm": 0.2625877857208252,
      "learning_rate": 7.634856275500315e-07,
      "log_odds_chosen": 1.18978750705719,
      "log_odds_ratio": -0.27729126811027527,
      "logits/chosen": -0.6717730164527893,
      "logits/rejected": -1.0993735790252686,
      "logps/chosen": -1.5346648693084717,
      "logps/rejected": -2.5651772022247314,
      "loss": 1.5613,
      "nll_loss": 1.5336045026779175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15346647799015045,
      "rewards/margins": 0.10305126756429672,
      "rewards/rejected": -0.25651776790618896,
      "step": 942
    },
    {
      "epoch": 1.4854215918045706,
      "grad_norm": 0.2064867913722992,
      "learning_rate": 7.591697157256991e-07,
      "log_odds_chosen": 1.461602807044983,
      "log_odds_ratio": -0.22322657704353333,
      "logits/chosen": -0.6569210290908813,
      "logits/rejected": -1.194642186164856,
      "logps/chosen": -1.365678310394287,
      "logps/rejected": -2.60880708694458,
      "loss": 1.3961,
      "nll_loss": 1.3737297058105469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1365678459405899,
      "rewards/margins": 0.1243128776550293,
      "rewards/rejected": -0.260880708694458,
      "step": 943
    },
    {
      "epoch": 1.4869976359338062,
      "grad_norm": 0.23511092364788055,
      "learning_rate": 7.548631777814995e-07,
      "log_odds_chosen": 1.5949876308441162,
      "log_odds_ratio": -0.20250055193901062,
      "logits/chosen": -0.6768131256103516,
      "logits/rejected": -1.2346688508987427,
      "logps/chosen": -1.4207197427749634,
      "logps/rejected": -2.807065486907959,
      "loss": 1.455,
      "nll_loss": 1.4347747564315796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14207197725772858,
      "rewards/margins": 0.1386345773935318,
      "rewards/rejected": -0.2807065546512604,
      "step": 944
    },
    {
      "epoch": 1.4885736800630418,
      "grad_norm": 0.21723121404647827,
      "learning_rate": 7.505660462512689e-07,
      "log_odds_chosen": 1.3031891584396362,
      "log_odds_ratio": -0.2469046711921692,
      "logits/chosen": -0.7324569821357727,
      "logits/rejected": -1.1604727506637573,
      "logps/chosen": -1.3469759225845337,
      "logps/rejected": -2.4353866577148438,
      "loss": 1.3961,
      "nll_loss": 1.3714450597763062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13469760119915009,
      "rewards/margins": 0.10884106904268265,
      "rewards/rejected": -0.24353866279125214,
      "step": 945
    },
    {
      "epoch": 1.4901497241922774,
      "grad_norm": 0.22847594320774078,
      "learning_rate": 7.462783535977842e-07,
      "log_odds_chosen": 1.5267882347106934,
      "log_odds_ratio": -0.20141902565956116,
      "logits/chosen": -0.7036858797073364,
      "logits/rejected": -1.2694916725158691,
      "logps/chosen": -1.3950082063674927,
      "logps/rejected": -2.7055087089538574,
      "loss": 1.4257,
      "nll_loss": 1.4055471420288086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13950082659721375,
      "rewards/margins": 0.13105003535747528,
      "rewards/rejected": -0.2705508768558502,
      "step": 946
    },
    {
      "epoch": 1.491725768321513,
      "grad_norm": 0.2100229561328888,
      "learning_rate": 7.420001322125156e-07,
      "log_odds_chosen": 1.0018866062164307,
      "log_odds_ratio": -0.32618248462677,
      "logits/chosen": -0.803820788860321,
      "logits/rejected": -1.151245355606079,
      "logps/chosen": -1.3797246217727661,
      "logps/rejected": -2.2159223556518555,
      "loss": 1.4186,
      "nll_loss": 1.3860046863555908,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13797247409820557,
      "rewards/margins": 0.08361979573965073,
      "rewards/rejected": -0.2215922474861145,
      "step": 947
    },
    {
      "epoch": 1.4933018124507487,
      "grad_norm": 0.22347095608711243,
      "learning_rate": 7.377314144153814e-07,
      "log_odds_chosen": 1.134749174118042,
      "log_odds_ratio": -0.283597469329834,
      "logits/chosen": -0.7413504123687744,
      "logits/rejected": -1.249704360961914,
      "logps/chosen": -1.4547138214111328,
      "logps/rejected": -2.416956663131714,
      "loss": 1.4925,
      "nll_loss": 1.464111089706421,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14547139406204224,
      "rewards/margins": 0.09622429311275482,
      "rewards/rejected": -0.24169565737247467,
      "step": 948
    },
    {
      "epoch": 1.4948778565799843,
      "grad_norm": 0.2236117124557495,
      "learning_rate": 7.334722324545064e-07,
      "log_odds_chosen": 1.4533774852752686,
      "log_odds_ratio": -0.21512113511562347,
      "logits/chosen": -0.6374566555023193,
      "logits/rejected": -1.3684566020965576,
      "logps/chosen": -1.4066067934036255,
      "logps/rejected": -2.650752305984497,
      "loss": 1.4198,
      "nll_loss": 1.398282766342163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14066068828105927,
      "rewards/margins": 0.12441454827785492,
      "rewards/rejected": -0.2650752365589142,
      "step": 949
    },
    {
      "epoch": 1.49645390070922,
      "grad_norm": 0.2206820696592331,
      "learning_rate": 7.292226185059756e-07,
      "log_odds_chosen": 1.2964389324188232,
      "log_odds_ratio": -0.24955812096595764,
      "logits/chosen": -0.6485929489135742,
      "logits/rejected": -1.1354975700378418,
      "logps/chosen": -1.4709134101867676,
      "logps/rejected": -2.5870397090911865,
      "loss": 1.5097,
      "nll_loss": 1.484703779220581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470913589000702,
      "rewards/margins": 0.11161260306835175,
      "rewards/rejected": -0.25870394706726074,
      "step": 950
    },
    {
      "epoch": 1.4980299448384555,
      "grad_norm": 0.21728011965751648,
      "learning_rate": 7.249826046735927e-07,
      "log_odds_chosen": 1.40239417552948,
      "log_odds_ratio": -0.2279975414276123,
      "logits/chosen": -0.7290428280830383,
      "logits/rejected": -1.323069453239441,
      "logps/chosen": -1.4079325199127197,
      "logps/rejected": -2.6088922023773193,
      "loss": 1.4373,
      "nll_loss": 1.4144505262374878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14079326391220093,
      "rewards/margins": 0.12009596824645996,
      "rewards/rejected": -0.2608892023563385,
      "step": 951
    },
    {
      "epoch": 1.4996059889676912,
      "grad_norm": 0.2572686970233917,
      "learning_rate": 7.207522229886379e-07,
      "log_odds_chosen": 1.4620333909988403,
      "log_odds_ratio": -0.22351157665252686,
      "logits/chosen": -0.7324269413948059,
      "logits/rejected": -1.139201283454895,
      "logps/chosen": -1.334355354309082,
      "logps/rejected": -2.5688791275024414,
      "loss": 1.3713,
      "nll_loss": 1.3489469289779663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13343554735183716,
      "rewards/margins": 0.12345238029956818,
      "rewards/rejected": -0.25688791275024414,
      "step": 952
    },
    {
      "epoch": 1.5011820330969265,
      "grad_norm": 0.23799435794353485,
      "learning_rate": 7.165315054096228e-07,
      "log_odds_chosen": 1.3695685863494873,
      "log_odds_ratio": -0.23961515724658966,
      "logits/chosen": -0.7156524658203125,
      "logits/rejected": -1.2838789224624634,
      "logps/chosen": -1.4874627590179443,
      "logps/rejected": -2.675133228302002,
      "loss": 1.5017,
      "nll_loss": 1.4777547121047974,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1487462818622589,
      "rewards/margins": 0.11876704543828964,
      "rewards/rejected": -0.26751330494880676,
      "step": 953
    },
    {
      "epoch": 1.5027580772261624,
      "grad_norm": 0.2196931093931198,
      "learning_rate": 7.123204838220533e-07,
      "log_odds_chosen": 1.575005054473877,
      "log_odds_ratio": -0.2048576921224594,
      "logits/chosen": -0.7845547199249268,
      "logits/rejected": -1.3786720037460327,
      "logps/chosen": -1.3682609796524048,
      "logps/rejected": -2.725167751312256,
      "loss": 1.3965,
      "nll_loss": 1.375985860824585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13682611286640167,
      "rewards/margins": 0.13569065928459167,
      "rewards/rejected": -0.27251678705215454,
      "step": 954
    },
    {
      "epoch": 1.5043341213553978,
      "grad_norm": 0.22504957020282745,
      "learning_rate": 7.081191900381862e-07,
      "log_odds_chosen": 1.4550952911376953,
      "log_odds_ratio": -0.21754170954227448,
      "logits/chosen": -0.7186540961265564,
      "logits/rejected": -1.2829034328460693,
      "logps/chosen": -1.4811155796051025,
      "logps/rejected": -2.7451171875,
      "loss": 1.5109,
      "nll_loss": 1.489145040512085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14811155200004578,
      "rewards/margins": 0.1264001727104187,
      "rewards/rejected": -0.2745117247104645,
      "step": 955
    },
    {
      "epoch": 1.5059101654846336,
      "grad_norm": 0.2168712168931961,
      "learning_rate": 7.039276557967895e-07,
      "log_odds_chosen": 1.7035603523254395,
      "log_odds_ratio": -0.17721766233444214,
      "logits/chosen": -0.6746881604194641,
      "logits/rejected": -1.3812861442565918,
      "logps/chosen": -1.3664357662200928,
      "logps/rejected": -2.8319969177246094,
      "loss": 1.403,
      "nll_loss": 1.3853123188018799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13664358854293823,
      "rewards/margins": 0.14655612409114838,
      "rewards/rejected": -0.2831997275352478,
      "step": 956
    },
    {
      "epoch": 1.507486209613869,
      "grad_norm": 0.21461330354213715,
      "learning_rate": 6.997459127629028e-07,
      "log_odds_chosen": 1.155877947807312,
      "log_odds_ratio": -0.2880905270576477,
      "logits/chosen": -0.6960952281951904,
      "logits/rejected": -1.2469191551208496,
      "logps/chosen": -1.4440693855285645,
      "logps/rejected": -2.426624298095703,
      "loss": 1.4917,
      "nll_loss": 1.4628902673721313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14440692961215973,
      "rewards/margins": 0.09825551509857178,
      "rewards/rejected": -0.2426624447107315,
      "step": 957
    },
    {
      "epoch": 1.5090622537431049,
      "grad_norm": 0.2311834692955017,
      "learning_rate": 6.955739925275963e-07,
      "log_odds_chosen": 1.742072343826294,
      "log_odds_ratio": -0.1742241382598877,
      "logits/chosen": -0.7597809433937073,
      "logits/rejected": -1.3029876947402954,
      "logps/chosen": -1.358821153640747,
      "logps/rejected": -2.86045503616333,
      "loss": 1.394,
      "nll_loss": 1.37660551071167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13588210940361023,
      "rewards/margins": 0.1501634120941162,
      "rewards/rejected": -0.28604552149772644,
      "step": 958
    },
    {
      "epoch": 1.5106382978723403,
      "grad_norm": 0.2305707037448883,
      "learning_rate": 6.914119266077354e-07,
      "log_odds_chosen": 1.5131490230560303,
      "log_odds_ratio": -0.20128558576107025,
      "logits/chosen": -0.6181042194366455,
      "logits/rejected": -1.4433350563049316,
      "logps/chosen": -1.4903432130813599,
      "logps/rejected": -2.809812307357788,
      "loss": 1.5219,
      "nll_loss": 1.5018187761306763,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149034321308136,
      "rewards/margins": 0.13194690644741058,
      "rewards/rejected": -0.28098124265670776,
      "step": 959
    },
    {
      "epoch": 1.5122143420015761,
      "grad_norm": 0.2355073243379593,
      "learning_rate": 6.872597464457397e-07,
      "log_odds_chosen": 1.302057147026062,
      "log_odds_ratio": -0.25287219882011414,
      "logits/chosen": -0.6811515688896179,
      "logits/rejected": -1.1880065202713013,
      "logps/chosen": -1.3721972703933716,
      "logps/rejected": -2.470228672027588,
      "loss": 1.3973,
      "nll_loss": 1.3720359802246094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13721974194049835,
      "rewards/margins": 0.10980312526226044,
      "rewards/rejected": -0.24702288210391998,
      "step": 960
    },
    {
      "epoch": 1.5137903861308115,
      "grad_norm": 0.24247051775455475,
      "learning_rate": 6.831174834093476e-07,
      "log_odds_chosen": 1.3397067785263062,
      "log_odds_ratio": -0.24190571904182434,
      "logits/chosen": -0.7289486527442932,
      "logits/rejected": -1.2343394756317139,
      "logps/chosen": -1.3921606540679932,
      "logps/rejected": -2.530585527420044,
      "loss": 1.4192,
      "nll_loss": 1.3950482606887817,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1392160803079605,
      "rewards/margins": 0.11384247988462448,
      "rewards/rejected": -0.2530585825443268,
      "step": 961
    },
    {
      "epoch": 1.5153664302600474,
      "grad_norm": 0.22678187489509583,
      "learning_rate": 6.789851687913784e-07,
      "log_odds_chosen": 1.1686012744903564,
      "log_odds_ratio": -0.2919328808784485,
      "logits/chosen": -0.7605901956558228,
      "logits/rejected": -1.3269751071929932,
      "logps/chosen": -1.345978021621704,
      "logps/rejected": -2.326450824737549,
      "loss": 1.3833,
      "nll_loss": 1.3540993928909302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13459782302379608,
      "rewards/margins": 0.09804728627204895,
      "rewards/rejected": -0.23264509439468384,
      "step": 962
    },
    {
      "epoch": 1.5169424743892828,
      "grad_norm": 0.2213265746831894,
      "learning_rate": 6.748628338094937e-07,
      "log_odds_chosen": 1.5603015422821045,
      "log_odds_ratio": -0.1977100670337677,
      "logits/chosen": -0.6883010268211365,
      "logits/rejected": -1.2167181968688965,
      "logps/chosen": -1.3591656684875488,
      "logps/rejected": -2.6891238689422607,
      "loss": 1.3918,
      "nll_loss": 1.3720567226409912,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1359165757894516,
      "rewards/margins": 0.1329958438873291,
      "rewards/rejected": -0.2689124047756195,
      "step": 963
    },
    {
      "epoch": 1.5185185185185186,
      "grad_norm": 0.22037872672080994,
      "learning_rate": 6.707505096059662e-07,
      "log_odds_chosen": 1.5061944723129272,
      "log_odds_ratio": -0.21080340445041656,
      "logits/chosen": -0.6917910575866699,
      "logits/rejected": -1.2615797519683838,
      "logps/chosen": -1.3713548183441162,
      "logps/rejected": -2.6624112129211426,
      "loss": 1.4118,
      "nll_loss": 1.3907363414764404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13713549077510834,
      "rewards/margins": 0.12910562753677368,
      "rewards/rejected": -0.2662411332130432,
      "step": 964
    },
    {
      "epoch": 1.520094562647754,
      "grad_norm": 0.22575455904006958,
      "learning_rate": 6.666482272474412e-07,
      "log_odds_chosen": 1.4216190576553345,
      "log_odds_ratio": -0.22107170522212982,
      "logits/chosen": -0.7077692747116089,
      "logits/rejected": -1.3194130659103394,
      "logps/chosen": -1.4198875427246094,
      "logps/rejected": -2.63924241065979,
      "loss": 1.4491,
      "nll_loss": 1.4270037412643433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14198875427246094,
      "rewards/margins": 0.12193548679351807,
      "rewards/rejected": -0.263924241065979,
      "step": 965
    },
    {
      "epoch": 1.5216706067769898,
      "grad_norm": 0.23184379935264587,
      "learning_rate": 6.625560177247023e-07,
      "log_odds_chosen": 1.4451392889022827,
      "log_odds_ratio": -0.21741940081119537,
      "logits/chosen": -0.7171230912208557,
      "logits/rejected": -1.2942519187927246,
      "logps/chosen": -1.516118049621582,
      "logps/rejected": -2.7726638317108154,
      "loss": 1.5376,
      "nll_loss": 1.515808343887329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1516118198633194,
      "rewards/margins": 0.12565456330776215,
      "rewards/rejected": -0.27726638317108154,
      "step": 966
    },
    {
      "epoch": 1.5232466509062252,
      "grad_norm": 0.2470589429140091,
      "learning_rate": 6.584739119524383e-07,
      "log_odds_chosen": 1.5914825201034546,
      "log_odds_ratio": -0.2017475962638855,
      "logits/chosen": -0.6873888969421387,
      "logits/rejected": -1.3693267107009888,
      "logps/chosen": -1.3818203210830688,
      "logps/rejected": -2.7532527446746826,
      "loss": 1.3962,
      "nll_loss": 1.3760173320770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13818204402923584,
      "rewards/margins": 0.13714322447776794,
      "rewards/rejected": -0.2753252685070038,
      "step": 967
    },
    {
      "epoch": 1.524822695035461,
      "grad_norm": 0.22959762811660767,
      "learning_rate": 6.544019407690077e-07,
      "log_odds_chosen": 1.301601529121399,
      "log_odds_ratio": -0.25241389870643616,
      "logits/chosen": -0.7045775651931763,
      "logits/rejected": -1.1604735851287842,
      "logps/chosen": -1.4770642518997192,
      "logps/rejected": -2.600731611251831,
      "loss": 1.4971,
      "nll_loss": 1.471899390220642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14770641922950745,
      "rewards/margins": 0.11236675828695297,
      "rewards/rejected": -0.260073184967041,
      "step": 968
    },
    {
      "epoch": 1.5263987391646965,
      "grad_norm": 0.23091484606266022,
      "learning_rate": 6.503401349362083e-07,
      "log_odds_chosen": 1.348669409751892,
      "log_odds_ratio": -0.24010084569454193,
      "logits/chosen": -0.826965868473053,
      "logits/rejected": -1.348397970199585,
      "logps/chosen": -1.5049937963485718,
      "logps/rejected": -2.674851894378662,
      "loss": 1.5295,
      "nll_loss": 1.5055359601974487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1504993736743927,
      "rewards/margins": 0.11698581278324127,
      "rewards/rejected": -0.26748523116111755,
      "step": 969
    },
    {
      "epoch": 1.5279747832939323,
      "grad_norm": 0.22754135727882385,
      "learning_rate": 6.462885251390433e-07,
      "log_odds_chosen": 1.3173844814300537,
      "log_odds_ratio": -0.24255065619945526,
      "logits/chosen": -0.7769299149513245,
      "logits/rejected": -1.1924445629119873,
      "logps/chosen": -1.494451880455017,
      "logps/rejected": -2.6321396827697754,
      "loss": 1.5081,
      "nll_loss": 1.4838593006134033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14944520592689514,
      "rewards/margins": 0.11376878619194031,
      "rewards/rejected": -0.26321399211883545,
      "step": 970
    },
    {
      "epoch": 1.5295508274231677,
      "grad_norm": 0.21151317656040192,
      "learning_rate": 6.422471419854898e-07,
      "log_odds_chosen": 1.4928009510040283,
      "log_odds_ratio": -0.20881310105323792,
      "logits/chosen": -0.6864855289459229,
      "logits/rejected": -1.2554219961166382,
      "logps/chosen": -1.3271548748016357,
      "logps/rejected": -2.587578773498535,
      "loss": 1.3607,
      "nll_loss": 1.339848518371582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13271550834178925,
      "rewards/margins": 0.12604239583015442,
      "rewards/rejected": -0.25875788927078247,
      "step": 971
    },
    {
      "epoch": 1.5311268715524036,
      "grad_norm": 0.24027584493160248,
      "learning_rate": 6.382160160062662e-07,
      "log_odds_chosen": 1.699940800666809,
      "log_odds_ratio": -0.18513135612010956,
      "logits/chosen": -0.6840096116065979,
      "logits/rejected": -1.315608024597168,
      "logps/chosen": -1.4163511991500854,
      "logps/rejected": -2.899308681488037,
      "loss": 1.4462,
      "nll_loss": 1.4276946783065796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14163513481616974,
      "rewards/margins": 0.14829576015472412,
      "rewards/rejected": -0.2899308502674103,
      "step": 972
    },
    {
      "epoch": 1.532702915681639,
      "grad_norm": 0.2348189353942871,
      "learning_rate": 6.341951776546044e-07,
      "log_odds_chosen": 1.3805712461471558,
      "log_odds_ratio": -0.23320038616657257,
      "logits/chosen": -0.6572248339653015,
      "logits/rejected": -1.2972402572631836,
      "logps/chosen": -1.42389976978302,
      "logps/rejected": -2.6066274642944336,
      "loss": 1.4509,
      "nll_loss": 1.427539587020874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14238998293876648,
      "rewards/margins": 0.11827276647090912,
      "rewards/rejected": -0.2606627643108368,
      "step": 973
    },
    {
      "epoch": 1.5342789598108748,
      "grad_norm": 0.22137556970119476,
      "learning_rate": 6.301846573060177e-07,
      "log_odds_chosen": 1.202012062072754,
      "log_odds_ratio": -0.27059850096702576,
      "logits/chosen": -0.7582395076751709,
      "logits/rejected": -1.1575336456298828,
      "logps/chosen": -1.3741958141326904,
      "logps/rejected": -2.384232521057129,
      "loss": 1.417,
      "nll_loss": 1.3899447917938232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13741959631443024,
      "rewards/margins": 0.10100366175174713,
      "rewards/rejected": -0.23842324316501617,
      "step": 974
    },
    {
      "epoch": 1.5358550039401102,
      "grad_norm": 0.22573129832744598,
      "learning_rate": 6.261844852580722e-07,
      "log_odds_chosen": 1.4604352712631226,
      "log_odds_ratio": -0.23023870587348938,
      "logits/chosen": -0.7485328316688538,
      "logits/rejected": -1.0415393114089966,
      "logps/chosen": -1.3264422416687012,
      "logps/rejected": -2.569178581237793,
      "loss": 1.3649,
      "nll_loss": 1.341881513595581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13264423608779907,
      "rewards/margins": 0.1242736279964447,
      "rewards/rejected": -0.2569178640842438,
      "step": 975
    },
    {
      "epoch": 1.537431048069346,
      "grad_norm": 0.21874772012233734,
      "learning_rate": 6.221946917301561e-07,
      "log_odds_chosen": 1.4869847297668457,
      "log_odds_ratio": -0.2150578796863556,
      "logits/chosen": -0.6522708535194397,
      "logits/rejected": -1.281939148902893,
      "logps/chosen": -1.4193254709243774,
      "logps/rejected": -2.6987578868865967,
      "loss": 1.4496,
      "nll_loss": 1.4281158447265625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14193254709243774,
      "rewards/margins": 0.1279432326555252,
      "rewards/rejected": -0.26987576484680176,
      "step": 976
    },
    {
      "epoch": 1.5390070921985815,
      "grad_norm": 0.22851106524467468,
      "learning_rate": 6.182153068632545e-07,
      "log_odds_chosen": 1.4246211051940918,
      "log_odds_ratio": -0.22736258804798126,
      "logits/chosen": -0.6819384694099426,
      "logits/rejected": -1.3521054983139038,
      "logps/chosen": -1.4322878122329712,
      "logps/rejected": -2.6613006591796875,
      "loss": 1.475,
      "nll_loss": 1.4522185325622559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14322878420352936,
      "rewards/margins": 0.1229013055562973,
      "rewards/rejected": -0.26613008975982666,
      "step": 977
    },
    {
      "epoch": 1.5405831363278173,
      "grad_norm": 0.23759952187538147,
      "learning_rate": 6.142463607197197e-07,
      "log_odds_chosen": 1.4491053819656372,
      "log_odds_ratio": -0.22233818471431732,
      "logits/chosen": -0.7541925311088562,
      "logits/rejected": -1.3975563049316406,
      "logps/chosen": -1.4454472064971924,
      "logps/rejected": -2.699002265930176,
      "loss": 1.4603,
      "nll_loss": 1.4381098747253418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14454472064971924,
      "rewards/margins": 0.12535551190376282,
      "rewards/rejected": -0.26990023255348206,
      "step": 978
    },
    {
      "epoch": 1.5421591804570527,
      "grad_norm": 0.21871303021907806,
      "learning_rate": 6.102878832830431e-07,
      "log_odds_chosen": 1.4103882312774658,
      "log_odds_ratio": -0.2234984040260315,
      "logits/chosen": -0.7719005346298218,
      "logits/rejected": -1.4227514266967773,
      "logps/chosen": -1.4454764127731323,
      "logps/rejected": -2.6616768836975098,
      "loss": 1.4811,
      "nll_loss": 1.4587870836257935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14454767107963562,
      "rewards/margins": 0.12162001430988312,
      "rewards/rejected": -0.26616767048835754,
      "step": 979
    },
    {
      "epoch": 1.5437352245862885,
      "grad_norm": 0.21796202659606934,
      "learning_rate": 6.063399044576316e-07,
      "log_odds_chosen": 1.4727609157562256,
      "log_odds_ratio": -0.2138061225414276,
      "logits/chosen": -0.719465970993042,
      "logits/rejected": -1.2844264507293701,
      "logps/chosen": -1.3865543603897095,
      "logps/rejected": -2.6444029808044434,
      "loss": 1.4207,
      "nll_loss": 1.3993679285049438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13865543901920319,
      "rewards/margins": 0.12578488886356354,
      "rewards/rejected": -0.2644403278827667,
      "step": 980
    },
    {
      "epoch": 1.545311268715524,
      "grad_norm": 0.22771282494068146,
      "learning_rate": 6.024024540685802e-07,
      "log_odds_chosen": 1.4691410064697266,
      "log_odds_ratio": -0.22497592866420746,
      "logits/chosen": -0.6690123677253723,
      "logits/rejected": -1.1946911811828613,
      "logps/chosen": -1.3911912441253662,
      "logps/rejected": -2.6496362686157227,
      "loss": 1.4268,
      "nll_loss": 1.4042632579803467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13911913335323334,
      "rewards/margins": 0.12584452331066132,
      "rewards/rejected": -0.26496365666389465,
      "step": 981
    },
    {
      "epoch": 1.5468873128447598,
      "grad_norm": 0.23051610589027405,
      "learning_rate": 5.984755618614443e-07,
      "log_odds_chosen": 1.3554091453552246,
      "log_odds_ratio": -0.2381577342748642,
      "logits/chosen": -0.7894393801689148,
      "logits/rejected": -1.365567922592163,
      "logps/chosen": -1.427924633026123,
      "logps/rejected": -2.58760666847229,
      "loss": 1.4586,
      "nll_loss": 1.4347411394119263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1427924633026123,
      "rewards/margins": 0.11596819013357162,
      "rewards/rejected": -0.2587606608867645,
      "step": 982
    },
    {
      "epoch": 1.5484633569739952,
      "grad_norm": 0.2140674889087677,
      "learning_rate": 5.945592575020199e-07,
      "log_odds_chosen": 1.4203743934631348,
      "log_odds_ratio": -0.23188234865665436,
      "logits/chosen": -0.5940582752227783,
      "logits/rejected": -1.1886006593704224,
      "logps/chosen": -1.3997769355773926,
      "logps/rejected": -2.616659164428711,
      "loss": 1.4411,
      "nll_loss": 1.4178688526153564,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13997769355773926,
      "rewards/margins": 0.12168824672698975,
      "rewards/rejected": -0.261665940284729,
      "step": 983
    },
    {
      "epoch": 1.550039401103231,
      "grad_norm": 0.2479635775089264,
      "learning_rate": 5.90653570576116e-07,
      "log_odds_chosen": 1.7492060661315918,
      "log_odds_ratio": -0.17725133895874023,
      "logits/chosen": -0.809583306312561,
      "logits/rejected": -1.325620412826538,
      "logps/chosen": -1.4484314918518066,
      "logps/rejected": -2.9866626262664795,
      "loss": 1.4714,
      "nll_loss": 1.4536775350570679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14484316110610962,
      "rewards/margins": 0.153823122382164,
      "rewards/rejected": -0.2986662983894348,
      "step": 984
    },
    {
      "epoch": 1.5516154452324664,
      "grad_norm": 0.22948485612869263,
      "learning_rate": 5.867585305893315e-07,
      "log_odds_chosen": 1.404991865158081,
      "log_odds_ratio": -0.2338658720254898,
      "logits/chosen": -0.7800405621528625,
      "logits/rejected": -1.1818283796310425,
      "logps/chosen": -1.3705575466156006,
      "logps/rejected": -2.5682461261749268,
      "loss": 1.4152,
      "nll_loss": 1.3918144702911377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13705576956272125,
      "rewards/margins": 0.11976885795593262,
      "rewards/rejected": -0.2568246126174927,
      "step": 985
    },
    {
      "epoch": 1.5531914893617023,
      "grad_norm": 0.21489939093589783,
      "learning_rate": 5.828741669668337e-07,
      "log_odds_chosen": 1.6612565517425537,
      "log_odds_ratio": -0.17742058634757996,
      "logits/chosen": -0.6967443227767944,
      "logits/rejected": -1.3925031423568726,
      "logps/chosen": -1.4220824241638184,
      "logps/rejected": -2.865713596343994,
      "loss": 1.4395,
      "nll_loss": 1.4218072891235352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1422082483768463,
      "rewards/margins": 0.144363135099411,
      "rewards/rejected": -0.2865713834762573,
      "step": 986
    },
    {
      "epoch": 1.5547675334909377,
      "grad_norm": 0.21180802583694458,
      "learning_rate": 5.790005090531333e-07,
      "log_odds_chosen": 1.603518009185791,
      "log_odds_ratio": -0.20220975577831268,
      "logits/chosen": -0.7544563412666321,
      "logits/rejected": -1.3845782279968262,
      "logps/chosen": -1.4346991777420044,
      "logps/rejected": -2.827078104019165,
      "loss": 1.4597,
      "nll_loss": 1.439489483833313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1434699147939682,
      "rewards/margins": 0.1392379105091095,
      "rewards/rejected": -0.2827078402042389,
      "step": 987
    },
    {
      "epoch": 1.5563435776201735,
      "grad_norm": 0.21749882400035858,
      "learning_rate": 5.751375861118666e-07,
      "log_odds_chosen": 1.5675867795944214,
      "log_odds_ratio": -0.2174535095691681,
      "logits/chosen": -0.7571312189102173,
      "logits/rejected": -1.2842272520065308,
      "logps/chosen": -1.447562575340271,
      "logps/rejected": -2.817909002304077,
      "loss": 1.4608,
      "nll_loss": 1.4390151500701904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1447562575340271,
      "rewards/margins": 0.13703465461730957,
      "rewards/rejected": -0.28179091215133667,
      "step": 988
    },
    {
      "epoch": 1.557919621749409,
      "grad_norm": 0.23365335166454315,
      "learning_rate": 5.712854273255707e-07,
      "log_odds_chosen": 1.3372693061828613,
      "log_odds_ratio": -0.23750026524066925,
      "logits/chosen": -0.6249842643737793,
      "logits/rejected": -1.3808866739273071,
      "logps/chosen": -1.4317543506622314,
      "logps/rejected": -2.5761537551879883,
      "loss": 1.475,
      "nll_loss": 1.451277732849121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14317545294761658,
      "rewards/margins": 0.1144399493932724,
      "rewards/rejected": -0.25761541724205017,
      "step": 989
    },
    {
      "epoch": 1.5594956658786447,
      "grad_norm": 0.2097553014755249,
      "learning_rate": 5.674440617954659e-07,
      "log_odds_chosen": 1.4624303579330444,
      "log_odds_ratio": -0.2183232307434082,
      "logits/chosen": -0.7617121934890747,
      "logits/rejected": -1.1527690887451172,
      "logps/chosen": -1.3377940654754639,
      "logps/rejected": -2.5756096839904785,
      "loss": 1.3648,
      "nll_loss": 1.3429436683654785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13377942144870758,
      "rewards/margins": 0.12378155440092087,
      "rewards/rejected": -0.25756096839904785,
      "step": 990
    },
    {
      "epoch": 1.5610717100078801,
      "grad_norm": 0.2206483632326126,
      "learning_rate": 5.63613518541234e-07,
      "log_odds_chosen": 1.4063078165054321,
      "log_odds_ratio": -0.24835076928138733,
      "logits/chosen": -0.66551274061203,
      "logits/rejected": -1.2632298469543457,
      "logps/chosen": -1.3813774585723877,
      "logps/rejected": -2.587003707885742,
      "loss": 1.4271,
      "nll_loss": 1.40231192111969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13813774287700653,
      "rewards/margins": 0.12056262791156769,
      "rewards/rejected": -0.2587003707885742,
      "step": 991
    },
    {
      "epoch": 1.562647754137116,
      "grad_norm": 0.23103219270706177,
      "learning_rate": 5.597938265007993e-07,
      "log_odds_chosen": 1.5438990592956543,
      "log_odds_ratio": -0.19840553402900696,
      "logits/chosen": -0.7387202382087708,
      "logits/rejected": -1.4001561403274536,
      "logps/chosen": -1.3714319467544556,
      "logps/rejected": -2.6893649101257324,
      "loss": 1.3994,
      "nll_loss": 1.3796077966690063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13714320957660675,
      "rewards/margins": 0.1317933201789856,
      "rewards/rejected": -0.26893651485443115,
      "step": 992
    },
    {
      "epoch": 1.5642237982663514,
      "grad_norm": 0.2439792901277542,
      "learning_rate": 5.559850145301106e-07,
      "log_odds_chosen": 1.2656465768814087,
      "log_odds_ratio": -0.25876447558403015,
      "logits/chosen": -0.6946025490760803,
      "logits/rejected": -1.1637576818466187,
      "logps/chosen": -1.4052958488464355,
      "logps/rejected": -2.479584217071533,
      "loss": 1.4477,
      "nll_loss": 1.421870231628418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1405295878648758,
      "rewards/margins": 0.10742883384227753,
      "rewards/rejected": -0.24795840680599213,
      "step": 993
    },
    {
      "epoch": 1.5657998423955872,
      "grad_norm": 0.2185717076063156,
      "learning_rate": 5.521871114029233e-07,
      "log_odds_chosen": 1.4958091974258423,
      "log_odds_ratio": -0.205339252948761,
      "logits/chosen": -0.7529298663139343,
      "logits/rejected": -1.3294923305511475,
      "logps/chosen": -1.330001950263977,
      "logps/rejected": -2.5966074466705322,
      "loss": 1.38,
      "nll_loss": 1.359473466873169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1330001950263977,
      "rewards/margins": 0.1266605406999588,
      "rewards/rejected": -0.2596607506275177,
      "step": 994
    },
    {
      "epoch": 1.5673758865248226,
      "grad_norm": 0.2557937204837799,
      "learning_rate": 5.484001458105823e-07,
      "log_odds_chosen": 1.3082319498062134,
      "log_odds_ratio": -0.2540491819381714,
      "logits/chosen": -0.8008450865745544,
      "logits/rejected": -1.089009165763855,
      "logps/chosen": -1.4155988693237305,
      "logps/rejected": -2.5304324626922607,
      "loss": 1.4467,
      "nll_loss": 1.4212942123413086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.141559898853302,
      "rewards/margins": 0.11148333549499512,
      "rewards/rejected": -0.2530432641506195,
      "step": 995
    },
    {
      "epoch": 1.5689519306540585,
      "grad_norm": 0.6362001895904541,
      "learning_rate": 5.446241463618027e-07,
      "log_odds_chosen": 1.7546403408050537,
      "log_odds_ratio": -0.21400946378707886,
      "logits/chosen": -0.7408161759376526,
      "logits/rejected": -1.1835956573486328,
      "logps/chosen": -1.4328913688659668,
      "logps/rejected": -2.9814467430114746,
      "loss": 1.4631,
      "nll_loss": 1.4416970014572144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14328914880752563,
      "rewards/margins": 0.15485553443431854,
      "rewards/rejected": -0.298144668340683,
      "step": 996
    },
    {
      "epoch": 1.5705279747832939,
      "grad_norm": 0.22958840429782867,
      "learning_rate": 5.40859141582457e-07,
      "log_odds_chosen": 1.6916431188583374,
      "log_odds_ratio": -0.17420552670955658,
      "logits/chosen": -0.7841975092887878,
      "logits/rejected": -1.2954723834991455,
      "logps/chosen": -1.3005943298339844,
      "logps/rejected": -2.7395286560058594,
      "loss": 1.3289,
      "nll_loss": 1.311484694480896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13005945086479187,
      "rewards/margins": 0.14389340579509735,
      "rewards/rejected": -0.273952841758728,
      "step": 997
    },
    {
      "epoch": 1.5721040189125297,
      "grad_norm": 0.25778648257255554,
      "learning_rate": 5.371051599153582e-07,
      "log_odds_chosen": 1.2076324224472046,
      "log_odds_ratio": -0.2855122685432434,
      "logits/chosen": -0.6911635398864746,
      "logits/rejected": -1.0361592769622803,
      "logps/chosen": -1.3604072332382202,
      "logps/rejected": -2.3706226348876953,
      "loss": 1.4027,
      "nll_loss": 1.3741856813430786,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13604071736335754,
      "rewards/margins": 0.10102155059576035,
      "rewards/rejected": -0.23706229031085968,
      "step": 998
    },
    {
      "epoch": 1.573680063041765,
      "grad_norm": 0.22509822249412537,
      "learning_rate": 5.333622297200449e-07,
      "log_odds_chosen": 1.2975940704345703,
      "log_odds_ratio": -0.2442600429058075,
      "logits/chosen": -0.7237396240234375,
      "logits/rejected": -1.1229147911071777,
      "logps/chosen": -1.3554954528808594,
      "logps/rejected": -2.438605785369873,
      "loss": 1.4073,
      "nll_loss": 1.3828771114349365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13554954528808594,
      "rewards/margins": 0.10831101983785629,
      "rewards/rejected": -0.24386055767536163,
      "step": 999
    },
    {
      "epoch": 1.575256107171001,
      "grad_norm": 0.2101544886827469,
      "learning_rate": 5.296303792725676e-07,
      "log_odds_chosen": 1.6225467920303345,
      "log_odds_ratio": -0.18489757180213928,
      "logits/chosen": -0.7715501189231873,
      "logits/rejected": -1.431501030921936,
      "logps/chosen": -1.4694812297821045,
      "logps/rejected": -2.885326623916626,
      "loss": 1.4868,
      "nll_loss": 1.4682875871658325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14694812893867493,
      "rewards/margins": 0.14158453047275543,
      "rewards/rejected": -0.28853267431259155,
      "step": 1000
    },
    {
      "epoch": 1.5768321513002364,
      "grad_norm": 0.23330575227737427,
      "learning_rate": 5.259096367652725e-07,
      "log_odds_chosen": 1.5761932134628296,
      "log_odds_ratio": -0.20117153227329254,
      "logits/chosen": -0.7493457198143005,
      "logits/rejected": -1.343841314315796,
      "logps/chosen": -1.4189023971557617,
      "logps/rejected": -2.7833783626556396,
      "loss": 1.449,
      "nll_loss": 1.428913950920105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1418902426958084,
      "rewards/margins": 0.13644760847091675,
      "rewards/rejected": -0.27833783626556396,
      "step": 1001
    },
    {
      "epoch": 1.578408195429472,
      "grad_norm": 0.22028324007987976,
      "learning_rate": 5.222000303065927e-07,
      "log_odds_chosen": 1.5242966413497925,
      "log_odds_ratio": -0.20954249799251556,
      "logits/chosen": -0.682687520980835,
      "logits/rejected": -1.4307681322097778,
      "logps/chosen": -1.4024947881698608,
      "logps/rejected": -2.7160215377807617,
      "loss": 1.4357,
      "nll_loss": 1.414715051651001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14024947583675385,
      "rewards/margins": 0.13135266304016113,
      "rewards/rejected": -0.27160215377807617,
      "step": 1002
    },
    {
      "epoch": 1.5799842395587076,
      "grad_norm": 0.23311971127986908,
      "learning_rate": 5.185015879208335e-07,
      "log_odds_chosen": 1.4669800996780396,
      "log_odds_ratio": -0.21098636090755463,
      "logits/chosen": -0.748795211315155,
      "logits/rejected": -1.2435390949249268,
      "logps/chosen": -1.4002372026443481,
      "logps/rejected": -2.6504364013671875,
      "loss": 1.4389,
      "nll_loss": 1.4178457260131836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14002372324466705,
      "rewards/margins": 0.12501990795135498,
      "rewards/rejected": -0.26504361629486084,
      "step": 1003
    },
    {
      "epoch": 1.5815602836879432,
      "grad_norm": 0.2261989712715149,
      "learning_rate": 5.148143375479601e-07,
      "log_odds_chosen": 1.512619972229004,
      "log_odds_ratio": -0.20081375539302826,
      "logits/chosen": -0.7466608285903931,
      "logits/rejected": -1.286081075668335,
      "logps/chosen": -1.4552388191223145,
      "logps/rejected": -2.7657721042633057,
      "loss": 1.4701,
      "nll_loss": 1.4499703645706177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14552389085292816,
      "rewards/margins": 0.13105329871177673,
      "rewards/rejected": -0.2765772044658661,
      "step": 1004
    },
    {
      "epoch": 1.5831363278171788,
      "grad_norm": 0.22575624287128448,
      "learning_rate": 5.111383070433887e-07,
      "log_odds_chosen": 1.627588152885437,
      "log_odds_ratio": -0.1908290833234787,
      "logits/chosen": -0.7101742029190063,
      "logits/rejected": -1.3571864366531372,
      "logps/chosen": -1.4556803703308105,
      "logps/rejected": -2.878706932067871,
      "loss": 1.4785,
      "nll_loss": 1.4593770503997803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14556805789470673,
      "rewards/margins": 0.14230266213417053,
      "rewards/rejected": -0.28787073493003845,
      "step": 1005
    },
    {
      "epoch": 1.5847123719464145,
      "grad_norm": 0.2494867891073227,
      "learning_rate": 5.074735241777733e-07,
      "log_odds_chosen": 1.151566505432129,
      "log_odds_ratio": -0.2852984666824341,
      "logits/chosen": -0.677973210811615,
      "logits/rejected": -1.2236948013305664,
      "logps/chosen": -1.5575027465820312,
      "logps/rejected": -2.5560998916625977,
      "loss": 1.5632,
      "nll_loss": 1.5346319675445557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15575027465820312,
      "rewards/margins": 0.09985969960689545,
      "rewards/rejected": -0.2556099593639374,
      "step": 1006
    },
    {
      "epoch": 1.58628841607565,
      "grad_norm": 0.2400100827217102,
      "learning_rate": 5.038200166367978e-07,
      "log_odds_chosen": 1.5715808868408203,
      "log_odds_ratio": -0.20167407393455505,
      "logits/chosen": -0.8228264451026917,
      "logits/rejected": -1.2758487462997437,
      "logps/chosen": -1.3895665407180786,
      "logps/rejected": -2.7419888973236084,
      "loss": 1.4215,
      "nll_loss": 1.401340126991272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13895666599273682,
      "rewards/margins": 0.13524223864078522,
      "rewards/rejected": -0.2741989195346832,
      "step": 1007
    },
    {
      "epoch": 1.5878644602048857,
      "grad_norm": 0.2240261435508728,
      "learning_rate": 5.00177812020967e-07,
      "log_odds_chosen": 1.4148606061935425,
      "log_odds_ratio": -0.2230863720178604,
      "logits/chosen": -0.7215375900268555,
      "logits/rejected": -1.2683002948760986,
      "logps/chosen": -1.4830286502838135,
      "logps/rejected": -2.7104544639587402,
      "loss": 1.5191,
      "nll_loss": 1.496785283088684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1483028531074524,
      "rewards/margins": 0.12274259328842163,
      "rewards/rejected": -0.271045446395874,
      "step": 1008
    },
    {
      "epoch": 1.5894405043341213,
      "grad_norm": 0.2438661903142929,
      "learning_rate": 4.965469378453979e-07,
      "log_odds_chosen": 1.4171829223632812,
      "log_odds_ratio": -0.2529681921005249,
      "logits/chosen": -0.80870521068573,
      "logits/rejected": -1.067436933517456,
      "logps/chosen": -1.3849254846572876,
      "logps/rejected": -2.5935986042022705,
      "loss": 1.4238,
      "nll_loss": 1.3985395431518555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13849255442619324,
      "rewards/margins": 0.12086734175682068,
      "rewards/rejected": -0.2593598961830139,
      "step": 1009
    },
    {
      "epoch": 1.591016548463357,
      "grad_norm": 0.22861014306545258,
      "learning_rate": 4.929274215396091e-07,
      "log_odds_chosen": 1.4019694328308105,
      "log_odds_ratio": -0.22800008952617645,
      "logits/chosen": -0.8238511085510254,
      "logits/rejected": -1.2801072597503662,
      "logps/chosen": -1.4422610998153687,
      "logps/rejected": -2.6428885459899902,
      "loss": 1.4644,
      "nll_loss": 1.4416420459747314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14422613382339478,
      "rewards/margins": 0.12006273120641708,
      "rewards/rejected": -0.26428887248039246,
      "step": 1010
    },
    {
      "epoch": 1.5925925925925926,
      "grad_norm": 0.22769996523857117,
      "learning_rate": 4.893192904473182e-07,
      "log_odds_chosen": 1.0523293018341064,
      "log_odds_ratio": -0.3144485354423523,
      "logits/chosen": -0.7861213088035583,
      "logits/rejected": -0.923714280128479,
      "logps/chosen": -1.4621021747589111,
      "logps/rejected": -2.3543262481689453,
      "loss": 1.5058,
      "nll_loss": 1.4743618965148926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1462102085351944,
      "rewards/margins": 0.08922242373228073,
      "rewards/rejected": -0.23543265461921692,
      "step": 1011
    },
    {
      "epoch": 1.5941686367218282,
      "grad_norm": 0.2346268892288208,
      "learning_rate": 4.85722571826233e-07,
      "log_odds_chosen": 1.4252359867095947,
      "log_odds_ratio": -0.2357722967863083,
      "logits/chosen": -0.7936029434204102,
      "logits/rejected": -1.2412831783294678,
      "logps/chosen": -1.3709121942520142,
      "logps/rejected": -2.589566230773926,
      "loss": 1.4095,
      "nll_loss": 1.385927677154541,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13709121942520142,
      "rewards/margins": 0.121865414083004,
      "rewards/rejected": -0.258956640958786,
      "step": 1012
    },
    {
      "epoch": 1.5957446808510638,
      "grad_norm": 0.2467648684978485,
      "learning_rate": 4.821372928478433e-07,
      "log_odds_chosen": 1.342909574508667,
      "log_odds_ratio": -0.24285058677196503,
      "logits/chosen": -0.7096128463745117,
      "logits/rejected": -1.2679332494735718,
      "logps/chosen": -1.4811835289001465,
      "logps/rejected": -2.6380019187927246,
      "loss": 1.5069,
      "nll_loss": 1.482595682144165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14811836183071136,
      "rewards/margins": 0.11568183451890945,
      "rewards/rejected": -0.2638002038002014,
      "step": 1013
    },
    {
      "epoch": 1.5973207249802994,
      "grad_norm": 0.2224494367837906,
      "learning_rate": 4.785634805972201e-07,
      "log_odds_chosen": 1.260195016860962,
      "log_odds_ratio": -0.266611784696579,
      "logits/chosen": -0.7116222977638245,
      "logits/rejected": -1.2118539810180664,
      "logps/chosen": -1.4194812774658203,
      "logps/rejected": -2.4916653633117676,
      "loss": 1.4534,
      "nll_loss": 1.4267396926879883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14194811880588531,
      "rewards/margins": 0.10721839964389801,
      "rewards/rejected": -0.24916653335094452,
      "step": 1014
    },
    {
      "epoch": 1.598896769109535,
      "grad_norm": 0.22571000456809998,
      "learning_rate": 4.750011620728085e-07,
      "log_odds_chosen": 1.4394540786743164,
      "log_odds_ratio": -0.2195713073015213,
      "logits/chosen": -0.7642476558685303,
      "logits/rejected": -1.1760566234588623,
      "logps/chosen": -1.4382249116897583,
      "logps/rejected": -2.67830491065979,
      "loss": 1.4642,
      "nll_loss": 1.4421939849853516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14382250607013702,
      "rewards/margins": 0.12400800734758377,
      "rewards/rejected": -0.267830491065979,
      "step": 1015
    },
    {
      "epoch": 1.6004728132387707,
      "grad_norm": 0.24502716958522797,
      "learning_rate": 4.714503641862224e-07,
      "log_odds_chosen": 1.4202399253845215,
      "log_odds_ratio": -0.22183164954185486,
      "logits/chosen": -0.7135217189788818,
      "logits/rejected": -1.2811405658721924,
      "logps/chosen": -1.4653769731521606,
      "logps/rejected": -2.6892154216766357,
      "loss": 1.4898,
      "nll_loss": 1.4675753116607666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14653770625591278,
      "rewards/margins": 0.12238381803035736,
      "rewards/rejected": -0.26892152428627014,
      "step": 1016
    },
    {
      "epoch": 1.6020488573680063,
      "grad_norm": 0.23795074224472046,
      "learning_rate": 4.679111137620442e-07,
      "log_odds_chosen": 1.794071912765503,
      "log_odds_ratio": -0.19583582878112793,
      "logits/chosen": -0.8574603796005249,
      "logits/rejected": -1.3071023225784302,
      "logps/chosen": -1.4241650104522705,
      "logps/rejected": -3.002999782562256,
      "loss": 1.437,
      "nll_loss": 1.4174017906188965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14241650700569153,
      "rewards/margins": 0.15788352489471436,
      "rewards/rejected": -0.3003000319004059,
      "step": 1017
    },
    {
      "epoch": 1.603624901497242,
      "grad_norm": 0.2424805462360382,
      "learning_rate": 4.6438343753762055e-07,
      "log_odds_chosen": 1.2536628246307373,
      "log_odds_ratio": -0.26696327328681946,
      "logits/chosen": -0.7689000368118286,
      "logits/rejected": -1.2007453441619873,
      "logps/chosen": -1.4178433418273926,
      "logps/rejected": -2.479580879211426,
      "loss": 1.4321,
      "nll_loss": 1.4053698778152466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14178434014320374,
      "rewards/margins": 0.10617375373840332,
      "rewards/rejected": -0.24795806407928467,
      "step": 1018
    },
    {
      "epoch": 1.6052009456264775,
      "grad_norm": 0.22874854505062103,
      "learning_rate": 4.608673621628609e-07,
      "log_odds_chosen": 1.4414875507354736,
      "log_odds_ratio": -0.21917863190174103,
      "logits/chosen": -0.724556565284729,
      "logits/rejected": -1.269260287284851,
      "logps/chosen": -1.4379795789718628,
      "logps/rejected": -2.678215980529785,
      "loss": 1.4527,
      "nll_loss": 1.430772066116333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14379796385765076,
      "rewards/margins": 0.12402364611625671,
      "rewards/rejected": -0.26782160997390747,
      "step": 1019
    },
    {
      "epoch": 1.6067769897557131,
      "grad_norm": 0.2338285893201828,
      "learning_rate": 4.5736291420003413e-07,
      "log_odds_chosen": 1.463491678237915,
      "log_odds_ratio": -0.22352954745292664,
      "logits/chosen": -0.707510769367218,
      "logits/rejected": -1.1545389890670776,
      "logps/chosen": -1.4270511865615845,
      "logps/rejected": -2.6919546127319336,
      "loss": 1.4589,
      "nll_loss": 1.4364970922470093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14270512759685516,
      "rewards/margins": 0.12649035453796387,
      "rewards/rejected": -0.26919546723365784,
      "step": 1020
    },
    {
      "epoch": 1.6083530338849488,
      "grad_norm": 0.24140799045562744,
      "learning_rate": 4.538701201235713e-07,
      "log_odds_chosen": 1.5078694820404053,
      "log_odds_ratio": -0.21403060853481293,
      "logits/chosen": -0.6843619346618652,
      "logits/rejected": -1.2965511083602905,
      "logps/chosen": -1.3842742443084717,
      "logps/rejected": -2.68135404586792,
      "loss": 1.4146,
      "nll_loss": 1.3932387828826904,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13842743635177612,
      "rewards/margins": 0.12970799207687378,
      "rewards/rejected": -0.2681354284286499,
      "step": 1021
    },
    {
      "epoch": 1.6099290780141844,
      "grad_norm": 0.2213946282863617,
      "learning_rate": 4.5038900631986367e-07,
      "log_odds_chosen": 1.4277207851409912,
      "log_odds_ratio": -0.2292112559080124,
      "logits/chosen": -0.6891590356826782,
      "logits/rejected": -1.2036867141723633,
      "logps/chosen": -1.3631871938705444,
      "logps/rejected": -2.576024055480957,
      "loss": 1.4162,
      "nll_loss": 1.3933242559432983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13631872832775116,
      "rewards/margins": 0.12128369510173798,
      "rewards/rejected": -0.25760242342948914,
      "step": 1022
    },
    {
      "epoch": 1.61150512214342,
      "grad_norm": 0.22965118288993835,
      "learning_rate": 4.4691959908706287e-07,
      "log_odds_chosen": 1.225237250328064,
      "log_odds_ratio": -0.2766522169113159,
      "logits/chosen": -0.7147809863090515,
      "logits/rejected": -1.1857569217681885,
      "logps/chosen": -1.4757550954818726,
      "logps/rejected": -2.5323753356933594,
      "loss": 1.4981,
      "nll_loss": 1.4704796075820923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1475755125284195,
      "rewards/margins": 0.10566201061010361,
      "rewards/rejected": -0.2532375454902649,
      "step": 1023
    },
    {
      "epoch": 1.6130811662726556,
      "grad_norm": 0.20671923458576202,
      "learning_rate": 4.434619246348843e-07,
      "log_odds_chosen": 1.4653304815292358,
      "log_odds_ratio": -0.2296721488237381,
      "logits/chosen": -0.6847785711288452,
      "logits/rejected": -1.3712282180786133,
      "logps/chosen": -1.3745789527893066,
      "logps/rejected": -2.6303200721740723,
      "loss": 1.417,
      "nll_loss": 1.3940684795379639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13745789229869843,
      "rewards/margins": 0.12557412683963776,
      "rewards/rejected": -0.2630320191383362,
      "step": 1024
    },
    {
      "epoch": 1.6146572104018913,
      "grad_norm": 0.2564661502838135,
      "learning_rate": 4.4001600908440564e-07,
      "log_odds_chosen": 1.4309289455413818,
      "log_odds_ratio": -0.22175653278827667,
      "logits/chosen": -0.6458380222320557,
      "logits/rejected": -1.2771897315979004,
      "logps/chosen": -1.4706470966339111,
      "logps/rejected": -2.709660768508911,
      "loss": 1.4876,
      "nll_loss": 1.4654474258422852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470647156238556,
      "rewards/margins": 0.1239013597369194,
      "rewards/rejected": -0.2709660828113556,
      "step": 1025
    },
    {
      "epoch": 1.6162332545311269,
      "grad_norm": 0.2329980731010437,
      "learning_rate": 4.365818784678736e-07,
      "log_odds_chosen": 1.4550331830978394,
      "log_odds_ratio": -0.2117491066455841,
      "logits/chosen": -0.7104726433753967,
      "logits/rejected": -1.2194942235946655,
      "logps/chosen": -1.3183393478393555,
      "logps/rejected": -2.541229248046875,
      "loss": 1.3684,
      "nll_loss": 1.3472259044647217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13183394074440002,
      "rewards/margins": 0.12228897958993912,
      "rewards/rejected": -0.25412294268608093,
      "step": 1026
    },
    {
      "epoch": 1.6178092986603625,
      "grad_norm": 0.27200257778167725,
      "learning_rate": 4.3315955872850464e-07,
      "log_odds_chosen": 1.3702400922775269,
      "log_odds_ratio": -0.2405995875597,
      "logits/chosen": -0.7693892121315002,
      "logits/rejected": -1.240039348602295,
      "logps/chosen": -1.4362014532089233,
      "logps/rejected": -2.612527847290039,
      "loss": 1.4663,
      "nll_loss": 1.4422272443771362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14362016320228577,
      "rewards/margins": 0.11763264238834381,
      "rewards/rejected": -0.2612527906894684,
      "step": 1027
    },
    {
      "epoch": 1.6193853427895981,
      "grad_norm": 0.22556494176387787,
      "learning_rate": 4.2974907572029016e-07,
      "log_odds_chosen": 1.4223389625549316,
      "log_odds_ratio": -0.23438873887062073,
      "logits/chosen": -0.7434157729148865,
      "logits/rejected": -1.3750102519989014,
      "logps/chosen": -1.4744399785995483,
      "logps/rejected": -2.707824468612671,
      "loss": 1.488,
      "nll_loss": 1.4645203351974487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1474439948797226,
      "rewards/margins": 0.12333843111991882,
      "rewards/rejected": -0.2707824409008026,
      "step": 1028
    },
    {
      "epoch": 1.6209613869188337,
      "grad_norm": 0.2386234849691391,
      "learning_rate": 4.2635045520780033e-07,
      "log_odds_chosen": 1.4891953468322754,
      "log_odds_ratio": -0.22021855413913727,
      "logits/chosen": -0.7640093564987183,
      "logits/rejected": -1.3112902641296387,
      "logps/chosen": -1.4194504022598267,
      "logps/rejected": -2.705399513244629,
      "loss": 1.4522,
      "nll_loss": 1.4301786422729492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1419450342655182,
      "rewards/margins": 0.12859492003917694,
      "rewards/rejected": -0.2705399692058563,
      "step": 1029
    },
    {
      "epoch": 1.6225374310480694,
      "grad_norm": 0.24485865235328674,
      "learning_rate": 4.2296372286598904e-07,
      "log_odds_chosen": 1.7651314735412598,
      "log_odds_ratio": -0.17434601485729218,
      "logits/chosen": -0.7912206053733826,
      "logits/rejected": -1.4533413648605347,
      "logps/chosen": -1.3964922428131104,
      "logps/rejected": -2.9355716705322266,
      "loss": 1.4113,
      "nll_loss": 1.3938628435134888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13964921236038208,
      "rewards/margins": 0.15390795469284058,
      "rewards/rejected": -0.29355716705322266,
      "step": 1030
    },
    {
      "epoch": 1.624113475177305,
      "grad_norm": 0.3065810799598694,
      "learning_rate": 4.1958890428000205e-07,
      "log_odds_chosen": 1.205777645111084,
      "log_odds_ratio": -0.2740621566772461,
      "logits/chosen": -0.7583440542221069,
      "logits/rejected": -1.2204135656356812,
      "logps/chosen": -1.4262950420379639,
      "logps/rejected": -2.4534482955932617,
      "loss": 1.4642,
      "nll_loss": 1.4367954730987549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1426294893026352,
      "rewards/margins": 0.10271532088518143,
      "rewards/rejected": -0.24534481763839722,
      "step": 1031
    },
    {
      "epoch": 1.6256895193065406,
      "grad_norm": 0.2436000555753708,
      "learning_rate": 4.162260249449816e-07,
      "log_odds_chosen": 1.3780319690704346,
      "log_odds_ratio": -0.2362157553434372,
      "logits/chosen": -0.7164447903633118,
      "logits/rejected": -1.1446393728256226,
      "logps/chosen": -1.3654944896697998,
      "logps/rejected": -2.5300917625427246,
      "loss": 1.4114,
      "nll_loss": 1.3877835273742676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1365494430065155,
      "rewards/margins": 0.11645975708961487,
      "rewards/rejected": -0.25300920009613037,
      "step": 1032
    },
    {
      "epoch": 1.6272655634357762,
      "grad_norm": 0.24678604304790497,
      "learning_rate": 4.128751102658752e-07,
      "log_odds_chosen": 1.4300299882888794,
      "log_odds_ratio": -0.23077574372291565,
      "logits/chosen": -0.7324338555335999,
      "logits/rejected": -1.1989775896072388,
      "logps/chosen": -1.4137389659881592,
      "logps/rejected": -2.6432199478149414,
      "loss": 1.4343,
      "nll_loss": 1.4112646579742432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1413739025592804,
      "rewards/margins": 0.12294812500476837,
      "rewards/rejected": -0.2643220126628876,
      "step": 1033
    },
    {
      "epoch": 1.6288416075650118,
      "grad_norm": 0.22600232064723969,
      "learning_rate": 4.09536185557243e-07,
      "log_odds_chosen": 1.7808507680892944,
      "log_odds_ratio": -0.17392060160636902,
      "logits/chosen": -0.655136227607727,
      "logits/rejected": -1.3819907903671265,
      "logps/chosen": -1.4806416034698486,
      "logps/rejected": -3.0562829971313477,
      "loss": 1.4917,
      "nll_loss": 1.4742894172668457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14806416630744934,
      "rewards/margins": 0.15756414830684662,
      "rewards/rejected": -0.30562829971313477,
      "step": 1034
    },
    {
      "epoch": 1.6304176516942475,
      "grad_norm": 0.22259069979190826,
      "learning_rate": 4.0620927604306533e-07,
      "log_odds_chosen": 1.5100860595703125,
      "log_odds_ratio": -0.2066231220960617,
      "logits/chosen": -0.7529453039169312,
      "logits/rejected": -1.4417768716812134,
      "logps/chosen": -1.4382884502410889,
      "logps/rejected": -2.7404818534851074,
      "loss": 1.4543,
      "nll_loss": 1.4336491823196411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1438288390636444,
      "rewards/margins": 0.13021934032440186,
      "rewards/rejected": -0.2740481495857239,
      "step": 1035
    },
    {
      "epoch": 1.631993695823483,
      "grad_norm": 0.2371823787689209,
      "learning_rate": 4.0289440685655516e-07,
      "log_odds_chosen": 1.366231918334961,
      "log_odds_ratio": -0.2387264519929886,
      "logits/chosen": -0.7763131260871887,
      "logits/rejected": -1.3410357236862183,
      "logps/chosen": -1.4500455856323242,
      "logps/rejected": -2.626335859298706,
      "loss": 1.4624,
      "nll_loss": 1.4384974241256714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14500455558300018,
      "rewards/margins": 0.1176290214061737,
      "rewards/rejected": -0.2626335918903351,
      "step": 1036
    },
    {
      "epoch": 1.6335697399527187,
      "grad_norm": 0.24410951137542725,
      "learning_rate": 3.9959160303996575e-07,
      "log_odds_chosen": 1.4228943586349487,
      "log_odds_ratio": -0.25725749135017395,
      "logits/chosen": -0.662165105342865,
      "logits/rejected": -1.0752835273742676,
      "logps/chosen": -1.4277582168579102,
      "logps/rejected": -2.6551713943481445,
      "loss": 1.4583,
      "nll_loss": 1.4325600862503052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14277583360671997,
      "rewards/margins": 0.12274131923913956,
      "rewards/rejected": -0.2655171751976013,
      "step": 1037
    },
    {
      "epoch": 1.6351457840819543,
      "grad_norm": 0.23364105820655823,
      "learning_rate": 3.9630088954440244e-07,
      "log_odds_chosen": 1.4563705921173096,
      "log_odds_ratio": -0.22325736284255981,
      "logits/chosen": -0.6992417573928833,
      "logits/rejected": -1.3359637260437012,
      "logps/chosen": -1.4622055292129517,
      "logps/rejected": -2.7263894081115723,
      "loss": 1.4931,
      "nll_loss": 1.4707622528076172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14622054994106293,
      "rewards/margins": 0.12641841173171997,
      "rewards/rejected": -0.2726389467716217,
      "step": 1038
    },
    {
      "epoch": 1.63672182821119,
      "grad_norm": 0.22445590794086456,
      "learning_rate": 3.930222912296346e-07,
      "log_odds_chosen": 1.8402867317199707,
      "log_odds_ratio": -0.16749165952205658,
      "logits/chosen": -0.7281829118728638,
      "logits/rejected": -1.342471957206726,
      "logps/chosen": -1.3723416328430176,
      "logps/rejected": -2.9813642501831055,
      "loss": 1.401,
      "nll_loss": 1.3842716217041016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1372341811656952,
      "rewards/margins": 0.1609022319316864,
      "rewards/rejected": -0.2981363832950592,
      "step": 1039
    },
    {
      "epoch": 1.6382978723404256,
      "grad_norm": 0.22943423688411713,
      "learning_rate": 3.8975583286390546e-07,
      "log_odds_chosen": 1.2713685035705566,
      "log_odds_ratio": -0.2500207722187042,
      "logits/chosen": -0.7335870265960693,
      "logits/rejected": -1.2935973405838013,
      "logps/chosen": -1.4628053903579712,
      "logps/rejected": -2.5515074729919434,
      "loss": 1.483,
      "nll_loss": 1.4579675197601318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14628054201602936,
      "rewards/margins": 0.10887020826339722,
      "rewards/rejected": -0.2551507353782654,
      "step": 1040
    },
    {
      "epoch": 1.6398739164696612,
      "grad_norm": 0.2208222895860672,
      "learning_rate": 3.8650153912374807e-07,
      "log_odds_chosen": 1.7247302532196045,
      "log_odds_ratio": -0.17473134398460388,
      "logits/chosen": -0.7309201955795288,
      "logits/rejected": -1.3389525413513184,
      "logps/chosen": -1.4383296966552734,
      "logps/rejected": -2.9430692195892334,
      "loss": 1.4509,
      "nll_loss": 1.433395504951477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1438329815864563,
      "rewards/margins": 0.150473952293396,
      "rewards/rejected": -0.2943069338798523,
      "step": 1041
    },
    {
      "epoch": 1.6414499605988968,
      "grad_norm": 0.26771509647369385,
      "learning_rate": 3.832594345937974e-07,
      "log_odds_chosen": 1.3588666915893555,
      "log_odds_ratio": -0.240315243601799,
      "logits/chosen": -0.7971328496932983,
      "logits/rejected": -1.0864603519439697,
      "logps/chosen": -1.3542962074279785,
      "logps/rejected": -2.500123977661133,
      "loss": 1.4094,
      "nll_loss": 1.385319471359253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13542962074279785,
      "rewards/margins": 0.11458276212215424,
      "rewards/rejected": -0.2500123977661133,
      "step": 1042
    },
    {
      "epoch": 1.6430260047281324,
      "grad_norm": 0.24574102461338043,
      "learning_rate": 3.8002954376660524e-07,
      "log_odds_chosen": 1.3776321411132812,
      "log_odds_ratio": -0.22766003012657166,
      "logits/chosen": -0.6891369819641113,
      "logits/rejected": -1.3985782861709595,
      "logps/chosen": -1.4987937211990356,
      "logps/rejected": -2.6934926509857178,
      "loss": 1.5179,
      "nll_loss": 1.4951064586639404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1498793512582779,
      "rewards/margins": 0.11946991086006165,
      "rewards/rejected": -0.26934927701950073,
      "step": 1043
    },
    {
      "epoch": 1.644602048857368,
      "grad_norm": 0.2260279506444931,
      "learning_rate": 3.7681189104245316e-07,
      "log_odds_chosen": 1.5078110694885254,
      "log_odds_ratio": -0.20770563185214996,
      "logits/chosen": -0.8532127141952515,
      "logits/rejected": -1.2204086780548096,
      "logps/chosen": -1.3179402351379395,
      "logps/rejected": -2.5901358127593994,
      "loss": 1.3635,
      "nll_loss": 1.3426926136016846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1317940205335617,
      "rewards/margins": 0.127219557762146,
      "rewards/rejected": -0.2590135931968689,
      "step": 1044
    },
    {
      "epoch": 1.6461780929866037,
      "grad_norm": 0.38637664914131165,
      "learning_rate": 3.7360650072917067e-07,
      "log_odds_chosen": 1.3873088359832764,
      "log_odds_ratio": -0.2390640825033188,
      "logits/chosen": -0.6200605034828186,
      "logits/rejected": -1.186801791191101,
      "logps/chosen": -1.3915066719055176,
      "logps/rejected": -2.5709846019744873,
      "loss": 1.4158,
      "nll_loss": 1.391933560371399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13915066421031952,
      "rewards/margins": 0.11794780194759369,
      "rewards/rejected": -0.2570984661579132,
      "step": 1045
    },
    {
      "epoch": 1.6477541371158393,
      "grad_norm": 0.21985970437526703,
      "learning_rate": 3.704133970419514e-07,
      "log_odds_chosen": 1.42316734790802,
      "log_odds_ratio": -0.23063690960407257,
      "logits/chosen": -0.8142727017402649,
      "logits/rejected": -1.282482385635376,
      "logps/chosen": -1.3918331861495972,
      "logps/rejected": -2.609302520751953,
      "loss": 1.4346,
      "nll_loss": 1.4115490913391113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13918332755565643,
      "rewards/margins": 0.12174694985151291,
      "rewards/rejected": -0.26093026995658875,
      "step": 1046
    },
    {
      "epoch": 1.649330181245075,
      "grad_norm": 0.25275611877441406,
      "learning_rate": 3.672326041031677e-07,
      "log_odds_chosen": 1.2324278354644775,
      "log_odds_ratio": -0.2623051404953003,
      "logits/chosen": -0.643879771232605,
      "logits/rejected": -1.2117925882339478,
      "logps/chosen": -1.4339498281478882,
      "logps/rejected": -2.4825000762939453,
      "loss": 1.461,
      "nll_loss": 1.4347654581069946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14339497685432434,
      "rewards/margins": 0.104855015873909,
      "rewards/rejected": -0.24824999272823334,
      "step": 1047
    },
    {
      "epoch": 1.6509062253743105,
      "grad_norm": 0.21049244701862335,
      "learning_rate": 3.6406414594219116e-07,
      "log_odds_chosen": 1.712800145149231,
      "log_odds_ratio": -0.17751756310462952,
      "logits/chosen": -0.7435751557350159,
      "logits/rejected": -1.4559879302978516,
      "logps/chosen": -1.4532015323638916,
      "logps/rejected": -2.9530813694000244,
      "loss": 1.4681,
      "nll_loss": 1.450369119644165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14532016217708588,
      "rewards/margins": 0.14998799562454224,
      "rewards/rejected": -0.2953081429004669,
      "step": 1048
    },
    {
      "epoch": 1.6524822695035462,
      "grad_norm": 0.2245529145002365,
      "learning_rate": 3.609080464952103e-07,
      "log_odds_chosen": 1.6266381740570068,
      "log_odds_ratio": -0.19851456582546234,
      "logits/chosen": -0.8403435945510864,
      "logits/rejected": -1.3426330089569092,
      "logps/chosen": -1.4386788606643677,
      "logps/rejected": -2.8511948585510254,
      "loss": 1.4597,
      "nll_loss": 1.4398469924926758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14386789500713348,
      "rewards/margins": 0.1412515938282013,
      "rewards/rejected": -0.2851194739341736,
      "step": 1049
    },
    {
      "epoch": 1.6540583136327816,
      "grad_norm": 0.25292253494262695,
      "learning_rate": 3.577643296050483e-07,
      "log_odds_chosen": 1.7232894897460938,
      "log_odds_ratio": -0.20050916075706482,
      "logits/chosen": -0.7236199378967285,
      "logits/rejected": -1.3849279880523682,
      "logps/chosen": -1.5565634965896606,
      "logps/rejected": -3.0943360328674316,
      "loss": 1.5531,
      "nll_loss": 1.5330204963684082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1556563377380371,
      "rewards/margins": 0.15377727150917053,
      "rewards/rejected": -0.30943360924720764,
      "step": 1050
    },
    {
      "epoch": 1.6556343577620174,
      "grad_norm": 0.23792169988155365,
      "learning_rate": 3.5463301902098496e-07,
      "log_odds_chosen": 1.5054265260696411,
      "log_odds_ratio": -0.22392994165420532,
      "logits/chosen": -0.7350411415100098,
      "logits/rejected": -1.174824833869934,
      "logps/chosen": -1.3381047248840332,
      "logps/rejected": -2.629563093185425,
      "loss": 1.3741,
      "nll_loss": 1.3517394065856934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13381047546863556,
      "rewards/margins": 0.12914584577083588,
      "rewards/rejected": -0.26295632123947144,
      "step": 1051
    },
    {
      "epoch": 1.6572104018912528,
      "grad_norm": 0.22243046760559082,
      "learning_rate": 3.5151413839857625e-07,
      "log_odds_chosen": 1.4250224828720093,
      "log_odds_ratio": -0.22721093893051147,
      "logits/chosen": -0.7038748264312744,
      "logits/rejected": -1.3001593351364136,
      "logps/chosen": -1.4629769325256348,
      "logps/rejected": -2.6976370811462402,
      "loss": 1.4785,
      "nll_loss": 1.455736756324768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14629767835140228,
      "rewards/margins": 0.12346599996089935,
      "rewards/rejected": -0.26976367831230164,
      "step": 1052
    },
    {
      "epoch": 1.6587864460204886,
      "grad_norm": 0.24289046227931976,
      "learning_rate": 3.484077112994759e-07,
      "log_odds_chosen": 1.6655735969543457,
      "log_odds_ratio": -0.18197569251060486,
      "logits/chosen": -0.842210590839386,
      "logits/rejected": -1.453803300857544,
      "logps/chosen": -1.4679410457611084,
      "logps/rejected": -2.927338123321533,
      "loss": 1.4861,
      "nll_loss": 1.4679443836212158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14679411053657532,
      "rewards/margins": 0.14593970775604248,
      "rewards/rejected": -0.2927338182926178,
      "step": 1053
    },
    {
      "epoch": 1.660362490149724,
      "grad_norm": 0.25047558546066284,
      "learning_rate": 3.45313761191256e-07,
      "log_odds_chosen": 1.1888450384140015,
      "log_odds_ratio": -0.28826919198036194,
      "logits/chosen": -0.7887861132621765,
      "logits/rejected": -1.1554025411605835,
      "logps/chosen": -1.396628975868225,
      "logps/rejected": -2.40462064743042,
      "loss": 1.4367,
      "nll_loss": 1.4078651666641235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13966290652751923,
      "rewards/margins": 0.10079918801784515,
      "rewards/rejected": -0.24046209454536438,
      "step": 1054
    },
    {
      "epoch": 1.6619385342789599,
      "grad_norm": 0.2311248481273651,
      "learning_rate": 3.4223231144723253e-07,
      "log_odds_chosen": 1.3660454750061035,
      "log_odds_ratio": -0.24093665182590485,
      "logits/chosen": -0.7444354295730591,
      "logits/rejected": -1.279778242111206,
      "logps/chosen": -1.4939483404159546,
      "logps/rejected": -2.678544521331787,
      "loss": 1.5344,
      "nll_loss": 1.5103377103805542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14939482510089874,
      "rewards/margins": 0.11845961213111877,
      "rewards/rejected": -0.2678544223308563,
      "step": 1055
    },
    {
      "epoch": 1.6635145784081953,
      "grad_norm": 0.26590391993522644,
      "learning_rate": 3.391633853462861e-07,
      "log_odds_chosen": 1.64162015914917,
      "log_odds_ratio": -0.18242497742176056,
      "logits/chosen": -0.7732189297676086,
      "logits/rejected": -1.3036084175109863,
      "logps/chosen": -1.4340802431106567,
      "logps/rejected": -2.86145281791687,
      "loss": 1.4755,
      "nll_loss": 1.457226037979126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14340803027153015,
      "rewards/margins": 0.1427372545003891,
      "rewards/rejected": -0.28614526987075806,
      "step": 1056
    },
    {
      "epoch": 1.6650906225374311,
      "grad_norm": 0.22892804443836212,
      "learning_rate": 3.3610700607268803e-07,
      "log_odds_chosen": 1.3321940898895264,
      "log_odds_ratio": -0.23921093344688416,
      "logits/chosen": -0.6748044490814209,
      "logits/rejected": -1.218104600906372,
      "logps/chosen": -1.4165993928909302,
      "logps/rejected": -2.553696870803833,
      "loss": 1.4547,
      "nll_loss": 1.4307947158813477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1416599154472351,
      "rewards/margins": 0.11370972543954849,
      "rewards/rejected": -0.2553696632385254,
      "step": 1057
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 0.2559583783149719,
      "learning_rate": 3.330631967159239e-07,
      "log_odds_chosen": 1.3838810920715332,
      "log_odds_ratio": -0.22764509916305542,
      "logits/chosen": -0.6965813636779785,
      "logits/rejected": -1.3854737281799316,
      "logps/chosen": -1.4167238473892212,
      "logps/rejected": -2.600916862487793,
      "loss": 1.4455,
      "nll_loss": 1.4226998090744019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14167238771915436,
      "rewards/margins": 0.11841931194067001,
      "rewards/rejected": -0.26009172201156616,
      "step": 1058
    },
    {
      "epoch": 1.6682427107959024,
      "grad_norm": 0.21866033971309662,
      "learning_rate": 3.300319802705189e-07,
      "log_odds_chosen": 1.7368463277816772,
      "log_odds_ratio": -0.19434627890586853,
      "logits/chosen": -0.6384781002998352,
      "logits/rejected": -1.4098758697509766,
      "logps/chosen": -1.471012830734253,
      "logps/rejected": -3.0071041584014893,
      "loss": 1.5035,
      "nll_loss": 1.4841136932373047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1471012830734253,
      "rewards/margins": 0.15360914170742035,
      "rewards/rejected": -0.30071043968200684,
      "step": 1059
    },
    {
      "epoch": 1.6698187549251378,
      "grad_norm": 0.21730639040470123,
      "learning_rate": 3.2701337963586604e-07,
      "log_odds_chosen": 1.4864357709884644,
      "log_odds_ratio": -0.21466375887393951,
      "logits/chosen": -0.7049427032470703,
      "logits/rejected": -1.2318035364151,
      "logps/chosen": -1.2843377590179443,
      "logps/rejected": -2.531402111053467,
      "loss": 1.3317,
      "nll_loss": 1.310271978378296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12843377888202667,
      "rewards/margins": 0.12470643222332001,
      "rewards/rejected": -0.2531402111053467,
      "step": 1060
    },
    {
      "epoch": 1.6713947990543736,
      "grad_norm": 0.22326745092868805,
      "learning_rate": 3.24007417616051e-07,
      "log_odds_chosen": 1.50540292263031,
      "log_odds_ratio": -0.21528325974941254,
      "logits/chosen": -0.8404073715209961,
      "logits/rejected": -1.375593900680542,
      "logps/chosen": -1.3753737211227417,
      "logps/rejected": -2.669926166534424,
      "loss": 1.396,
      "nll_loss": 1.3744949102401733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1375373750925064,
      "rewards/margins": 0.12945526838302612,
      "rewards/rejected": -0.26699262857437134,
      "step": 1061
    },
    {
      "epoch": 1.672970843183609,
      "grad_norm": 0.23941734433174133,
      "learning_rate": 3.2101411691968115e-07,
      "log_odds_chosen": 1.5097068548202515,
      "log_odds_ratio": -0.20746314525604248,
      "logits/chosen": -0.7454752326011658,
      "logits/rejected": -1.3938078880310059,
      "logps/chosen": -1.3707630634307861,
      "logps/rejected": -2.6578426361083984,
      "loss": 1.4083,
      "nll_loss": 1.3875243663787842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13707630336284637,
      "rewards/margins": 0.12870797514915466,
      "rewards/rejected": -0.26578426361083984,
      "step": 1062
    },
    {
      "epoch": 1.6745468873128448,
      "grad_norm": 0.2342255860567093,
      "learning_rate": 3.180335001597143e-07,
      "log_odds_chosen": 1.6418344974517822,
      "log_odds_ratio": -0.1940932422876358,
      "logits/chosen": -0.8314214944839478,
      "logits/rejected": -1.334609866142273,
      "logps/chosen": -1.4253090620040894,
      "logps/rejected": -2.855656385421753,
      "loss": 1.4469,
      "nll_loss": 1.4275100231170654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1425309181213379,
      "rewards/margins": 0.14303474128246307,
      "rewards/rejected": -0.28556567430496216,
      "step": 1063
    },
    {
      "epoch": 1.6761229314420802,
      "grad_norm": 0.23353338241577148,
      "learning_rate": 3.1506558985328523e-07,
      "log_odds_chosen": 1.5735435485839844,
      "log_odds_ratio": -0.20354756712913513,
      "logits/chosen": -0.7156309485435486,
      "logits/rejected": -1.460034728050232,
      "logps/chosen": -1.37836754322052,
      "logps/rejected": -2.729607105255127,
      "loss": 1.4039,
      "nll_loss": 1.3835139274597168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1378367692232132,
      "rewards/margins": 0.13512398302555084,
      "rewards/rejected": -0.27296072244644165,
      "step": 1064
    },
    {
      "epoch": 1.677698975571316,
      "grad_norm": 0.2617426812648773,
      "learning_rate": 3.1211040842153934e-07,
      "log_odds_chosen": 1.3426097631454468,
      "log_odds_ratio": -0.23880839347839355,
      "logits/chosen": -0.7077674865722656,
      "logits/rejected": -1.3756884336471558,
      "logps/chosen": -1.571168303489685,
      "logps/rejected": -2.747244358062744,
      "loss": 1.577,
      "nll_loss": 1.5531212091445923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1571168303489685,
      "rewards/margins": 0.11760764569044113,
      "rewards/rejected": -0.27472448348999023,
      "step": 1065
    },
    {
      "epoch": 1.6792750197005515,
      "grad_norm": 0.2381659746170044,
      "learning_rate": 3.091679781894605e-07,
      "log_odds_chosen": 1.5407084226608276,
      "log_odds_ratio": -0.2015790343284607,
      "logits/chosen": -0.7404255270957947,
      "logits/rejected": -1.21885347366333,
      "logps/chosen": -1.4179164171218872,
      "logps/rejected": -2.746882677078247,
      "loss": 1.4342,
      "nll_loss": 1.4140264987945557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14179165661334991,
      "rewards/margins": 0.13289663195610046,
      "rewards/rejected": -0.2746882736682892,
      "step": 1066
    },
    {
      "epoch": 1.6808510638297873,
      "grad_norm": 0.21804937720298767,
      "learning_rate": 3.0623832138570473e-07,
      "log_odds_chosen": 1.711024522781372,
      "log_odds_ratio": -0.1701139211654663,
      "logits/chosen": -0.7400306463241577,
      "logits/rejected": -1.3705081939697266,
      "logps/chosen": -1.3155884742736816,
      "logps/rejected": -2.772404193878174,
      "loss": 1.3408,
      "nll_loss": 1.323782205581665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1315588504076004,
      "rewards/margins": 0.14568156003952026,
      "rewards/rejected": -0.2772403955459595,
      "step": 1067
    },
    {
      "epoch": 1.6824271079590227,
      "grad_norm": 0.23263327777385712,
      "learning_rate": 3.033214601424283e-07,
      "log_odds_chosen": 1.463895320892334,
      "log_odds_ratio": -0.22359143197536469,
      "logits/chosen": -0.6863577365875244,
      "logits/rejected": -1.1970839500427246,
      "logps/chosen": -1.4432278871536255,
      "logps/rejected": -2.7123863697052,
      "loss": 1.4674,
      "nll_loss": 1.445003628730774,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14432279765605927,
      "rewards/margins": 0.12691587209701538,
      "rewards/rejected": -0.27123865485191345,
      "step": 1068
    },
    {
      "epoch": 1.6840031520882586,
      "grad_norm": 0.2301708161830902,
      "learning_rate": 3.0041741649512584e-07,
      "log_odds_chosen": 1.6341521739959717,
      "log_odds_ratio": -0.18711106479167938,
      "logits/chosen": -0.6537185907363892,
      "logits/rejected": -1.3739891052246094,
      "logps/chosen": -1.4426209926605225,
      "logps/rejected": -2.866335868835449,
      "loss": 1.4612,
      "nll_loss": 1.4425331354141235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14426212012767792,
      "rewards/margins": 0.14237147569656372,
      "rewards/rejected": -0.28663361072540283,
      "step": 1069
    },
    {
      "epoch": 1.685579196217494,
      "grad_norm": 0.2430332899093628,
      "learning_rate": 2.975262123824596e-07,
      "log_odds_chosen": 1.5421723127365112,
      "log_odds_ratio": -0.20895227789878845,
      "logits/chosen": -0.8156710267066956,
      "logits/rejected": -1.2880693674087524,
      "logps/chosen": -1.4010854959487915,
      "logps/rejected": -2.7313692569732666,
      "loss": 1.4308,
      "nll_loss": 1.4098557233810425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14010855555534363,
      "rewards/margins": 0.13302837312221527,
      "rewards/rejected": -0.2731369137763977,
      "step": 1070
    },
    {
      "epoch": 1.6871552403467298,
      "grad_norm": 0.2374456226825714,
      "learning_rate": 2.9464786964609567e-07,
      "log_odds_chosen": 1.4157829284667969,
      "log_odds_ratio": -0.22964000701904297,
      "logits/chosen": -0.7588300108909607,
      "logits/rejected": -1.1369354724884033,
      "logps/chosen": -1.4608426094055176,
      "logps/rejected": -2.6864449977874756,
      "loss": 1.4779,
      "nll_loss": 1.454981803894043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1460842788219452,
      "rewards/margins": 0.12256024032831192,
      "rewards/rejected": -0.2686445116996765,
      "step": 1071
    },
    {
      "epoch": 1.6887312844759652,
      "grad_norm": 0.21584849059581757,
      "learning_rate": 2.917824100305393e-07,
      "log_odds_chosen": 1.582011342048645,
      "log_odds_ratio": -0.1954343020915985,
      "logits/chosen": -0.6648775339126587,
      "logits/rejected": -1.4148600101470947,
      "logps/chosen": -1.4431657791137695,
      "logps/rejected": -2.819998025894165,
      "loss": 1.4691,
      "nll_loss": 1.4495729207992554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14431656897068024,
      "rewards/margins": 0.1376832276582718,
      "rewards/rejected": -0.281999796628952,
      "step": 1072
    },
    {
      "epoch": 1.690307328605201,
      "grad_norm": 0.22042663395404816,
      "learning_rate": 2.889298551829682e-07,
      "log_odds_chosen": 1.439175009727478,
      "log_odds_ratio": -0.22690895199775696,
      "logits/chosen": -0.7273336052894592,
      "logits/rejected": -1.1528249979019165,
      "logps/chosen": -1.3568432331085205,
      "logps/rejected": -2.5756707191467285,
      "loss": 1.3909,
      "nll_loss": 1.3681716918945312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1356843113899231,
      "rewards/margins": 0.12188276648521423,
      "rewards/rejected": -0.25756707787513733,
      "step": 1073
    },
    {
      "epoch": 1.6918833727344365,
      "grad_norm": 0.2148721069097519,
      "learning_rate": 2.8609022665307226e-07,
      "log_odds_chosen": 1.3996821641921997,
      "log_odds_ratio": -0.2275838553905487,
      "logits/chosen": -0.7625545263290405,
      "logits/rejected": -1.2648324966430664,
      "logps/chosen": -1.3885960578918457,
      "logps/rejected": -2.5809502601623535,
      "loss": 1.4258,
      "nll_loss": 1.4030067920684814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13885962963104248,
      "rewards/margins": 0.11923542618751526,
      "rewards/rejected": -0.25809502601623535,
      "step": 1074
    },
    {
      "epoch": 1.6934594168636723,
      "grad_norm": 0.22514860332012177,
      "learning_rate": 2.832635458928887e-07,
      "log_odds_chosen": 1.5033140182495117,
      "log_odds_ratio": -0.20596297085285187,
      "logits/chosen": -0.685197651386261,
      "logits/rejected": -1.232975959777832,
      "logps/chosen": -1.3430708646774292,
      "logps/rejected": -2.6143925189971924,
      "loss": 1.3905,
      "nll_loss": 1.3698762655258179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1343071013689041,
      "rewards/margins": 0.12713216245174408,
      "rewards/rejected": -0.2614392638206482,
      "step": 1075
    },
    {
      "epoch": 1.6950354609929077,
      "grad_norm": 0.22375792264938354,
      "learning_rate": 2.804498342566406e-07,
      "log_odds_chosen": 1.7807624340057373,
      "log_odds_ratio": -0.1771838217973709,
      "logits/chosen": -0.7387442588806152,
      "logits/rejected": -1.2459298372268677,
      "logps/chosen": -1.3063157796859741,
      "logps/rejected": -2.835942268371582,
      "loss": 1.3543,
      "nll_loss": 1.3366039991378784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13063156604766846,
      "rewards/margins": 0.15296265482902527,
      "rewards/rejected": -0.2835942208766937,
      "step": 1076
    },
    {
      "epoch": 1.6966115051221435,
      "grad_norm": 0.24096116423606873,
      "learning_rate": 2.776491130005758e-07,
      "log_odds_chosen": 1.412062644958496,
      "log_odds_ratio": -0.23172523081302643,
      "logits/chosen": -0.6718806028366089,
      "logits/rejected": -1.2279958724975586,
      "logps/chosen": -1.3862122297286987,
      "logps/rejected": -2.582707643508911,
      "loss": 1.4405,
      "nll_loss": 1.4173099994659424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13862121105194092,
      "rewards/margins": 0.11964955925941467,
      "rewards/rejected": -0.2582707703113556,
      "step": 1077
    },
    {
      "epoch": 1.698187549251379,
      "grad_norm": 0.2236708104610443,
      "learning_rate": 2.7486140328280473e-07,
      "log_odds_chosen": 1.4870802164077759,
      "log_odds_ratio": -0.21007102727890015,
      "logits/chosen": -0.8041332960128784,
      "logits/rejected": -1.4371874332427979,
      "logps/chosen": -1.4593466520309448,
      "logps/rejected": -2.7455554008483887,
      "loss": 1.4776,
      "nll_loss": 1.4565882682800293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14593465626239777,
      "rewards/margins": 0.12862087786197662,
      "rewards/rejected": -0.2745555341243744,
      "step": 1078
    },
    {
      "epoch": 1.6997635933806148,
      "grad_norm": 0.241652712225914,
      "learning_rate": 2.7208672616314345e-07,
      "log_odds_chosen": 1.5433874130249023,
      "log_odds_ratio": -0.20241758227348328,
      "logits/chosen": -0.7827467918395996,
      "logits/rejected": -1.3265055418014526,
      "logps/chosen": -1.449453353881836,
      "logps/rejected": -2.789214611053467,
      "loss": 1.4755,
      "nll_loss": 1.4552533626556396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14494533836841583,
      "rewards/margins": 0.13397614657878876,
      "rewards/rejected": -0.2789214849472046,
      "step": 1079
    },
    {
      "epoch": 1.7013396375098502,
      "grad_norm": 0.23656415939331055,
      "learning_rate": 2.6932510260295237e-07,
      "log_odds_chosen": 1.4882760047912598,
      "log_odds_ratio": -0.22133508324623108,
      "logits/chosen": -0.7218590378761292,
      "logits/rejected": -1.1563297510147095,
      "logps/chosen": -1.3360663652420044,
      "logps/rejected": -2.5973565578460693,
      "loss": 1.3717,
      "nll_loss": 1.3495519161224365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13360662758350372,
      "rewards/margins": 0.12612903118133545,
      "rewards/rejected": -0.25973567366600037,
      "step": 1080
    },
    {
      "epoch": 1.702915681639086,
      "grad_norm": 0.24389007687568665,
      "learning_rate": 2.66576553464978e-07,
      "log_odds_chosen": 1.3599222898483276,
      "log_odds_ratio": -0.23216691613197327,
      "logits/chosen": -0.7494752407073975,
      "logits/rejected": -1.3491950035095215,
      "logps/chosen": -1.5306495428085327,
      "logps/rejected": -2.715559244155884,
      "loss": 1.5464,
      "nll_loss": 1.5231562852859497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15306496620178223,
      "rewards/margins": 0.11849097907543182,
      "rewards/rejected": -0.27155596017837524,
      "step": 1081
    },
    {
      "epoch": 1.7044917257683214,
      "grad_norm": 0.23120272159576416,
      "learning_rate": 2.6384109951319654e-07,
      "log_odds_chosen": 1.3483827114105225,
      "log_odds_ratio": -0.2449389398097992,
      "logits/chosen": -0.701062023639679,
      "logits/rejected": -1.2585502862930298,
      "logps/chosen": -1.4386335611343384,
      "logps/rejected": -2.5963330268859863,
      "loss": 1.4642,
      "nll_loss": 1.439679741859436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14386335015296936,
      "rewards/margins": 0.11576996743679047,
      "rewards/rejected": -0.259633332490921,
      "step": 1082
    },
    {
      "epoch": 1.7060677698975573,
      "grad_norm": 0.22506409883499146,
      "learning_rate": 2.6111876141265646e-07,
      "log_odds_chosen": 1.357731580734253,
      "log_odds_ratio": -0.23910564184188843,
      "logits/chosen": -0.7155004143714905,
      "logits/rejected": -1.3416612148284912,
      "logps/chosen": -1.4062689542770386,
      "logps/rejected": -2.5689046382904053,
      "loss": 1.4377,
      "nll_loss": 1.4137547016143799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14062689244747162,
      "rewards/margins": 0.11626356840133667,
      "rewards/rejected": -0.2568904757499695,
      "step": 1083
    },
    {
      "epoch": 1.7076438140267927,
      "grad_norm": 0.225145623087883,
      "learning_rate": 2.58409559729321e-07,
      "log_odds_chosen": 1.4400783777236938,
      "log_odds_ratio": -0.22819840908050537,
      "logits/chosen": -0.7862902283668518,
      "logits/rejected": -1.2531520128250122,
      "logps/chosen": -1.3739902973175049,
      "logps/rejected": -2.6072325706481934,
      "loss": 1.4104,
      "nll_loss": 1.3875616788864136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13739903271198273,
      "rewards/margins": 0.12332425266504288,
      "rewards/rejected": -0.2607232630252838,
      "step": 1084
    },
    {
      "epoch": 1.7092198581560285,
      "grad_norm": 0.24570077657699585,
      "learning_rate": 2.55713514929915e-07,
      "log_odds_chosen": 1.5189685821533203,
      "log_odds_ratio": -0.2180192768573761,
      "logits/chosen": -0.8139381408691406,
      "logits/rejected": -1.3012667894363403,
      "logps/chosen": -1.3721256256103516,
      "logps/rejected": -2.6742498874664307,
      "loss": 1.408,
      "nll_loss": 1.3862184286117554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13721255958080292,
      "rewards/margins": 0.1302124410867691,
      "rewards/rejected": -0.267425000667572,
      "step": 1085
    },
    {
      "epoch": 1.710795902285264,
      "grad_norm": 0.2492953985929489,
      "learning_rate": 2.530306473817696e-07,
      "log_odds_chosen": 1.3926929235458374,
      "log_odds_ratio": -0.22502940893173218,
      "logits/chosen": -0.6770783066749573,
      "logits/rejected": -1.3658583164215088,
      "logps/chosen": -1.4940253496170044,
      "logps/rejected": -2.7020163536071777,
      "loss": 1.5142,
      "nll_loss": 1.4917197227478027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14940252900123596,
      "rewards/margins": 0.12079912424087524,
      "rewards/rejected": -0.2702016532421112,
      "step": 1086
    },
    {
      "epoch": 1.7123719464144997,
      "grad_norm": 0.24708017706871033,
      "learning_rate": 2.50360977352668e-07,
      "log_odds_chosen": 1.561624526977539,
      "log_odds_ratio": -0.20098355412483215,
      "logits/chosen": -0.7767598628997803,
      "logits/rejected": -1.2110247611999512,
      "logps/chosen": -1.4537392854690552,
      "logps/rejected": -2.809616804122925,
      "loss": 1.4836,
      "nll_loss": 1.4634649753570557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14537391066551208,
      "rewards/margins": 0.13558779656887054,
      "rewards/rejected": -0.2809617221355438,
      "step": 1087
    },
    {
      "epoch": 1.7139479905437351,
      "grad_norm": 0.24226981401443481,
      "learning_rate": 2.477045250106913e-07,
      "log_odds_chosen": 1.4408555030822754,
      "log_odds_ratio": -0.23368826508522034,
      "logits/chosen": -0.6611303687095642,
      "logits/rejected": -1.338610291481018,
      "logps/chosen": -1.5297034978866577,
      "logps/rejected": -2.792124032974243,
      "loss": 1.5428,
      "nll_loss": 1.5194445848464966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1529703587293625,
      "rewards/margins": 0.12624205648899078,
      "rewards/rejected": -0.27921241521835327,
      "step": 1088
    },
    {
      "epoch": 1.715524034672971,
      "grad_norm": 0.22355736792087555,
      "learning_rate": 2.450613104240684e-07,
      "log_odds_chosen": 1.8325200080871582,
      "log_odds_ratio": -0.15662840008735657,
      "logits/chosen": -0.7575389742851257,
      "logits/rejected": -1.4759278297424316,
      "logps/chosen": -1.3920414447784424,
      "logps/rejected": -2.980879306793213,
      "loss": 1.4056,
      "nll_loss": 1.3899106979370117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13920414447784424,
      "rewards/margins": 0.15888378024101257,
      "rewards/rejected": -0.2980879247188568,
      "step": 1089
    },
    {
      "epoch": 1.7171000788022064,
      "grad_norm": 0.25072941184043884,
      "learning_rate": 2.42431353561023e-07,
      "log_odds_chosen": 1.5309935808181763,
      "log_odds_ratio": -0.20592352747917175,
      "logits/chosen": -0.7396482229232788,
      "logits/rejected": -1.2973253726959229,
      "logps/chosen": -1.4700946807861328,
      "logps/rejected": -2.8036961555480957,
      "loss": 1.5005,
      "nll_loss": 1.4798779487609863,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1470094919204712,
      "rewards/margins": 0.13336016237735748,
      "rewards/rejected": -0.28036966919898987,
      "step": 1090
    },
    {
      "epoch": 1.7186761229314422,
      "grad_norm": 0.5032855868339539,
      "learning_rate": 2.3981467428962364e-07,
      "log_odds_chosen": 1.6297082901000977,
      "log_odds_ratio": -0.21596840023994446,
      "logits/chosen": -0.6906307339668274,
      "logits/rejected": -1.3518569469451904,
      "logps/chosen": -1.3840055465698242,
      "logps/rejected": -2.8017306327819824,
      "loss": 1.4139,
      "nll_loss": 1.392266869544983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13840055465698242,
      "rewards/margins": 0.1417725384235382,
      "rewards/rejected": -0.28017309308052063,
      "step": 1091
    },
    {
      "epoch": 1.7202521670606776,
      "grad_norm": 0.24462655186653137,
      "learning_rate": 2.372112923776315e-07,
      "log_odds_chosen": 1.4947786331176758,
      "log_odds_ratio": -0.2135419249534607,
      "logits/chosen": -0.7018687725067139,
      "logits/rejected": -1.2385170459747314,
      "logps/chosen": -1.47986900806427,
      "logps/rejected": -2.780205488204956,
      "loss": 1.5065,
      "nll_loss": 1.485137701034546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14798690378665924,
      "rewards/margins": 0.1300336718559265,
      "rewards/rejected": -0.27802056074142456,
      "step": 1092
    },
    {
      "epoch": 1.7218282111899135,
      "grad_norm": 0.22754500806331635,
      "learning_rate": 2.346212274923538e-07,
      "log_odds_chosen": 1.266727328300476,
      "log_odds_ratio": -0.26117709279060364,
      "logits/chosen": -0.788493812084198,
      "logits/rejected": -1.241527795791626,
      "logps/chosen": -1.3921703100204468,
      "logps/rejected": -2.467926025390625,
      "loss": 1.4336,
      "nll_loss": 1.407468557357788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13921703398227692,
      "rewards/margins": 0.10757556557655334,
      "rewards/rejected": -0.24679259955883026,
      "step": 1093
    },
    {
      "epoch": 1.7234042553191489,
      "grad_norm": 0.22975799441337585,
      "learning_rate": 2.3204449920049372e-07,
      "log_odds_chosen": 1.4731096029281616,
      "log_odds_ratio": -0.21840548515319824,
      "logits/chosen": -0.7798600792884827,
      "logits/rejected": -1.3665807247161865,
      "logps/chosen": -1.545700192451477,
      "logps/rejected": -2.8392493724823,
      "loss": 1.5494,
      "nll_loss": 1.5275554656982422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15456999838352203,
      "rewards/margins": 0.12935496866703033,
      "rewards/rejected": -0.28392499685287476,
      "step": 1094
    },
    {
      "epoch": 1.7249802994483847,
      "grad_norm": 0.2388199269771576,
      "learning_rate": 2.2948112696800283e-07,
      "log_odds_chosen": 1.4039790630340576,
      "log_odds_ratio": -0.2224840223789215,
      "logits/chosen": -0.7020440697669983,
      "logits/rejected": -1.357128381729126,
      "logps/chosen": -1.5009918212890625,
      "logps/rejected": -2.719858407974243,
      "loss": 1.5183,
      "nll_loss": 1.4960153102874756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15009920299053192,
      "rewards/margins": 0.12188665568828583,
      "rewards/rejected": -0.27198585867881775,
      "step": 1095
    },
    {
      "epoch": 1.7265563435776201,
      "grad_norm": 0.2343885749578476,
      "learning_rate": 2.2693113015993438e-07,
      "log_odds_chosen": 1.3631253242492676,
      "log_odds_ratio": -0.24772821366786957,
      "logits/chosen": -0.7610951066017151,
      "logits/rejected": -1.2148545980453491,
      "logps/chosen": -1.5088468790054321,
      "logps/rejected": -2.6980226039886475,
      "loss": 1.5286,
      "nll_loss": 1.5037950277328491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1508846879005432,
      "rewards/margins": 0.1189175620675087,
      "rewards/rejected": -0.2698022425174713,
      "step": 1096
    },
    {
      "epoch": 1.728132387706856,
      "grad_norm": 0.23901303112506866,
      "learning_rate": 2.2439452804029524e-07,
      "log_odds_chosen": 1.5531229972839355,
      "log_odds_ratio": -0.20120078325271606,
      "logits/chosen": -0.6074272990226746,
      "logits/rejected": -1.278278112411499,
      "logps/chosen": -1.486811637878418,
      "logps/rejected": -2.8438713550567627,
      "loss": 1.5,
      "nll_loss": 1.4799227714538574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486811637878418,
      "rewards/margins": 0.13570597767829895,
      "rewards/rejected": -0.28438714146614075,
      "step": 1097
    },
    {
      "epoch": 1.7297084318360914,
      "grad_norm": 0.23279216885566711,
      "learning_rate": 2.2187133977190343e-07,
      "log_odds_chosen": 1.5228835344314575,
      "log_odds_ratio": -0.21214833855628967,
      "logits/chosen": -0.7547963857650757,
      "logits/rejected": -1.2279223203659058,
      "logps/chosen": -1.4209389686584473,
      "logps/rejected": -2.7366926670074463,
      "loss": 1.4348,
      "nll_loss": 1.4135842323303223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14209389686584473,
      "rewards/margins": 0.13157537579536438,
      "rewards/rejected": -0.2736692428588867,
      "step": 1098
    },
    {
      "epoch": 1.731284475965327,
      "grad_norm": 0.26026374101638794,
      "learning_rate": 2.193615844162411e-07,
      "log_odds_chosen": 1.4330053329467773,
      "log_odds_ratio": -0.2262742519378662,
      "logits/chosen": -0.7496172189712524,
      "logits/rejected": -1.3490166664123535,
      "logps/chosen": -1.4422742128372192,
      "logps/rejected": -2.677133560180664,
      "loss": 1.4613,
      "nll_loss": 1.4387012720108032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14422741532325745,
      "rewards/margins": 0.12348593771457672,
      "rewards/rejected": -0.26771336793899536,
      "step": 1099
    },
    {
      "epoch": 1.7328605200945626,
      "grad_norm": 0.22931641340255737,
      "learning_rate": 2.1686528093331113e-07,
      "log_odds_chosen": 1.769337773323059,
      "log_odds_ratio": -0.17033161222934723,
      "logits/chosen": -0.8707290887832642,
      "logits/rejected": -1.350868582725525,
      "logps/chosen": -1.351546287536621,
      "logps/rejected": -2.8783702850341797,
      "loss": 1.3709,
      "nll_loss": 1.3538639545440674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1351546198129654,
      "rewards/margins": 0.15268239378929138,
      "rewards/rejected": -0.28783702850341797,
      "step": 1100
    },
    {
      "epoch": 1.7344365642237982,
      "grad_norm": 0.23415732383728027,
      "learning_rate": 2.1438244818149465e-07,
      "log_odds_chosen": 1.5698161125183105,
      "log_odds_ratio": -0.191875621676445,
      "logits/chosen": -0.8444148898124695,
      "logits/rejected": -1.3804283142089844,
      "logps/chosen": -1.4618314504623413,
      "logps/rejected": -2.8277666568756104,
      "loss": 1.4789,
      "nll_loss": 1.4596917629241943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14618316292762756,
      "rewards/margins": 0.1365935206413269,
      "rewards/rejected": -0.28277668356895447,
      "step": 1101
    },
    {
      "epoch": 1.7360126083530338,
      "grad_norm": 0.2691333293914795,
      "learning_rate": 2.1191310491740633e-07,
      "log_odds_chosen": 1.1653332710266113,
      "log_odds_ratio": -0.2780286371707916,
      "logits/chosen": -0.6778568625450134,
      "logits/rejected": -1.1628938913345337,
      "logps/chosen": -1.5779513120651245,
      "logps/rejected": -2.5918850898742676,
      "loss": 1.5971,
      "nll_loss": 1.5692942142486572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15779513120651245,
      "rewards/margins": 0.10139335691928864,
      "rewards/rejected": -0.2591885030269623,
      "step": 1102
    },
    {
      "epoch": 1.7375886524822695,
      "grad_norm": 0.22865431010723114,
      "learning_rate": 2.0945726979575528e-07,
      "log_odds_chosen": 1.674941897392273,
      "log_odds_ratio": -0.1934119611978531,
      "logits/chosen": -0.7219018936157227,
      "logits/rejected": -1.3789584636688232,
      "logps/chosen": -1.453956127166748,
      "logps/rejected": -2.9248759746551514,
      "loss": 1.4739,
      "nll_loss": 1.4545761346817017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14539560675621033,
      "rewards/margins": 0.14709198474884033,
      "rewards/rejected": -0.29248762130737305,
      "step": 1103
    },
    {
      "epoch": 1.739164696611505,
      "grad_norm": 0.2604381740093231,
      "learning_rate": 2.0701496136920315e-07,
      "log_odds_chosen": 1.607742428779602,
      "log_odds_ratio": -0.1846640706062317,
      "logits/chosen": -0.7378330826759338,
      "logits/rejected": -1.2699545621871948,
      "logps/chosen": -1.4123570919036865,
      "logps/rejected": -2.79866623878479,
      "loss": 1.4284,
      "nll_loss": 1.4099434614181519,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14123570919036865,
      "rewards/margins": 0.1386309117078781,
      "rewards/rejected": -0.27986663579940796,
      "step": 1104
    },
    {
      "epoch": 1.7407407407407407,
      "grad_norm": 0.2146187126636505,
      "learning_rate": 2.0458619808822352e-07,
      "log_odds_chosen": 1.5596054792404175,
      "log_odds_ratio": -0.19823578000068665,
      "logits/chosen": -0.7286146879196167,
      "logits/rejected": -1.2656656503677368,
      "logps/chosen": -1.3671915531158447,
      "logps/rejected": -2.700838565826416,
      "loss": 1.404,
      "nll_loss": 1.3841631412506104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13671916723251343,
      "rewards/margins": 0.13336467742919922,
      "rewards/rejected": -0.27008384466171265,
      "step": 1105
    },
    {
      "epoch": 1.7423167848699763,
      "grad_norm": 0.22836244106292725,
      "learning_rate": 2.0217099830096297e-07,
      "log_odds_chosen": 1.6839842796325684,
      "log_odds_ratio": -0.17526845633983612,
      "logits/chosen": -0.7671194672584534,
      "logits/rejected": -1.31965172290802,
      "logps/chosen": -1.3381903171539307,
      "logps/rejected": -2.777988910675049,
      "loss": 1.3684,
      "nll_loss": 1.3508535623550415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13381902873516083,
      "rewards/margins": 0.14397984743118286,
      "rewards/rejected": -0.2777988910675049,
      "step": 1106
    },
    {
      "epoch": 1.743892828999212,
      "grad_norm": 0.22614014148712158,
      "learning_rate": 1.9976938025310196e-07,
      "log_odds_chosen": 1.6070036888122559,
      "log_odds_ratio": -0.19464688003063202,
      "logits/chosen": -0.8087075352668762,
      "logits/rejected": -1.2915080785751343,
      "logps/chosen": -1.4368107318878174,
      "logps/rejected": -2.834345817565918,
      "loss": 1.4555,
      "nll_loss": 1.4359925985336304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14368107914924622,
      "rewards/margins": 0.1397535353899002,
      "rewards/rejected": -0.28343459963798523,
      "step": 1107
    },
    {
      "epoch": 1.7454688731284476,
      "grad_norm": 0.22526815533638,
      "learning_rate": 1.9738136208771783e-07,
      "log_odds_chosen": 1.5386714935302734,
      "log_odds_ratio": -0.19958914816379547,
      "logits/chosen": -0.778233528137207,
      "logits/rejected": -1.431180477142334,
      "logps/chosen": -1.4460015296936035,
      "logps/rejected": -2.7798283100128174,
      "loss": 1.458,
      "nll_loss": 1.4380881786346436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14460016787052155,
      "rewards/margins": 0.13338269293308258,
      "rewards/rejected": -0.27798283100128174,
      "step": 1108
    },
    {
      "epoch": 1.7470449172576832,
      "grad_norm": 0.22066886723041534,
      "learning_rate": 1.950069618451473e-07,
      "log_odds_chosen": 1.3709889650344849,
      "log_odds_ratio": -0.23923352360725403,
      "logits/chosen": -0.7511844038963318,
      "logits/rejected": -1.2242106199264526,
      "logps/chosen": -1.3519700765609741,
      "logps/rejected": -2.510502338409424,
      "loss": 1.3941,
      "nll_loss": 1.3702255487442017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1351970136165619,
      "rewards/margins": 0.1158532202243805,
      "rewards/rejected": -0.2510502338409424,
      "step": 1109
    },
    {
      "epoch": 1.7486209613869188,
      "grad_norm": 0.2260175496339798,
      "learning_rate": 1.9264619746285016e-07,
      "log_odds_chosen": 1.2615056037902832,
      "log_odds_ratio": -0.26120030879974365,
      "logits/chosen": -0.8300253748893738,
      "logits/rejected": -1.190067172050476,
      "logps/chosen": -1.4061715602874756,
      "logps/rejected": -2.4754204750061035,
      "loss": 1.4338,
      "nll_loss": 1.4076846837997437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14061716198921204,
      "rewards/margins": 0.1069248840212822,
      "rewards/rejected": -0.24754205346107483,
      "step": 1110
    },
    {
      "epoch": 1.7501970055161544,
      "grad_norm": 0.23676197230815887,
      "learning_rate": 1.9029908677527406e-07,
      "log_odds_chosen": 1.6500422954559326,
      "log_odds_ratio": -0.1856077015399933,
      "logits/chosen": -0.7866255044937134,
      "logits/rejected": -1.3045965433120728,
      "logps/chosen": -1.4182076454162598,
      "logps/rejected": -2.8487212657928467,
      "loss": 1.4368,
      "nll_loss": 1.4182072877883911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1418207436800003,
      "rewards/margins": 0.1430513709783554,
      "rewards/rejected": -0.2848721146583557,
      "step": 1111
    },
    {
      "epoch": 1.75177304964539,
      "grad_norm": 0.22159172594547272,
      "learning_rate": 1.8796564751371836e-07,
      "log_odds_chosen": 1.3228178024291992,
      "log_odds_ratio": -0.2504967153072357,
      "logits/chosen": -0.6887884140014648,
      "logits/rejected": -1.2024201154708862,
      "logps/chosen": -1.45133638381958,
      "logps/rejected": -2.591181755065918,
      "loss": 1.4938,
      "nll_loss": 1.4687652587890625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14513364434242249,
      "rewards/margins": 0.11398456990718842,
      "rewards/rejected": -0.2591181993484497,
      "step": 1112
    },
    {
      "epoch": 1.7533490937746257,
      "grad_norm": 0.23400218784809113,
      "learning_rate": 1.8564589730620272e-07,
      "log_odds_chosen": 1.4892470836639404,
      "log_odds_ratio": -0.21311186254024506,
      "logits/chosen": -0.772849977016449,
      "logits/rejected": -1.3079144954681396,
      "logps/chosen": -1.455101490020752,
      "logps/rejected": -2.7471742630004883,
      "loss": 1.4907,
      "nll_loss": 1.4693611860275269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14551016688346863,
      "rewards/margins": 0.1292072832584381,
      "rewards/rejected": -0.27471745014190674,
      "step": 1113
    },
    {
      "epoch": 1.7549251379038613,
      "grad_norm": 0.22183656692504883,
      "learning_rate": 1.8333985367733206e-07,
      "log_odds_chosen": 1.6202951669692993,
      "log_odds_ratio": -0.19165916740894318,
      "logits/chosen": -0.6893625259399414,
      "logits/rejected": -1.4959697723388672,
      "logps/chosen": -1.4905458688735962,
      "logps/rejected": -2.915602207183838,
      "loss": 1.5087,
      "nll_loss": 1.4895038604736328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14905458688735962,
      "rewards/margins": 0.14250563085079193,
      "rewards/rejected": -0.29156023263931274,
      "step": 1114
    },
    {
      "epoch": 1.756501182033097,
      "grad_norm": 0.23356889188289642,
      "learning_rate": 1.810475340481641e-07,
      "log_odds_chosen": 1.356624960899353,
      "log_odds_ratio": -0.2530226409435272,
      "logits/chosen": -0.7619829773902893,
      "logits/rejected": -1.0908077955245972,
      "logps/chosen": -1.401320457458496,
      "logps/rejected": -2.5607895851135254,
      "loss": 1.4217,
      "nll_loss": 1.3964362144470215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14013205468654633,
      "rewards/margins": 0.11594691872596741,
      "rewards/rejected": -0.25607895851135254,
      "step": 1115
    },
    {
      "epoch": 1.7580772261623325,
      "grad_norm": 0.22855843603610992,
      "learning_rate": 1.7876895573607896e-07,
      "log_odds_chosen": 1.3119741678237915,
      "log_odds_ratio": -0.24236130714416504,
      "logits/chosen": -0.7270234823226929,
      "logits/rejected": -1.3173134326934814,
      "logps/chosen": -1.4478003978729248,
      "logps/rejected": -2.569993734359741,
      "loss": 1.4709,
      "nll_loss": 1.44668447971344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14478005468845367,
      "rewards/margins": 0.11221931874752045,
      "rewards/rejected": -0.2569993734359741,
      "step": 1116
    },
    {
      "epoch": 1.7596532702915682,
      "grad_norm": 0.24856072664260864,
      "learning_rate": 1.7650413595464776e-07,
      "log_odds_chosen": 1.4170514345169067,
      "log_odds_ratio": -0.22651295363903046,
      "logits/chosen": -0.8547971248626709,
      "logits/rejected": -1.3236238956451416,
      "logps/chosen": -1.4374067783355713,
      "logps/rejected": -2.6572020053863525,
      "loss": 1.4568,
      "nll_loss": 1.4341700077056885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437406837940216,
      "rewards/margins": 0.12197950482368469,
      "rewards/rejected": -0.2657201886177063,
      "step": 1117
    },
    {
      "epoch": 1.7612293144208038,
      "grad_norm": 0.22628292441368103,
      "learning_rate": 1.7425309181350167e-07,
      "log_odds_chosen": 1.357126235961914,
      "log_odds_ratio": -0.23728694021701813,
      "logits/chosen": -0.7134295701980591,
      "logits/rejected": -1.3132483959197998,
      "logps/chosen": -1.4017131328582764,
      "logps/rejected": -2.55692458152771,
      "loss": 1.4289,
      "nll_loss": 1.4051454067230225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14017131924629211,
      "rewards/margins": 0.115521140396595,
      "rewards/rejected": -0.2556924521923065,
      "step": 1118
    },
    {
      "epoch": 1.7628053585500394,
      "grad_norm": 0.21013616025447845,
      "learning_rate": 1.7201584031820415e-07,
      "log_odds_chosen": 1.3031381368637085,
      "log_odds_ratio": -0.24637249112129211,
      "logits/chosen": -0.656343936920166,
      "logits/rejected": -1.3181886672973633,
      "logps/chosen": -1.3066339492797852,
      "logps/rejected": -2.3811233043670654,
      "loss": 1.3551,
      "nll_loss": 1.3304535150527954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1306634247303009,
      "rewards/margins": 0.10744892060756683,
      "rewards/rejected": -0.23811233043670654,
      "step": 1119
    },
    {
      "epoch": 1.764381402679275,
      "grad_norm": 0.2354174703359604,
      "learning_rate": 1.6979239837012193e-07,
      "log_odds_chosen": 1.46872878074646,
      "log_odds_ratio": -0.22346088290214539,
      "logits/chosen": -0.6933087110519409,
      "logits/rejected": -1.2366865873336792,
      "logps/chosen": -1.3901395797729492,
      "logps/rejected": -2.6444902420043945,
      "loss": 1.417,
      "nll_loss": 1.3946441411972046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13901396095752716,
      "rewards/margins": 0.12543508410453796,
      "rewards/rejected": -0.2644490599632263,
      "step": 1120
    },
    {
      "epoch": 1.7659574468085106,
      "grad_norm": 0.21876153349876404,
      "learning_rate": 1.6758278276629655e-07,
      "log_odds_chosen": 1.4676830768585205,
      "log_odds_ratio": -0.21079806983470917,
      "logits/chosen": -0.6159226298332214,
      "logits/rejected": -1.3478585481643677,
      "logps/chosen": -1.4726089239120483,
      "logps/rejected": -2.7473206520080566,
      "loss": 1.5003,
      "nll_loss": 1.4792267084121704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1472609043121338,
      "rewards/margins": 0.1274711638689041,
      "rewards/rejected": -0.2747320532798767,
      "step": 1121
    },
    {
      "epoch": 1.7675334909377463,
      "grad_norm": 0.23661108314990997,
      "learning_rate": 1.6538701019931867e-07,
      "log_odds_chosen": 1.5063228607177734,
      "log_odds_ratio": -0.20788441598415375,
      "logits/chosen": -0.8456476926803589,
      "logits/rejected": -1.2916640043258667,
      "logps/chosen": -1.337881326675415,
      "logps/rejected": -2.611572742462158,
      "loss": 1.3735,
      "nll_loss": 1.352730631828308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13378813862800598,
      "rewards/margins": 0.12736913561820984,
      "rewards/rejected": -0.2611572742462158,
      "step": 1122
    },
    {
      "epoch": 1.7691095350669819,
      "grad_norm": 0.2350674718618393,
      "learning_rate": 1.6320509725720123e-07,
      "log_odds_chosen": 1.623064637184143,
      "log_odds_ratio": -0.1886724829673767,
      "logits/chosen": -0.7603362202644348,
      "logits/rejected": -1.3391234874725342,
      "logps/chosen": -1.432381510734558,
      "logps/rejected": -2.8437681198120117,
      "loss": 1.4664,
      "nll_loss": 1.4475340843200684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14323817193508148,
      "rewards/margins": 0.14113864302635193,
      "rewards/rejected": -0.2843768000602722,
      "step": 1123
    },
    {
      "epoch": 1.7706855791962175,
      "grad_norm": 0.24000860750675201,
      "learning_rate": 1.6103706042325426e-07,
      "log_odds_chosen": 1.7501269578933716,
      "log_odds_ratio": -0.17429955303668976,
      "logits/chosen": -0.8209456205368042,
      "logits/rejected": -1.2817790508270264,
      "logps/chosen": -1.4667109251022339,
      "logps/rejected": -3.0067834854125977,
      "loss": 1.4858,
      "nll_loss": 1.468417763710022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1466710865497589,
      "rewards/margins": 0.15400725603103638,
      "rewards/rejected": -0.3006783723831177,
      "step": 1124
    },
    {
      "epoch": 1.7722616233254531,
      "grad_norm": 0.2564554512500763,
      "learning_rate": 1.5888291607596127e-07,
      "log_odds_chosen": 1.7544467449188232,
      "log_odds_ratio": -0.16740372776985168,
      "logits/chosen": -0.8628431558609009,
      "logits/rejected": -1.3423247337341309,
      "logps/chosen": -1.3637850284576416,
      "logps/rejected": -2.882831335067749,
      "loss": 1.3893,
      "nll_loss": 1.3725788593292236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13637851178646088,
      "rewards/margins": 0.15190459787845612,
      "rewards/rejected": -0.288283109664917,
      "step": 1125
    },
    {
      "epoch": 1.7738376674546887,
      "grad_norm": 0.2419404685497284,
      "learning_rate": 1.5674268048885275e-07,
      "log_odds_chosen": 1.5502468347549438,
      "log_odds_ratio": -0.20392455160617828,
      "logits/chosen": -0.7084717154502869,
      "logits/rejected": -1.305037498474121,
      "logps/chosen": -1.445732831954956,
      "logps/rejected": -2.795175075531006,
      "loss": 1.4822,
      "nll_loss": 1.4618072509765625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14457327127456665,
      "rewards/margins": 0.13494426012039185,
      "rewards/rejected": -0.2795175313949585,
      "step": 1126
    },
    {
      "epoch": 1.7754137115839244,
      "grad_norm": 0.23606880009174347,
      "learning_rate": 1.5461636983038685e-07,
      "log_odds_chosen": 1.4226720333099365,
      "log_odds_ratio": -0.22592544555664062,
      "logits/chosen": -0.6881420612335205,
      "logits/rejected": -1.343613624572754,
      "logps/chosen": -1.5351693630218506,
      "logps/rejected": -2.7749063968658447,
      "loss": 1.5312,
      "nll_loss": 1.5085994005203247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.153516948223114,
      "rewards/margins": 0.12397372722625732,
      "rewards/rejected": -0.27749064564704895,
      "step": 1127
    },
    {
      "epoch": 1.77698975571316,
      "grad_norm": 0.262669175863266,
      "learning_rate": 1.525040001638249e-07,
      "log_odds_chosen": 1.4314956665039062,
      "log_odds_ratio": -0.22615548968315125,
      "logits/chosen": -0.759786069393158,
      "logits/rejected": -1.3004142045974731,
      "logps/chosen": -1.4292715787887573,
      "logps/rejected": -2.6597883701324463,
      "loss": 1.4497,
      "nll_loss": 1.4271137714385986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1429271548986435,
      "rewards/margins": 0.12305169552564621,
      "rewards/rejected": -0.2659788429737091,
      "step": 1128
    },
    {
      "epoch": 1.7785657998423956,
      "grad_norm": 0.2353297621011734,
      "learning_rate": 1.5040558744711086e-07,
      "log_odds_chosen": 1.7406706809997559,
      "log_odds_ratio": -0.1696540117263794,
      "logits/chosen": -0.7915275692939758,
      "logits/rejected": -1.370506763458252,
      "logps/chosen": -1.4114065170288086,
      "logps/rejected": -2.927774429321289,
      "loss": 1.439,
      "nll_loss": 1.422033429145813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411406397819519,
      "rewards/margins": 0.15163680911064148,
      "rewards/rejected": -0.2927774488925934,
      "step": 1129
    },
    {
      "epoch": 1.7801418439716312,
      "grad_norm": 0.22322209179401398,
      "learning_rate": 1.4832114753275082e-07,
      "log_odds_chosen": 1.370383381843567,
      "log_odds_ratio": -0.24547740817070007,
      "logits/chosen": -0.850324809551239,
      "logits/rejected": -1.232373833656311,
      "logps/chosen": -1.3797056674957275,
      "logps/rejected": -2.550323486328125,
      "loss": 1.4104,
      "nll_loss": 1.3858219385147095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13797056674957275,
      "rewards/margins": 0.1170617938041687,
      "rewards/rejected": -0.25503236055374146,
      "step": 1130
    },
    {
      "epoch": 1.7817178881008668,
      "grad_norm": 0.2630787491798401,
      "learning_rate": 1.4625069616769215e-07,
      "log_odds_chosen": 1.6178879737854004,
      "log_odds_ratio": -0.1839151680469513,
      "logits/chosen": -0.829473614692688,
      "logits/rejected": -1.2938966751098633,
      "logps/chosen": -1.4278398752212524,
      "logps/rejected": -2.8271429538726807,
      "loss": 1.4494,
      "nll_loss": 1.431006669998169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1427839696407318,
      "rewards/margins": 0.13993030786514282,
      "rewards/rejected": -0.282714307308197,
      "step": 1131
    },
    {
      "epoch": 1.7832939322301025,
      "grad_norm": 0.2403954565525055,
      "learning_rate": 1.4419424899320643e-07,
      "log_odds_chosen": 1.6058695316314697,
      "log_odds_ratio": -0.18963071703910828,
      "logits/chosen": -0.7552633285522461,
      "logits/rejected": -1.3706896305084229,
      "logps/chosen": -1.4973756074905396,
      "logps/rejected": -2.900517463684082,
      "loss": 1.5048,
      "nll_loss": 1.4858349561691284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14973755180835724,
      "rewards/margins": 0.14031419157981873,
      "rewards/rejected": -0.29005175828933716,
      "step": 1132
    },
    {
      "epoch": 1.784869976359338,
      "grad_norm": 0.23331791162490845,
      "learning_rate": 1.4215182154476945e-07,
      "log_odds_chosen": 1.614305019378662,
      "log_odds_ratio": -0.19632524251937866,
      "logits/chosen": -0.7328482270240784,
      "logits/rejected": -1.3926458358764648,
      "logps/chosen": -1.4333982467651367,
      "logps/rejected": -2.8395960330963135,
      "loss": 1.4627,
      "nll_loss": 1.4430866241455078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433398425579071,
      "rewards/margins": 0.14061978459358215,
      "rewards/rejected": -0.28395962715148926,
      "step": 1133
    },
    {
      "epoch": 1.7864460204885737,
      "grad_norm": 0.22593872249126434,
      "learning_rate": 1.401234292519453e-07,
      "log_odds_chosen": 1.804638147354126,
      "log_odds_ratio": -0.15968160331249237,
      "logits/chosen": -0.7729154825210571,
      "logits/rejected": -1.453240990638733,
      "logps/chosen": -1.3926777839660645,
      "logps/rejected": -2.963444709777832,
      "loss": 1.4199,
      "nll_loss": 1.4039463996887207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13926777243614197,
      "rewards/margins": 0.15707670152187347,
      "rewards/rejected": -0.29634445905685425,
      "step": 1134
    },
    {
      "epoch": 1.7880220646178093,
      "grad_norm": 0.24465757608413696,
      "learning_rate": 1.3810908743826933e-07,
      "log_odds_chosen": 1.478600263595581,
      "log_odds_ratio": -0.21053171157836914,
      "logits/chosen": -0.8441611528396606,
      "logits/rejected": -1.2586935758590698,
      "logps/chosen": -1.408511757850647,
      "logps/rejected": -2.672636032104492,
      "loss": 1.4392,
      "nll_loss": 1.4180972576141357,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1408511847257614,
      "rewards/margins": 0.12641240656375885,
      "rewards/rejected": -0.26726359128952026,
      "step": 1135
    },
    {
      "epoch": 1.789598108747045,
      "grad_norm": 0.22021645307540894,
      "learning_rate": 1.3610881132113107e-07,
      "log_odds_chosen": 1.4241762161254883,
      "log_odds_ratio": -0.2186574786901474,
      "logits/chosen": -0.7946289777755737,
      "logits/rejected": -1.174445629119873,
      "logps/chosen": -1.3291914463043213,
      "logps/rejected": -2.528582811355591,
      "loss": 1.3661,
      "nll_loss": 1.3442609310150146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13291914761066437,
      "rewards/margins": 0.11993914842605591,
      "rewards/rejected": -0.2528582811355591,
      "step": 1136
    },
    {
      "epoch": 1.7911741528762806,
      "grad_norm": 0.22597524523735046,
      "learning_rate": 1.3412261601166131e-07,
      "log_odds_chosen": 1.3984589576721191,
      "log_odds_ratio": -0.22886118292808533,
      "logits/chosen": -0.5737169981002808,
      "logits/rejected": -1.2498873472213745,
      "logps/chosen": -1.319138526916504,
      "logps/rejected": -2.4965386390686035,
      "loss": 1.3568,
      "nll_loss": 1.3339362144470215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13191385567188263,
      "rewards/margins": 0.11774002760648727,
      "rewards/rejected": -0.2496538758277893,
      "step": 1137
    },
    {
      "epoch": 1.7927501970055162,
      "grad_norm": 0.238573357462883,
      "learning_rate": 1.3215051651461706e-07,
      "log_odds_chosen": 1.6665571928024292,
      "log_odds_ratio": -0.18614430725574493,
      "logits/chosen": -0.7269477248191833,
      "logits/rejected": -1.3768529891967773,
      "logps/chosen": -1.3470789194107056,
      "logps/rejected": -2.7812910079956055,
      "loss": 1.3749,
      "nll_loss": 1.3563309907913208,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13470788300037384,
      "rewards/margins": 0.1434212327003479,
      "rewards/rejected": -0.27812913060188293,
      "step": 1138
    },
    {
      "epoch": 1.7943262411347518,
      "grad_norm": 0.2678523063659668,
      "learning_rate": 1.3019252772826872e-07,
      "log_odds_chosen": 1.6223804950714111,
      "log_odds_ratio": -0.18182188272476196,
      "logits/chosen": -0.7724013328552246,
      "logits/rejected": -1.3519530296325684,
      "logps/chosen": -1.4258885383605957,
      "logps/rejected": -2.831117630004883,
      "loss": 1.4572,
      "nll_loss": 1.4390065670013428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14258883893489838,
      "rewards/margins": 0.14052295684814453,
      "rewards/rejected": -0.2831118106842041,
      "step": 1139
    },
    {
      "epoch": 1.7959022852639874,
      "grad_norm": 0.23565053939819336,
      "learning_rate": 1.2824866444428527e-07,
      "log_odds_chosen": 1.5018936395645142,
      "log_odds_ratio": -0.2357957363128662,
      "logits/chosen": -0.7087265253067017,
      "logits/rejected": -1.3272160291671753,
      "logps/chosen": -1.539255142211914,
      "logps/rejected": -2.8667397499084473,
      "loss": 1.5456,
      "nll_loss": 1.5220212936401367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15392550826072693,
      "rewards/margins": 0.13274846971035004,
      "rewards/rejected": -0.28667399287223816,
      "step": 1140
    },
    {
      "epoch": 1.797478329393223,
      "grad_norm": 0.2504424750804901,
      "learning_rate": 1.2631894134762577e-07,
      "log_odds_chosen": 1.6188279390335083,
      "log_odds_ratio": -0.1868515908718109,
      "logits/chosen": -0.7609795331954956,
      "logits/rejected": -1.440920114517212,
      "logps/chosen": -1.4435534477233887,
      "logps/rejected": -2.8535211086273193,
      "loss": 1.4529,
      "nll_loss": 1.4342293739318848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14435535669326782,
      "rewards/margins": 0.14099673926830292,
      "rewards/rejected": -0.28535208106040955,
      "step": 1141
    },
    {
      "epoch": 1.7990543735224587,
      "grad_norm": 0.22281765937805176,
      "learning_rate": 1.2440337301642668e-07,
      "log_odds_chosen": 1.678703784942627,
      "log_odds_ratio": -0.18105661869049072,
      "logits/chosen": -0.8117111921310425,
      "logits/rejected": -1.3589638471603394,
      "logps/chosen": -1.3575385808944702,
      "logps/rejected": -2.80513596534729,
      "loss": 1.4033,
      "nll_loss": 1.3852283954620361,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13575385510921478,
      "rewards/margins": 0.14475972950458527,
      "rewards/rejected": -0.28051358461380005,
      "step": 1142
    },
    {
      "epoch": 1.8006304176516943,
      "grad_norm": 0.21029916405677795,
      "learning_rate": 1.2250197392189176e-07,
      "log_odds_chosen": 1.6705410480499268,
      "log_odds_ratio": -0.19274136424064636,
      "logits/chosen": -0.6778602600097656,
      "logits/rejected": -1.3861171007156372,
      "logps/chosen": -1.3475359678268433,
      "logps/rejected": -2.781855821609497,
      "loss": 1.3923,
      "nll_loss": 1.3730212450027466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13475358486175537,
      "rewards/margins": 0.14343202114105225,
      "rewards/rejected": -0.27818557620048523,
      "step": 1143
    },
    {
      "epoch": 1.80220646178093,
      "grad_norm": 0.2694167196750641,
      "learning_rate": 1.2061475842818335e-07,
      "log_odds_chosen": 1.6148478984832764,
      "log_odds_ratio": -0.19423368573188782,
      "logits/chosen": -0.7575897574424744,
      "logits/rejected": -1.1197636127471924,
      "logps/chosen": -1.3775649070739746,
      "logps/rejected": -2.76629376411438,
      "loss": 1.3997,
      "nll_loss": 1.380262851715088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13775651156902313,
      "rewards/margins": 0.1388728767633438,
      "rewards/rejected": -0.27662938833236694,
      "step": 1144
    },
    {
      "epoch": 1.8037825059101655,
      "grad_norm": 0.23819100856781006,
      "learning_rate": 1.1874174079231236e-07,
      "log_odds_chosen": 1.7266193628311157,
      "log_odds_ratio": -0.17709007859230042,
      "logits/chosen": -0.7968311309814453,
      "logits/rejected": -1.2926256656646729,
      "logps/chosen": -1.4249471426010132,
      "logps/rejected": -2.9351184368133545,
      "loss": 1.4609,
      "nll_loss": 1.443165898323059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14249470829963684,
      "rewards/margins": 0.15101714432239532,
      "rewards/rejected": -0.29351186752319336,
      "step": 1145
    },
    {
      "epoch": 1.8053585500394012,
      "grad_norm": 0.2868101894855499,
      "learning_rate": 1.1688293516403258e-07,
      "log_odds_chosen": 1.6836891174316406,
      "log_odds_ratio": -0.1828700304031372,
      "logits/chosen": -0.6653310060501099,
      "logits/rejected": -1.265391230583191,
      "logps/chosen": -1.3820736408233643,
      "logps/rejected": -2.836557149887085,
      "loss": 1.4256,
      "nll_loss": 1.4073606729507446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1382073611021042,
      "rewards/margins": 0.14544837176799774,
      "rewards/rejected": -0.28365573287010193,
      "step": 1146
    },
    {
      "epoch": 1.8069345941686368,
      "grad_norm": 0.23426437377929688,
      "learning_rate": 1.1503835558573306e-07,
      "log_odds_chosen": 1.5159103870391846,
      "log_odds_ratio": -0.21577304601669312,
      "logits/chosen": -0.7920680046081543,
      "logits/rejected": -1.1937962770462036,
      "logps/chosen": -1.3875632286071777,
      "logps/rejected": -2.6916627883911133,
      "loss": 1.4189,
      "nll_loss": 1.397277593612671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13875631988048553,
      "rewards/margins": 0.13040995597839355,
      "rewards/rejected": -0.2691662907600403,
      "step": 1147
    },
    {
      "epoch": 1.8085106382978724,
      "grad_norm": 0.25842002034187317,
      "learning_rate": 1.1320801599233144e-07,
      "log_odds_chosen": 1.5368105173110962,
      "log_odds_ratio": -0.19752755761146545,
      "logits/chosen": -0.908570408821106,
      "logits/rejected": -1.3800472021102905,
      "logps/chosen": -1.4037823677062988,
      "logps/rejected": -2.7265493869781494,
      "loss": 1.452,
      "nll_loss": 1.4322675466537476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14037825167179108,
      "rewards/margins": 0.13227669894695282,
      "rewards/rejected": -0.2726549506187439,
      "step": 1148
    },
    {
      "epoch": 1.8100866824271078,
      "grad_norm": 0.22058841586112976,
      "learning_rate": 1.1139193021116877e-07,
      "log_odds_chosen": 1.6911609172821045,
      "log_odds_ratio": -0.18108227849006653,
      "logits/chosen": -0.7455383539199829,
      "logits/rejected": -1.2593343257904053,
      "logps/chosen": -1.3778568506240845,
      "logps/rejected": -2.8417134284973145,
      "loss": 1.4225,
      "nll_loss": 1.4044313430786133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13778570294380188,
      "rewards/margins": 0.14638563990592957,
      "rewards/rejected": -0.28417134284973145,
      "step": 1149
    },
    {
      "epoch": 1.8116627265563436,
      "grad_norm": 0.29143232107162476,
      "learning_rate": 1.0959011196190582e-07,
      "log_odds_chosen": 1.2145510911941528,
      "log_odds_ratio": -0.2672911286354065,
      "logits/chosen": -0.7598279714584351,
      "logits/rejected": -1.1288349628448486,
      "logps/chosen": -1.4554879665374756,
      "logps/rejected": -2.491856098175049,
      "loss": 1.4842,
      "nll_loss": 1.457451581954956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14554880559444427,
      "rewards/margins": 0.10363679379224777,
      "rewards/rejected": -0.24918560683727264,
      "step": 1150
    },
    {
      "epoch": 1.813238770685579,
      "grad_norm": 0.23166853189468384,
      "learning_rate": 1.0780257485641908e-07,
      "log_odds_chosen": 1.4863921403884888,
      "log_odds_ratio": -0.20701424777507782,
      "logits/chosen": -0.7513512372970581,
      "logits/rejected": -1.2983371019363403,
      "logps/chosen": -1.448461890220642,
      "logps/rejected": -2.733353853225708,
      "loss": 1.472,
      "nll_loss": 1.4512637853622437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14484617114067078,
      "rewards/margins": 0.12848922610282898,
      "rewards/rejected": -0.27333539724349976,
      "step": 1151
    },
    {
      "epoch": 1.8148148148148149,
      "grad_norm": 0.23110385239124298,
      "learning_rate": 1.0602933239869738e-07,
      "log_odds_chosen": 1.5836052894592285,
      "log_odds_ratio": -0.19645115733146667,
      "logits/chosen": -0.7566462755203247,
      "logits/rejected": -1.4299547672271729,
      "logps/chosen": -1.365604281425476,
      "logps/rejected": -2.7250795364379883,
      "loss": 1.3907,
      "nll_loss": 1.3710839748382568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13656042516231537,
      "rewards/margins": 0.13594751060009003,
      "rewards/rejected": -0.2725079357624054,
      "step": 1152
    },
    {
      "epoch": 1.8163908589440503,
      "grad_norm": 0.2281046211719513,
      "learning_rate": 1.0427039798474035e-07,
      "log_odds_chosen": 1.6549566984176636,
      "log_odds_ratio": -0.1806105375289917,
      "logits/chosen": -0.7454723715782166,
      "logits/rejected": -1.400409460067749,
      "logps/chosen": -1.387014627456665,
      "logps/rejected": -2.8159637451171875,
      "loss": 1.4114,
      "nll_loss": 1.393372654914856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13870146870613098,
      "rewards/margins": 0.1428949236869812,
      "rewards/rejected": -0.2815963923931122,
      "step": 1153
    },
    {
      "epoch": 1.8179669030732861,
      "grad_norm": 0.25486576557159424,
      "learning_rate": 1.0252578490245811e-07,
      "log_odds_chosen": 1.70491361618042,
      "log_odds_ratio": -0.18127894401550293,
      "logits/chosen": -0.8830330967903137,
      "logits/rejected": -1.3656814098358154,
      "logps/chosen": -1.4611085653305054,
      "logps/rejected": -2.9586310386657715,
      "loss": 1.48,
      "nll_loss": 1.4618921279907227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14611086249351501,
      "rewards/margins": 0.14975225925445557,
      "rewards/rejected": -0.2958631217479706,
      "step": 1154
    },
    {
      "epoch": 1.8195429472025215,
      "grad_norm": 0.2581021785736084,
      "learning_rate": 1.007955063315682e-07,
      "log_odds_chosen": 1.2911866903305054,
      "log_odds_ratio": -0.24953977763652802,
      "logits/chosen": -0.7768049240112305,
      "logits/rejected": -1.3318356275558472,
      "logps/chosen": -1.4672133922576904,
      "logps/rejected": -2.578296422958374,
      "loss": 1.489,
      "nll_loss": 1.4640580415725708,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14672134816646576,
      "rewards/margins": 0.11110830307006836,
      "rewards/rejected": -0.2578296661376953,
      "step": 1155
    },
    {
      "epoch": 1.8211189913317574,
      "grad_norm": 0.2579277455806732,
      "learning_rate": 9.90795753434992e-08,
      "log_odds_chosen": 1.5036219358444214,
      "log_odds_ratio": -0.21895289421081543,
      "logits/chosen": -0.6659034490585327,
      "logits/rejected": -1.3177258968353271,
      "logps/chosen": -1.4911359548568726,
      "logps/rejected": -2.8046486377716064,
      "loss": 1.5246,
      "nll_loss": 1.5026625394821167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14911359548568726,
      "rewards/margins": 0.1313512772321701,
      "rewards/rejected": -0.28046488761901855,
      "step": 1156
    },
    {
      "epoch": 1.8226950354609928,
      "grad_norm": 0.23177717626094818,
      "learning_rate": 9.737800490129023e-08,
      "log_odds_chosen": 1.7746219635009766,
      "log_odds_ratio": -0.1633782833814621,
      "logits/chosen": -0.845552921295166,
      "logits/rejected": -1.388468623161316,
      "logps/chosen": -1.4347904920578003,
      "logps/rejected": -2.9870874881744385,
      "loss": 1.4462,
      "nll_loss": 1.4298994541168213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14347907900810242,
      "rewards/margins": 0.15522968769073486,
      "rewards/rejected": -0.29870879650115967,
      "step": 1157
    },
    {
      "epoch": 1.8242710795902286,
      "grad_norm": 0.22257472574710846,
      "learning_rate": 9.569080785949335e-08,
      "log_odds_chosen": 1.4915363788604736,
      "log_odds_ratio": -0.21492330729961395,
      "logits/chosen": -0.7876976132392883,
      "logits/rejected": -1.3065129518508911,
      "logps/chosen": -1.3312245607376099,
      "logps/rejected": -2.59820556640625,
      "loss": 1.3817,
      "nll_loss": 1.360255479812622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13312245905399323,
      "rewards/margins": 0.12669812142848969,
      "rewards/rejected": -0.2598205804824829,
      "step": 1158
    },
    {
      "epoch": 1.825847123719464,
      "grad_norm": 0.23324795067310333,
      "learning_rate": 9.401799696407642e-08,
      "log_odds_chosen": 1.3423179388046265,
      "log_odds_ratio": -0.24107107520103455,
      "logits/chosen": -0.767767608165741,
      "logits/rejected": -1.3344504833221436,
      "logps/chosen": -1.4657601118087769,
      "logps/rejected": -2.616499662399292,
      "loss": 1.4839,
      "nll_loss": 1.459787130355835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14657600224018097,
      "rewards/margins": 0.11507395654916763,
      "rewards/rejected": -0.2616499662399292,
      "step": 1159
    },
    {
      "epoch": 1.8274231678486998,
      "grad_norm": 0.23683474957942963,
      "learning_rate": 9.23595848523262e-08,
      "log_odds_chosen": 1.4470669031143188,
      "log_odds_ratio": -0.2163505256175995,
      "logits/chosen": -0.8487231135368347,
      "logits/rejected": -1.3417795896530151,
      "logps/chosen": -1.4167948961257935,
      "logps/rejected": -2.65696120262146,
      "loss": 1.4282,
      "nll_loss": 1.4065958261489868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14167949557304382,
      "rewards/margins": 0.12401662766933441,
      "rewards/rejected": -0.26569613814353943,
      "step": 1160
    },
    {
      "epoch": 1.8289992119779352,
      "grad_norm": 0.23447521030902863,
      "learning_rate": 9.071558405275426e-08,
      "log_odds_chosen": 1.5585451126098633,
      "log_odds_ratio": -0.20580995082855225,
      "logits/chosen": -0.7815247774124146,
      "logits/rejected": -1.3727054595947266,
      "logps/chosen": -1.4294443130493164,
      "logps/rejected": -2.7840020656585693,
      "loss": 1.4531,
      "nll_loss": 1.43251633644104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14294444024562836,
      "rewards/margins": 0.13545577228069305,
      "rewards/rejected": -0.2784002423286438,
      "step": 1161
    },
    {
      "epoch": 1.830575256107171,
      "grad_norm": 0.21697047352790833,
      "learning_rate": 8.90860069850019e-08,
      "log_odds_chosen": 1.6873326301574707,
      "log_odds_ratio": -0.18107396364212036,
      "logits/chosen": -0.7690492868423462,
      "logits/rejected": -1.4026575088500977,
      "logps/chosen": -1.3633358478546143,
      "logps/rejected": -2.816706657409668,
      "loss": 1.3778,
      "nll_loss": 1.3596746921539307,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13633358478546143,
      "rewards/margins": 0.14533713459968567,
      "rewards/rejected": -0.2816707193851471,
      "step": 1162
    },
    {
      "epoch": 1.8321513002364065,
      "grad_norm": 0.2262960821390152,
      "learning_rate": 8.747086595974518e-08,
      "log_odds_chosen": 1.5514763593673706,
      "log_odds_ratio": -0.19570371508598328,
      "logits/chosen": -0.7489581108093262,
      "logits/rejected": -1.4756872653961182,
      "logps/chosen": -1.5099241733551025,
      "logps/rejected": -2.8686232566833496,
      "loss": 1.5146,
      "nll_loss": 1.4950100183486938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15099240839481354,
      "rewards/margins": 0.13586990535259247,
      "rewards/rejected": -0.286862313747406,
      "step": 1163
    },
    {
      "epoch": 1.8337273443656423,
      "grad_norm": 0.24947589635849,
      "learning_rate": 8.58701731786029e-08,
      "log_odds_chosen": 1.6015291213989258,
      "log_odds_ratio": -0.19060420989990234,
      "logits/chosen": -0.7654444575309753,
      "logits/rejected": -1.284880518913269,
      "logps/chosen": -1.4458667039871216,
      "logps/rejected": -2.838963270187378,
      "loss": 1.4707,
      "nll_loss": 1.4516552686691284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14458666741847992,
      "rewards/margins": 0.13930967450141907,
      "rewards/rejected": -0.2838963568210602,
      "step": 1164
    },
    {
      "epoch": 1.8353033884948777,
      "grad_norm": 0.2500733733177185,
      "learning_rate": 8.428394073404477e-08,
      "log_odds_chosen": 1.4967719316482544,
      "log_odds_ratio": -0.2110338807106018,
      "logits/chosen": -0.7898083329200745,
      "logits/rejected": -1.37348473072052,
      "logps/chosen": -1.4781450033187866,
      "logps/rejected": -2.7799336910247803,
      "loss": 1.5093,
      "nll_loss": 1.4881658554077148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14781451225280762,
      "rewards/margins": 0.13017886877059937,
      "rewards/rejected": -0.277993381023407,
      "step": 1165
    },
    {
      "epoch": 1.8368794326241136,
      "grad_norm": 0.22932526469230652,
      "learning_rate": 8.271218060929919e-08,
      "log_odds_chosen": 1.4440068006515503,
      "log_odds_ratio": -0.21521611511707306,
      "logits/chosen": -0.7285287380218506,
      "logits/rejected": -1.300408124923706,
      "logps/chosen": -1.3708878755569458,
      "logps/rejected": -2.594846487045288,
      "loss": 1.4057,
      "nll_loss": 1.3841726779937744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13708879053592682,
      "rewards/margins": 0.12239589542150497,
      "rewards/rejected": -0.2594846487045288,
      "step": 1166
    },
    {
      "epoch": 1.838455476753349,
      "grad_norm": 0.239097461104393,
      "learning_rate": 8.115490467826291e-08,
      "log_odds_chosen": 1.7073861360549927,
      "log_odds_ratio": -0.17147977650165558,
      "logits/chosen": -0.8338723182678223,
      "logits/rejected": -1.3982293605804443,
      "logps/chosen": -1.3771203756332397,
      "logps/rejected": -2.8488290309906006,
      "loss": 1.3874,
      "nll_loss": 1.3702096939086914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1377120465040207,
      "rewards/margins": 0.14717084169387817,
      "rewards/rejected": -0.28488287329673767,
      "step": 1167
    },
    {
      "epoch": 1.8400315208825848,
      "grad_norm": 0.2430301308631897,
      "learning_rate": 7.961212470541245e-08,
      "log_odds_chosen": 1.5764104127883911,
      "log_odds_ratio": -0.1918560266494751,
      "logits/chosen": -0.6446243524551392,
      "logits/rejected": -1.308946967124939,
      "logps/chosen": -1.3944798707962036,
      "logps/rejected": -2.7528529167175293,
      "loss": 1.4173,
      "nll_loss": 1.3981373310089111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13944798707962036,
      "rewards/margins": 0.13583731651306152,
      "rewards/rejected": -0.2752853035926819,
      "step": 1168
    },
    {
      "epoch": 1.8416075650118202,
      "grad_norm": 0.23744869232177734,
      "learning_rate": 7.808385234571302e-08,
      "log_odds_chosen": 1.3301584720611572,
      "log_odds_ratio": -0.24593155086040497,
      "logits/chosen": -0.829200029373169,
      "logits/rejected": -1.3096238374710083,
      "logps/chosen": -1.5045562982559204,
      "logps/rejected": -2.6582000255584717,
      "loss": 1.5138,
      "nll_loss": 1.4891588687896729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15045562386512756,
      "rewards/margins": 0.11536438018083572,
      "rewards/rejected": -0.2658200263977051,
      "step": 1169
    },
    {
      "epoch": 1.843183609141056,
      "grad_norm": 0.22702471911907196,
      "learning_rate": 7.657009914453239e-08,
      "log_odds_chosen": 1.5219887495040894,
      "log_odds_ratio": -0.208509162068367,
      "logits/chosen": -0.749748706817627,
      "logits/rejected": -1.335373878479004,
      "logps/chosen": -1.343932032585144,
      "logps/rejected": -2.6377310752868652,
      "loss": 1.3848,
      "nll_loss": 1.3639109134674072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13439320027828217,
      "rewards/margins": 0.12937989830970764,
      "rewards/rejected": -0.263773113489151,
      "step": 1170
    },
    {
      "epoch": 1.8447596532702915,
      "grad_norm": 0.23578287661075592,
      "learning_rate": 7.507087653755318e-08,
      "log_odds_chosen": 1.957099199295044,
      "log_odds_ratio": -0.14211328327655792,
      "logits/chosen": -0.8127181529998779,
      "logits/rejected": -1.428924560546875,
      "logps/chosen": -1.3919384479522705,
      "logps/rejected": -3.1065356731414795,
      "loss": 1.4197,
      "nll_loss": 1.4055062532424927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1391938328742981,
      "rewards/margins": 0.17145973443984985,
      "rewards/rejected": -0.31065356731414795,
      "step": 1171
    },
    {
      "epoch": 1.8463356973995273,
      "grad_norm": 0.5080424547195435,
      "learning_rate": 7.358619585068582e-08,
      "log_odds_chosen": 1.4170259237289429,
      "log_odds_ratio": -0.23142869770526886,
      "logits/chosen": -0.7551918029785156,
      "logits/rejected": -1.2131149768829346,
      "logps/chosen": -1.2815405130386353,
      "logps/rejected": -2.4578733444213867,
      "loss": 1.349,
      "nll_loss": 1.3258106708526611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12815405428409576,
      "rewards/margins": 0.11763329058885574,
      "rewards/rejected": -0.2457873374223709,
      "step": 1172
    },
    {
      "epoch": 1.8479117415287627,
      "grad_norm": 0.2694801092147827,
      "learning_rate": 7.21160682999844e-08,
      "log_odds_chosen": 1.1463894844055176,
      "log_odds_ratio": -0.2816517949104309,
      "logits/chosen": -0.7518802881240845,
      "logits/rejected": -1.1687921285629272,
      "logps/chosen": -1.4794623851776123,
      "logps/rejected": -2.4587175846099854,
      "loss": 1.4957,
      "nll_loss": 1.467515468597412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14794623851776123,
      "rewards/margins": 0.09792554378509521,
      "rewards/rejected": -0.24587178230285645,
      "step": 1173
    },
    {
      "epoch": 1.8494877856579985,
      "grad_norm": 0.2364802211523056,
      "learning_rate": 7.06605049915594e-08,
      "log_odds_chosen": 1.4850190877914429,
      "log_odds_ratio": -0.21154266595840454,
      "logits/chosen": -0.8156763911247253,
      "logits/rejected": -1.2560564279556274,
      "logps/chosen": -1.395694613456726,
      "logps/rejected": -2.670949697494507,
      "loss": 1.4328,
      "nll_loss": 1.411668062210083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1395694613456726,
      "rewards/margins": 0.12752549350261688,
      "rewards/rejected": -0.2670949697494507,
      "step": 1174
    },
    {
      "epoch": 1.851063829787234,
      "grad_norm": 0.22933442890644073,
      "learning_rate": 6.92195169214973e-08,
      "log_odds_chosen": 1.5471454858779907,
      "log_odds_ratio": -0.19602057337760925,
      "logits/chosen": -0.8003851175308228,
      "logits/rejected": -1.3081281185150146,
      "logps/chosen": -1.3926957845687866,
      "logps/rejected": -2.719444990158081,
      "loss": 1.4206,
      "nll_loss": 1.4009604454040527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13926959037780762,
      "rewards/margins": 0.1326749175786972,
      "rewards/rejected": -0.271944522857666,
      "step": 1175
    },
    {
      "epoch": 1.8526398739164698,
      "grad_norm": 0.2356058955192566,
      "learning_rate": 6.779311497577401e-08,
      "log_odds_chosen": 1.428855538368225,
      "log_odds_ratio": -0.23158419132232666,
      "logits/chosen": -0.8531579971313477,
      "logits/rejected": -1.2958588600158691,
      "logps/chosen": -1.4294098615646362,
      "logps/rejected": -2.6552999019622803,
      "loss": 1.4576,
      "nll_loss": 1.4344730377197266,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14294099807739258,
      "rewards/margins": 0.12258899211883545,
      "rewards/rejected": -0.265529990196228,
      "step": 1176
    },
    {
      "epoch": 1.8542159180457052,
      "grad_norm": 0.3221268355846405,
      "learning_rate": 6.638130993017532e-08,
      "log_odds_chosen": 1.6565438508987427,
      "log_odds_ratio": -0.17625360190868378,
      "logits/chosen": -0.8129989504814148,
      "logits/rejected": -1.3256232738494873,
      "logps/chosen": -1.4045671224594116,
      "logps/rejected": -2.839496612548828,
      "loss": 1.4279,
      "nll_loss": 1.4103198051452637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14045670628547668,
      "rewards/margins": 0.1434929370880127,
      "rewards/rejected": -0.28394967317581177,
      "step": 1177
    },
    {
      "epoch": 1.855791962174941,
      "grad_norm": 0.23813945055007935,
      "learning_rate": 6.49841124502144e-08,
      "log_odds_chosen": 1.428239107131958,
      "log_odds_ratio": -0.230551615357399,
      "logits/chosen": -0.7899944186210632,
      "logits/rejected": -1.2597733736038208,
      "logps/chosen": -1.4341102838516235,
      "logps/rejected": -2.666048049926758,
      "loss": 1.461,
      "nll_loss": 1.437941312789917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14341102540493011,
      "rewards/margins": 0.12319377809762955,
      "rewards/rejected": -0.26660484075546265,
      "step": 1178
    },
    {
      "epoch": 1.8573680063041764,
      "grad_norm": 0.2471741884946823,
      "learning_rate": 6.360153309104998e-08,
      "log_odds_chosen": 1.7148985862731934,
      "log_odds_ratio": -0.18987813591957092,
      "logits/chosen": -0.9019498825073242,
      "logits/rejected": -1.3972867727279663,
      "logps/chosen": -1.4612141847610474,
      "logps/rejected": -2.9729816913604736,
      "loss": 1.4854,
      "nll_loss": 1.4664392471313477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14612144231796265,
      "rewards/margins": 0.15117672085762024,
      "rewards/rejected": -0.2972981631755829,
      "step": 1179
    },
    {
      "epoch": 1.8589440504334123,
      "grad_norm": 0.2292109876871109,
      "learning_rate": 6.223358229740916e-08,
      "log_odds_chosen": 1.5936087369918823,
      "log_odds_ratio": -0.19544288516044617,
      "logits/chosen": -0.7793794870376587,
      "logits/rejected": -1.2922043800354004,
      "logps/chosen": -1.4776686429977417,
      "logps/rejected": -2.871058464050293,
      "loss": 1.4829,
      "nll_loss": 1.4633288383483887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477668434381485,
      "rewards/margins": 0.13933901488780975,
      "rewards/rejected": -0.28710585832595825,
      "step": 1180
    },
    {
      "epoch": 1.8605200945626477,
      "grad_norm": 0.31120431423187256,
      "learning_rate": 6.088027040350674e-08,
      "log_odds_chosen": 1.5605891942977905,
      "log_odds_ratio": -0.20016053318977356,
      "logits/chosen": -0.7806318998336792,
      "logits/rejected": -1.352982997894287,
      "logps/chosen": -1.4601013660430908,
      "logps/rejected": -2.8189563751220703,
      "loss": 1.4818,
      "nll_loss": 1.461807370185852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.146010160446167,
      "rewards/margins": 0.13588550686836243,
      "rewards/rejected": -0.2818956673145294,
      "step": 1181
    },
    {
      "epoch": 1.8620961386918835,
      "grad_norm": 0.24096953868865967,
      "learning_rate": 5.9541607632968003e-08,
      "log_odds_chosen": 1.4629324674606323,
      "log_odds_ratio": -0.214307501912117,
      "logits/chosen": -0.6632521748542786,
      "logits/rejected": -1.3271327018737793,
      "logps/chosen": -1.461748719215393,
      "logps/rejected": -2.7239956855773926,
      "loss": 1.4744,
      "nll_loss": 1.4529916048049927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1461748629808426,
      "rewards/margins": 0.12622471153736115,
      "rewards/rejected": -0.27239957451820374,
      "step": 1182
    },
    {
      "epoch": 1.863672182821119,
      "grad_norm": 0.24063006043434143,
      "learning_rate": 5.821760409875054e-08,
      "log_odds_chosen": 1.4353123903274536,
      "log_odds_ratio": -0.22455193102359772,
      "logits/chosen": -0.7796550989151001,
      "logits/rejected": -1.310289978981018,
      "logps/chosen": -1.437607765197754,
      "logps/rejected": -2.6747031211853027,
      "loss": 1.4766,
      "nll_loss": 1.4541045427322388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437607705593109,
      "rewards/margins": 0.1237095519900322,
      "rewards/rejected": -0.2674703001976013,
      "step": 1183
    },
    {
      "epoch": 1.8652482269503547,
      "grad_norm": 0.2508472502231598,
      "learning_rate": 5.690826980306851e-08,
      "log_odds_chosen": 1.5333073139190674,
      "log_odds_ratio": -0.21129953861236572,
      "logits/chosen": -0.8489570021629333,
      "logits/rejected": -1.2010853290557861,
      "logps/chosen": -1.386409044265747,
      "logps/rejected": -2.7052369117736816,
      "loss": 1.4234,
      "nll_loss": 1.4022496938705444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13864091038703918,
      "rewards/margins": 0.13188278675079346,
      "rewards/rejected": -0.27052369713783264,
      "step": 1184
    },
    {
      "epoch": 1.8668242710795901,
      "grad_norm": 0.23192022740840912,
      "learning_rate": 5.56136146373174e-08,
      "log_odds_chosen": 1.6685271263122559,
      "log_odds_ratio": -0.18715275824069977,
      "logits/chosen": -0.8720384836196899,
      "logits/rejected": -1.4680557250976562,
      "logps/chosen": -1.4731988906860352,
      "logps/rejected": -2.9368128776550293,
      "loss": 1.4926,
      "nll_loss": 1.4738750457763672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14731988310813904,
      "rewards/margins": 0.1463613659143448,
      "rewards/rejected": -0.29368123412132263,
      "step": 1185
    },
    {
      "epoch": 1.868400315208826,
      "grad_norm": 0.2327415645122528,
      "learning_rate": 5.4333648381998276e-08,
      "log_odds_chosen": 1.4310559034347534,
      "log_odds_ratio": -0.21925795078277588,
      "logits/chosen": -0.7699723243713379,
      "logits/rejected": -1.3403470516204834,
      "logps/chosen": -1.4980679750442505,
      "logps/rejected": -2.7410902976989746,
      "loss": 1.519,
      "nll_loss": 1.4970988035202026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14980679750442505,
      "rewards/margins": 0.12430226057767868,
      "rewards/rejected": -0.2741090655326843,
      "step": 1186
    },
    {
      "epoch": 1.8699763593380614,
      "grad_norm": 0.23456309735774994,
      "learning_rate": 5.306838070664521e-08,
      "log_odds_chosen": 1.1313691139221191,
      "log_odds_ratio": -0.2864971160888672,
      "logits/chosen": -0.874768078327179,
      "logits/rejected": -1.2376160621643066,
      "logps/chosen": -1.4223510026931763,
      "logps/rejected": -2.3756332397460938,
      "loss": 1.4687,
      "nll_loss": 1.440053939819336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14223511517047882,
      "rewards/margins": 0.0953282043337822,
      "rewards/rejected": -0.23756329715251923,
      "step": 1187
    },
    {
      "epoch": 1.8715524034672972,
      "grad_norm": 0.22074750065803528,
      "learning_rate": 5.181782116975109e-08,
      "log_odds_chosen": 1.6352784633636475,
      "log_odds_ratio": -0.1859792172908783,
      "logits/chosen": -0.816644549369812,
      "logits/rejected": -1.482271671295166,
      "logps/chosen": -1.3955309391021729,
      "logps/rejected": -2.810620069503784,
      "loss": 1.4168,
      "nll_loss": 1.3982378244400024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13955309987068176,
      "rewards/margins": 0.14150892198085785,
      "rewards/rejected": -0.2810620367527008,
      "step": 1188
    },
    {
      "epoch": 1.8731284475965326,
      "grad_norm": 0.2120906561613083,
      "learning_rate": 5.058197921869567e-08,
      "log_odds_chosen": 1.4577536582946777,
      "log_odds_ratio": -0.21351027488708496,
      "logits/chosen": -0.6640850901603699,
      "logits/rejected": -1.2398422956466675,
      "logps/chosen": -1.342494249343872,
      "logps/rejected": -2.577146053314209,
      "loss": 1.3821,
      "nll_loss": 1.3607932329177856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13424943387508392,
      "rewards/margins": 0.12346518039703369,
      "rewards/rejected": -0.2577146291732788,
      "step": 1189
    },
    {
      "epoch": 1.8747044917257685,
      "grad_norm": 0.25168418884277344,
      "learning_rate": 4.9360864189674775e-08,
      "log_odds_chosen": 1.6113379001617432,
      "log_odds_ratio": -0.1919972449541092,
      "logits/chosen": -0.8032915592193604,
      "logits/rejected": -1.3409048318862915,
      "logps/chosen": -1.409225583076477,
      "logps/rejected": -2.8020753860473633,
      "loss": 1.4246,
      "nll_loss": 1.4053645133972168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14092256128787994,
      "rewards/margins": 0.13928498327732086,
      "rewards/rejected": -0.2802075743675232,
      "step": 1190
    },
    {
      "epoch": 1.8762805358550039,
      "grad_norm": 0.22970625758171082,
      "learning_rate": 4.8154485307629226e-08,
      "log_odds_chosen": 1.7915855646133423,
      "log_odds_ratio": -0.16638942062854767,
      "logits/chosen": -0.8406122922897339,
      "logits/rejected": -1.3625636100769043,
      "logps/chosen": -1.335875153541565,
      "logps/rejected": -2.879091501235962,
      "loss": 1.3812,
      "nll_loss": 1.364559531211853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1335875242948532,
      "rewards/margins": 0.15432165563106537,
      "rewards/rejected": -0.2879091799259186,
      "step": 1191
    },
    {
      "epoch": 1.8778565799842397,
      "grad_norm": 0.22309426963329315,
      "learning_rate": 4.6962851686175306e-08,
      "log_odds_chosen": 1.2613862752914429,
      "log_odds_ratio": -0.27728071808815,
      "logits/chosen": -0.6647701263427734,
      "logits/rejected": -1.0728554725646973,
      "logps/chosen": -1.3386201858520508,
      "logps/rejected": -2.404294490814209,
      "loss": 1.3841,
      "nll_loss": 1.3563350439071655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13386201858520508,
      "rewards/margins": 0.10656743496656418,
      "rewards/rejected": -0.24042946100234985,
      "step": 1192
    },
    {
      "epoch": 1.8794326241134751,
      "grad_norm": 0.24446307122707367,
      "learning_rate": 4.578597232753556e-08,
      "log_odds_chosen": 1.682569980621338,
      "log_odds_ratio": -0.1723230481147766,
      "logits/chosen": -0.7605375051498413,
      "logits/rejected": -1.3323601484298706,
      "logps/chosen": -1.46147620677948,
      "logps/rejected": -2.9330191612243652,
      "loss": 1.4699,
      "nll_loss": 1.4526996612548828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14614760875701904,
      "rewards/margins": 0.147154301404953,
      "rewards/rejected": -0.29330191016197205,
      "step": 1193
    },
    {
      "epoch": 1.881008668242711,
      "grad_norm": 0.22915317118167877,
      "learning_rate": 4.462385612247166e-08,
      "log_odds_chosen": 1.607132911682129,
      "log_odds_ratio": -0.1904926896095276,
      "logits/chosen": -0.775658905506134,
      "logits/rejected": -1.4108302593231201,
      "logps/chosen": -1.5192160606384277,
      "logps/rejected": -2.933771848678589,
      "loss": 1.5244,
      "nll_loss": 1.5053898096084595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1519216001033783,
      "rewards/margins": 0.14145559072494507,
      "rewards/rejected": -0.29337719082832336,
      "step": 1194
    },
    {
      "epoch": 1.8825847123719464,
      "grad_norm": 0.23997674882411957,
      "learning_rate": 4.34765118502165e-08,
      "log_odds_chosen": 1.5435338020324707,
      "log_odds_ratio": -0.20021316409111023,
      "logits/chosen": -0.746945858001709,
      "logits/rejected": -1.43413507938385,
      "logps/chosen": -1.3648643493652344,
      "logps/rejected": -2.685670852661133,
      "loss": 1.4049,
      "nll_loss": 1.384880781173706,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13648644089698792,
      "rewards/margins": 0.13208064436912537,
      "rewards/rejected": -0.2685670554637909,
      "step": 1195
    },
    {
      "epoch": 1.8841607565011822,
      "grad_norm": 0.2195524126291275,
      "learning_rate": 4.234394817840803e-08,
      "log_odds_chosen": 1.5772924423217773,
      "log_odds_ratio": -0.20687992870807648,
      "logits/chosen": -0.7699701189994812,
      "logits/rejected": -1.1730819940567017,
      "logps/chosen": -1.3805919885635376,
      "logps/rejected": -2.7385993003845215,
      "loss": 1.4252,
      "nll_loss": 1.4044829607009888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13805921375751495,
      "rewards/margins": 0.13580076396465302,
      "rewards/rejected": -0.27385997772216797,
      "step": 1196
    },
    {
      "epoch": 1.8857368006304176,
      "grad_norm": 0.24775627255439758,
      "learning_rate": 4.122617366302395e-08,
      "log_odds_chosen": 1.5994571447372437,
      "log_odds_ratio": -0.1897655576467514,
      "logits/chosen": -0.7943511009216309,
      "logits/rejected": -1.3700577020645142,
      "logps/chosen": -1.4631595611572266,
      "logps/rejected": -2.8581459522247314,
      "loss": 1.4839,
      "nll_loss": 1.4649481773376465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14631594717502594,
      "rewards/margins": 0.13949865102767944,
      "rewards/rejected": -0.2858146131038666,
      "step": 1197
    },
    {
      "epoch": 1.8873128447596532,
      "grad_norm": 0.21299691498279572,
      "learning_rate": 4.0123196748316436e-08,
      "log_odds_chosen": 1.3663235902786255,
      "log_odds_ratio": -0.23697176575660706,
      "logits/chosen": -0.7681499719619751,
      "logits/rejected": -1.3550536632537842,
      "logps/chosen": -1.3371795415878296,
      "logps/rejected": -2.488131046295166,
      "loss": 1.3731,
      "nll_loss": 1.3494389057159424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13371795415878296,
      "rewards/margins": 0.1150951236486435,
      "rewards/rejected": -0.24881309270858765,
      "step": 1198
    },
    {
      "epoch": 1.8888888888888888,
      "grad_norm": 0.23141250014305115,
      "learning_rate": 3.903502576674933e-08,
      "log_odds_chosen": 1.5047311782836914,
      "log_odds_ratio": -0.21784567832946777,
      "logits/chosen": -0.8047406673431396,
      "logits/rejected": -1.3700785636901855,
      "logps/chosen": -1.4658337831497192,
      "logps/rejected": -2.7762491703033447,
      "loss": 1.4903,
      "nll_loss": 1.4684890508651733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14658337831497192,
      "rewards/margins": 0.13104155659675598,
      "rewards/rejected": -0.2776249051094055,
      "step": 1199
    },
    {
      "epoch": 1.8904649330181245,
      "grad_norm": 0.2448730319738388,
      "learning_rate": 3.796166893893504e-08,
      "log_odds_chosen": 1.38612961769104,
      "log_odds_ratio": -0.22823438048362732,
      "logits/chosen": -0.6828239560127258,
      "logits/rejected": -1.3461682796478271,
      "logps/chosen": -1.5352051258087158,
      "logps/rejected": -2.7440927028656006,
      "loss": 1.5458,
      "nll_loss": 1.5229291915893555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15352049469947815,
      "rewards/margins": 0.12088877707719803,
      "rewards/rejected": -0.27440929412841797,
      "step": 1200
    },
    {
      "epoch": 1.89204097714736,
      "grad_norm": 0.23689720034599304,
      "learning_rate": 3.690313437357151e-08,
      "log_odds_chosen": 1.506507158279419,
      "log_odds_ratio": -0.20693618059158325,
      "logits/chosen": -0.7311481237411499,
      "logits/rejected": -1.2489502429962158,
      "logps/chosen": -1.356451392173767,
      "logps/rejected": -2.6414475440979004,
      "loss": 1.3927,
      "nll_loss": 1.3720234632492065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13564515113830566,
      "rewards/margins": 0.1284996122121811,
      "rewards/rejected": -0.26414474844932556,
      "step": 1201
    },
    {
      "epoch": 1.8936170212765957,
      "grad_norm": 0.24505552649497986,
      "learning_rate": 3.585943006738179e-08,
      "log_odds_chosen": 1.409112572669983,
      "log_odds_ratio": -0.22610175609588623,
      "logits/chosen": -0.7405913472175598,
      "logits/rejected": -1.1777433156967163,
      "logps/chosen": -1.4550225734710693,
      "logps/rejected": -2.6710236072540283,
      "loss": 1.4908,
      "nll_loss": 1.4681718349456787,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1455022543668747,
      "rewards/margins": 0.12160012125968933,
      "rewards/rejected": -0.26710236072540283,
      "step": 1202
    },
    {
      "epoch": 1.8951930654058313,
      "grad_norm": 0.25874462723731995,
      "learning_rate": 3.483056390505346e-08,
      "log_odds_chosen": 1.5968526601791382,
      "log_odds_ratio": -0.19228553771972656,
      "logits/chosen": -0.8400677442550659,
      "logits/rejected": -1.3975201845169067,
      "logps/chosen": -1.4146164655685425,
      "logps/rejected": -2.7961795330047607,
      "loss": 1.4297,
      "nll_loss": 1.4104512929916382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14146165549755096,
      "rewards/margins": 0.13815632462501526,
      "rewards/rejected": -0.27961796522140503,
      "step": 1203
    },
    {
      "epoch": 1.896769109535067,
      "grad_norm": 0.237021341919899,
      "learning_rate": 3.381654365917863e-08,
      "log_odds_chosen": 1.7031371593475342,
      "log_odds_ratio": -0.17307336628437042,
      "logits/chosen": -0.7872185111045837,
      "logits/rejected": -1.6255191564559937,
      "logps/chosen": -1.5250517129898071,
      "logps/rejected": -3.0295698642730713,
      "loss": 1.5344,
      "nll_loss": 1.5170533657073975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15250517427921295,
      "rewards/margins": 0.15045180916786194,
      "rewards/rejected": -0.3029569983482361,
      "step": 1204
    },
    {
      "epoch": 1.8983451536643026,
      "grad_norm": 0.2206202894449234,
      "learning_rate": 3.281737699019627e-08,
      "log_odds_chosen": 1.4491595029830933,
      "log_odds_ratio": -0.21479639410972595,
      "logits/chosen": -0.7172896862030029,
      "logits/rejected": -1.2247743606567383,
      "logps/chosen": -1.3457709550857544,
      "logps/rejected": -2.5746476650238037,
      "loss": 1.3967,
      "nll_loss": 1.3751863241195679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13457709550857544,
      "rewards/margins": 0.12288767844438553,
      "rewards/rejected": -0.25746479630470276,
      "step": 1205
    },
    {
      "epoch": 1.8999211977935382,
      "grad_norm": 0.23292219638824463,
      "learning_rate": 3.1833071446333295e-08,
      "log_odds_chosen": 1.3854937553405762,
      "log_odds_ratio": -0.23441529273986816,
      "logits/chosen": -0.7704402208328247,
      "logits/rejected": -1.1891682147979736,
      "logps/chosen": -1.4424054622650146,
      "logps/rejected": -2.6359026432037354,
      "loss": 1.4649,
      "nll_loss": 1.4414942264556885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14424052834510803,
      "rewards/margins": 0.11934974789619446,
      "rewards/rejected": -0.2635902762413025,
      "step": 1206
    },
    {
      "epoch": 1.9014972419227738,
      "grad_norm": 0.22912542521953583,
      "learning_rate": 3.0863634463548005e-08,
      "log_odds_chosen": 1.6800591945648193,
      "log_odds_ratio": -0.18198001384735107,
      "logits/chosen": -0.6901691555976868,
      "logits/rejected": -1.3353017568588257,
      "logps/chosen": -1.3718442916870117,
      "logps/rejected": -2.8201751708984375,
      "loss": 1.4087,
      "nll_loss": 1.3904800415039062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13718442618846893,
      "rewards/margins": 0.14483307301998138,
      "rewards/rejected": -0.2820174992084503,
      "step": 1207
    },
    {
      "epoch": 1.9030732860520094,
      "grad_norm": 0.22989633679389954,
      "learning_rate": 2.9909073365473435e-08,
      "log_odds_chosen": 1.8111798763275146,
      "log_odds_ratio": -0.1628579944372177,
      "logits/chosen": -0.7574429512023926,
      "logits/rejected": -1.3963072299957275,
      "logps/chosen": -1.4176511764526367,
      "logps/rejected": -3.003248691558838,
      "loss": 1.4389,
      "nll_loss": 1.4226422309875488,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14176513254642487,
      "rewards/margins": 0.15855976939201355,
      "rewards/rejected": -0.3003248870372772,
      "step": 1208
    },
    {
      "epoch": 1.904649330181245,
      "grad_norm": 0.2589717209339142,
      "learning_rate": 2.896939536336296e-08,
      "log_odds_chosen": 1.3924800157546997,
      "log_odds_ratio": -0.22703072428703308,
      "logits/chosen": -0.7431458234786987,
      "logits/rejected": -1.131448745727539,
      "logps/chosen": -1.4394726753234863,
      "logps/rejected": -2.632551431655884,
      "loss": 1.4656,
      "nll_loss": 1.4428737163543701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1439472734928131,
      "rewards/margins": 0.11930785328149796,
      "rewards/rejected": -0.26325511932373047,
      "step": 1209
    },
    {
      "epoch": 1.9062253743104807,
      "grad_norm": 0.23652757704257965,
      "learning_rate": 2.8044607556035216e-08,
      "log_odds_chosen": 1.7628989219665527,
      "log_odds_ratio": -0.1829545646905899,
      "logits/chosen": -0.8862708210945129,
      "logits/rejected": -1.4152332544326782,
      "logps/chosen": -1.4553879499435425,
      "logps/rejected": -3.0121703147888184,
      "loss": 1.4667,
      "nll_loss": 1.4483951330184937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1455388069152832,
      "rewards/margins": 0.15567824244499207,
      "rewards/rejected": -0.3012170195579529,
      "step": 1210
    },
    {
      "epoch": 1.9078014184397163,
      "grad_norm": 0.22456270456314087,
      "learning_rate": 2.7134716929820167e-08,
      "log_odds_chosen": 1.477555274963379,
      "log_odds_ratio": -0.2097799926996231,
      "logits/chosen": -0.6600494384765625,
      "logits/rejected": -1.3426413536071777,
      "logps/chosen": -1.4735187292099,
      "logps/rejected": -2.756434440612793,
      "loss": 1.4935,
      "nll_loss": 1.472505807876587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735189080238342,
      "rewards/margins": 0.12829159200191498,
      "rewards/rejected": -0.2756434679031372,
      "step": 1211
    },
    {
      "epoch": 1.909377462568952,
      "grad_norm": 0.23248450458049774,
      "learning_rate": 2.6239730358506905e-08,
      "log_odds_chosen": 1.41584050655365,
      "log_odds_ratio": -0.22305071353912354,
      "logits/chosen": -0.7314263582229614,
      "logits/rejected": -1.3808902502059937,
      "logps/chosen": -1.4230473041534424,
      "logps/rejected": -2.6381332874298096,
      "loss": 1.4544,
      "nll_loss": 1.4321231842041016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14230473339557648,
      "rewards/margins": 0.12150861322879791,
      "rewards/rejected": -0.2638133466243744,
      "step": 1212
    },
    {
      "epoch": 1.9109535066981875,
      "grad_norm": 0.2457541674375534,
      "learning_rate": 2.535965460329148e-08,
      "log_odds_chosen": 1.8812100887298584,
      "log_odds_ratio": -0.14745645225048065,
      "logits/chosen": -0.8360577821731567,
      "logits/rejected": -1.3872625827789307,
      "logps/chosen": -1.3484761714935303,
      "logps/rejected": -2.9829132556915283,
      "loss": 1.3737,
      "nll_loss": 1.3589469194412231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13484761118888855,
      "rewards/margins": 0.16344371438026428,
      "rewards/rejected": -0.29829132556915283,
      "step": 1213
    },
    {
      "epoch": 1.9125295508274232,
      "grad_norm": 0.23900792002677917,
      "learning_rate": 2.4494496312726043e-08,
      "log_odds_chosen": 1.7620255947113037,
      "log_odds_ratio": -0.16456454992294312,
      "logits/chosen": -0.6880412697792053,
      "logits/rejected": -1.5508886575698853,
      "logps/chosen": -1.350751280784607,
      "logps/rejected": -2.871666669845581,
      "loss": 1.3789,
      "nll_loss": 1.362492322921753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1350751370191574,
      "rewards/margins": 0.15209153294563293,
      "rewards/rejected": -0.28716668486595154,
      "step": 1214
    },
    {
      "epoch": 1.9141055949566588,
      "grad_norm": 0.22004970908164978,
      "learning_rate": 2.3644262022668005e-08,
      "log_odds_chosen": 1.6123836040496826,
      "log_odds_ratio": -0.1862034946680069,
      "logits/chosen": -0.7620582580566406,
      "logits/rejected": -1.4052793979644775,
      "logps/chosen": -1.3717292547225952,
      "logps/rejected": -2.756727933883667,
      "loss": 1.3979,
      "nll_loss": 1.37932288646698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13717292249202728,
      "rewards/margins": 0.1384998857975006,
      "rewards/rejected": -0.2756728231906891,
      "step": 1215
    },
    {
      "epoch": 1.9156816390858944,
      "grad_norm": 0.27064749598503113,
      "learning_rate": 2.280895815623185e-08,
      "log_odds_chosen": 1.4462248086929321,
      "log_odds_ratio": -0.22827844321727753,
      "logits/chosen": -0.7006045579910278,
      "logits/rejected": -1.2017916440963745,
      "logps/chosen": -1.4166349172592163,
      "logps/rejected": -2.661740303039551,
      "loss": 1.4605,
      "nll_loss": 1.4376773834228516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14166350662708282,
      "rewards/margins": 0.12451054155826569,
      "rewards/rejected": -0.2661740779876709,
      "step": 1216
    },
    {
      "epoch": 1.91725768321513,
      "grad_norm": 0.26126623153686523,
      "learning_rate": 2.1988591023738513e-08,
      "log_odds_chosen": 1.4409046173095703,
      "log_odds_ratio": -0.2235446274280548,
      "logits/chosen": -0.7363305687904358,
      "logits/rejected": -1.1807034015655518,
      "logps/chosen": -1.5095551013946533,
      "logps/rejected": -2.769727945327759,
      "loss": 1.5319,
      "nll_loss": 1.5095187425613403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15095551311969757,
      "rewards/margins": 0.12601730227470398,
      "rewards/rejected": -0.27697280049324036,
      "step": 1217
    },
    {
      "epoch": 1.9188337273443656,
      "grad_norm": 0.2397678792476654,
      "learning_rate": 2.1183166822670518e-08,
      "log_odds_chosen": 1.617363452911377,
      "log_odds_ratio": -0.18810036778450012,
      "logits/chosen": -0.7104983329772949,
      "logits/rejected": -1.4066905975341797,
      "logps/chosen": -1.4353305101394653,
      "logps/rejected": -2.8398780822753906,
      "loss": 1.462,
      "nll_loss": 1.4431637525558472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14353305101394653,
      "rewards/margins": 0.14045476913452148,
      "rewards/rejected": -0.2839878499507904,
      "step": 1218
    },
    {
      "epoch": 1.9204097714736013,
      "grad_norm": 0.2268221229314804,
      "learning_rate": 2.0392691637622696e-08,
      "log_odds_chosen": 1.5027459859848022,
      "log_odds_ratio": -0.20392416417598724,
      "logits/chosen": -0.688973069190979,
      "logits/rejected": -1.4112927913665771,
      "logps/chosen": -1.4371167421340942,
      "logps/rejected": -2.7299351692199707,
      "loss": 1.4742,
      "nll_loss": 1.4537967443466187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14371167123317719,
      "rewards/margins": 0.12928181886672974,
      "rewards/rejected": -0.2729935050010681,
      "step": 1219
    },
    {
      "epoch": 1.9219858156028369,
      "grad_norm": 0.23557861149311066,
      "learning_rate": 1.961717144025732e-08,
      "log_odds_chosen": 1.4746688604354858,
      "log_odds_ratio": -0.2105305790901184,
      "logits/chosen": -0.7015765309333801,
      "logits/rejected": -1.272491455078125,
      "logps/chosen": -1.3923325538635254,
      "logps/rejected": -2.650669574737549,
      "loss": 1.4404,
      "nll_loss": 1.4193477630615234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13923326134681702,
      "rewards/margins": 0.1258336752653122,
      "rewards/rejected": -0.265066921710968,
      "step": 1220
    },
    {
      "epoch": 1.9235618597320725,
      "grad_norm": 0.23435471951961517,
      "learning_rate": 1.8856612089259482e-08,
      "log_odds_chosen": 1.7672837972640991,
      "log_odds_ratio": -0.17940396070480347,
      "logits/chosen": -0.7988055944442749,
      "logits/rejected": -1.390700340270996,
      "logps/chosen": -1.4556410312652588,
      "logps/rejected": -3.0116050243377686,
      "loss": 1.4813,
      "nll_loss": 1.4633718729019165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14556410908699036,
      "rewards/margins": 0.15559639036655426,
      "rewards/rejected": -0.3011604845523834,
      "step": 1221
    },
    {
      "epoch": 1.9251379038613081,
      "grad_norm": 0.2487335056066513,
      "learning_rate": 1.8111019330291798e-08,
      "log_odds_chosen": 1.5161601305007935,
      "log_odds_ratio": -0.20423564314842224,
      "logits/chosen": -0.788711667060852,
      "logits/rejected": -1.2180840969085693,
      "logps/chosen": -1.4426312446594238,
      "logps/rejected": -2.756505012512207,
      "loss": 1.4712,
      "nll_loss": 1.4507604837417603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1442631185054779,
      "rewards/margins": 0.1313873827457428,
      "rewards/rejected": -0.2756505012512207,
      "step": 1222
    },
    {
      "epoch": 1.9267139479905437,
      "grad_norm": 0.2172604203224182,
      "learning_rate": 1.738039879595088e-08,
      "log_odds_chosen": 1.736552357673645,
      "log_odds_ratio": -0.1931544542312622,
      "logits/chosen": -0.7642362713813782,
      "logits/rejected": -1.3178571462631226,
      "logps/chosen": -1.3916239738464355,
      "logps/rejected": -2.908555269241333,
      "loss": 1.4125,
      "nll_loss": 1.393176794052124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13916240632534027,
      "rewards/margins": 0.15169313549995422,
      "rewards/rejected": -0.2908555269241333,
      "step": 1223
    },
    {
      "epoch": 1.9282899921197794,
      "grad_norm": 0.22269536554813385,
      "learning_rate": 1.6664756005726478e-08,
      "log_odds_chosen": 1.4885640144348145,
      "log_odds_ratio": -0.2402397245168686,
      "logits/chosen": -0.8421351909637451,
      "logits/rejected": -1.341312050819397,
      "logps/chosen": -1.455059289932251,
      "logps/rejected": -2.7438786029815674,
      "loss": 1.4715,
      "nll_loss": 1.4474490880966187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14550594985485077,
      "rewards/margins": 0.12888190150260925,
      "rewards/rejected": -0.27438783645629883,
      "step": 1224
    },
    {
      "epoch": 1.929866036249015,
      "grad_norm": 0.22902579605579376,
      "learning_rate": 1.596409636595797e-08,
      "log_odds_chosen": 1.6315475702285767,
      "log_odds_ratio": -0.1869073212146759,
      "logits/chosen": -0.7849953174591064,
      "logits/rejected": -1.3133690357208252,
      "logps/chosen": -1.3917748928070068,
      "logps/rejected": -2.7943906784057617,
      "loss": 1.3987,
      "nll_loss": 1.3800312280654907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13917748630046844,
      "rewards/margins": 0.14026156067848206,
      "rewards/rejected": -0.2794390320777893,
      "step": 1225
    },
    {
      "epoch": 1.9314420803782506,
      "grad_norm": 0.2516661286354065,
      "learning_rate": 1.5278425169794163e-08,
      "log_odds_chosen": 1.6236011981964111,
      "log_odds_ratio": -0.18852658569812775,
      "logits/chosen": -0.7662158608436584,
      "logits/rejected": -1.443966269493103,
      "logps/chosen": -1.4340133666992188,
      "logps/rejected": -2.8462185859680176,
      "loss": 1.459,
      "nll_loss": 1.4401147365570068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1434013396501541,
      "rewards/margins": 0.14122051000595093,
      "rewards/rejected": -0.28462183475494385,
      "step": 1226
    },
    {
      "epoch": 1.9330181245074862,
      "grad_norm": 0.21839259564876556,
      "learning_rate": 1.4607747597153109e-08,
      "log_odds_chosen": 1.5441559553146362,
      "log_odds_ratio": -0.21618308126926422,
      "logits/chosen": -0.7298059463500977,
      "logits/rejected": -1.2939796447753906,
      "logps/chosen": -1.3840653896331787,
      "logps/rejected": -2.7162299156188965,
      "loss": 1.4129,
      "nll_loss": 1.3912906646728516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13840653002262115,
      "rewards/margins": 0.1332164704799652,
      "rewards/rejected": -0.27162301540374756,
      "step": 1227
    },
    {
      "epoch": 1.9345941686367218,
      "grad_norm": 0.25088730454444885,
      "learning_rate": 1.3952068714684129e-08,
      "log_odds_chosen": 1.490285038948059,
      "log_odds_ratio": -0.22947844862937927,
      "logits/chosen": -0.818786084651947,
      "logits/rejected": -1.206778645515442,
      "logps/chosen": -1.4527573585510254,
      "logps/rejected": -2.748887300491333,
      "loss": 1.4704,
      "nll_loss": 1.4474586248397827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14527572691440582,
      "rewards/margins": 0.129612997174263,
      "rewards/rejected": -0.2748887240886688,
      "step": 1228
    },
    {
      "epoch": 1.9361702127659575,
      "grad_norm": 0.23627997934818268,
      "learning_rate": 1.3311393475727628e-08,
      "log_odds_chosen": 1.620850920677185,
      "log_odds_ratio": -0.19206514954566956,
      "logits/chosen": -0.7950037121772766,
      "logits/rejected": -1.478283166885376,
      "logps/chosen": -1.446081280708313,
      "logps/rejected": -2.8517377376556396,
      "loss": 1.4688,
      "nll_loss": 1.4496277570724487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14460813999176025,
      "rewards/margins": 0.1405656635761261,
      "rewards/rejected": -0.28517377376556396,
      "step": 1229
    },
    {
      "epoch": 1.937746256895193,
      "grad_norm": 0.2545762360095978,
      "learning_rate": 1.2685726720279344e-08,
      "log_odds_chosen": 1.5598962306976318,
      "log_odds_ratio": -0.19649472832679749,
      "logits/chosen": -0.7846990823745728,
      "logits/rejected": -1.3278943300247192,
      "logps/chosen": -1.5361363887786865,
      "logps/rejected": -2.91109299659729,
      "loss": 1.554,
      "nll_loss": 1.534339189529419,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1536136269569397,
      "rewards/margins": 0.13749565184116364,
      "rewards/rejected": -0.2911092936992645,
      "step": 1230
    },
    {
      "epoch": 1.9393223010244287,
      "grad_norm": 0.2528761923313141,
      "learning_rate": 1.2075073174952378e-08,
      "log_odds_chosen": 1.6762032508850098,
      "log_odds_ratio": -0.18338176608085632,
      "logits/chosen": -0.7630194425582886,
      "logits/rejected": -1.235816478729248,
      "logps/chosen": -1.3879940509796143,
      "logps/rejected": -2.8387134075164795,
      "loss": 1.4109,
      "nll_loss": 1.392590045928955,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13879939913749695,
      "rewards/margins": 0.14507193863391876,
      "rewards/rejected": -0.2838713526725769,
      "step": 1231
    },
    {
      "epoch": 1.9408983451536643,
      "grad_norm": 0.23820649087429047,
      "learning_rate": 1.1479437452942997e-08,
      "log_odds_chosen": 1.4386112689971924,
      "log_odds_ratio": -0.2164590209722519,
      "logits/chosen": -0.7249760031700134,
      "logits/rejected": -1.3802849054336548,
      "logps/chosen": -1.4090783596038818,
      "logps/rejected": -2.6418566703796387,
      "loss": 1.4235,
      "nll_loss": 1.4019025564193726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14090783894062042,
      "rewards/margins": 0.12327784299850464,
      "rewards/rejected": -0.26418569684028625,
      "step": 1232
    },
    {
      "epoch": 1.9424743892829,
      "grad_norm": 0.284380704164505,
      "learning_rate": 1.0898824053994227e-08,
      "log_odds_chosen": 1.6143016815185547,
      "log_odds_ratio": -0.19296769797801971,
      "logits/chosen": -0.7456787824630737,
      "logits/rejected": -1.4365206956863403,
      "logps/chosen": -1.4949467182159424,
      "logps/rejected": -2.9143850803375244,
      "loss": 1.4967,
      "nll_loss": 1.4773778915405273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14949466288089752,
      "rewards/margins": 0.14194384217262268,
      "rewards/rejected": -0.291438490152359,
      "step": 1233
    },
    {
      "epoch": 1.9440504334121356,
      "grad_norm": 0.2727202773094177,
      "learning_rate": 1.0333237364362978e-08,
      "log_odds_chosen": 1.5819599628448486,
      "log_odds_ratio": -0.21232542395591736,
      "logits/chosen": -0.710331380367279,
      "logits/rejected": -1.2239055633544922,
      "logps/chosen": -1.4189815521240234,
      "logps/rejected": -2.7932064533233643,
      "loss": 1.4433,
      "nll_loss": 1.422045350074768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14189817011356354,
      "rewards/margins": 0.13742247223854065,
      "rewards/rejected": -0.2793206572532654,
      "step": 1234
    },
    {
      "epoch": 1.9456264775413712,
      "grad_norm": 0.23125796020030975,
      "learning_rate": 9.782681656786973e-09,
      "log_odds_chosen": 1.5437463521957397,
      "log_odds_ratio": -0.1945345103740692,
      "logits/chosen": -0.7653860449790955,
      "logits/rejected": -1.3079421520233154,
      "logps/chosen": -1.3887964487075806,
      "logps/rejected": -2.711942195892334,
      "loss": 1.429,
      "nll_loss": 1.4095553159713745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13887964189052582,
      "rewards/margins": 0.13231457769870758,
      "rewards/rejected": -0.2711942195892334,
      "step": 1235
    },
    {
      "epoch": 1.9472025216706068,
      "grad_norm": 0.26429030299186707,
      "learning_rate": 9.247161090451206e-09,
      "log_odds_chosen": 1.4349366426467896,
      "log_odds_ratio": -0.21592332422733307,
      "logits/chosen": -0.7594494819641113,
      "logits/rejected": -1.2081475257873535,
      "logps/chosen": -1.5095630884170532,
      "logps/rejected": -2.7598342895507812,
      "loss": 1.5069,
      "nll_loss": 1.4852813482284546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15095631778240204,
      "rewards/margins": 0.1250271052122116,
      "rewards/rejected": -0.27598342299461365,
      "step": 1236
    },
    {
      "epoch": 1.9487785657998424,
      "grad_norm": 0.2365633100271225,
      "learning_rate": 8.726679710957752e-09,
      "log_odds_chosen": 1.4832595586776733,
      "log_odds_ratio": -0.21238204836845398,
      "logits/chosen": -0.7697382569313049,
      "logits/rejected": -1.307991862297058,
      "logps/chosen": -1.4403411149978638,
      "logps/rejected": -2.721630573272705,
      "loss": 1.4592,
      "nll_loss": 1.437933325767517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14403408765792847,
      "rewards/margins": 0.12812893092632294,
      "rewards/rejected": -0.272163063287735,
      "step": 1237
    },
    {
      "epoch": 1.950354609929078,
      "grad_norm": 0.23853683471679688,
      "learning_rate": 8.221241450295124e-09,
      "log_odds_chosen": 1.6107919216156006,
      "log_odds_ratio": -0.2015560269355774,
      "logits/chosen": -0.776233434677124,
      "logits/rejected": -1.3372327089309692,
      "logps/chosen": -1.4536361694335938,
      "logps/rejected": -2.8585546016693115,
      "loss": 1.4641,
      "nll_loss": 1.4439501762390137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14536362886428833,
      "rewards/margins": 0.14049184322357178,
      "rewards/rejected": -0.2858554720878601,
      "step": 1238
    },
    {
      "epoch": 1.9519306540583137,
      "grad_norm": 0.2296527475118637,
      "learning_rate": 7.730850126807853e-09,
      "log_odds_chosen": 1.8064652681350708,
      "log_odds_ratio": -0.15572592616081238,
      "logits/chosen": -0.7467914819717407,
      "logits/rejected": -1.3867499828338623,
      "logps/chosen": -1.3386505842208862,
      "logps/rejected": -2.8952395915985107,
      "loss": 1.3809,
      "nll_loss": 1.3653761148452759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13386505842208862,
      "rewards/margins": 0.15565890073776245,
      "rewards/rejected": -0.2895239591598511,
      "step": 1239
    },
    {
      "epoch": 1.9535066981875493,
      "grad_norm": 0.23420169949531555,
      "learning_rate": 7.255509445168062e-09,
      "log_odds_chosen": 1.4591403007507324,
      "log_odds_ratio": -0.23922351002693176,
      "logits/chosen": -0.6376044750213623,
      "logits/rejected": -1.2790347337722778,
      "logps/chosen": -1.3779648542404175,
      "logps/rejected": -2.632021427154541,
      "loss": 1.399,
      "nll_loss": 1.3750474452972412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13779649138450623,
      "rewards/margins": 0.12540563941001892,
      "rewards/rejected": -0.26320210099220276,
      "step": 1240
    },
    {
      "epoch": 1.955082742316785,
      "grad_norm": 0.21736040711402893,
      "learning_rate": 6.795222996347494e-09,
      "log_odds_chosen": 1.5018223524093628,
      "log_odds_ratio": -0.20875079929828644,
      "logits/chosen": -0.7147260308265686,
      "logits/rejected": -1.2035918235778809,
      "logps/chosen": -1.3343160152435303,
      "logps/rejected": -2.6042556762695312,
      "loss": 1.3702,
      "nll_loss": 1.3492860794067383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13343161344528198,
      "rewards/margins": 0.12699396908283234,
      "rewards/rejected": -0.2604255676269531,
      "step": 1241
    },
    {
      "epoch": 1.9566587864460205,
      "grad_norm": 0.295773446559906,
      "learning_rate": 6.349994257590862e-09,
      "log_odds_chosen": 1.5494824647903442,
      "log_odds_ratio": -0.2077752947807312,
      "logits/chosen": -0.7979918718338013,
      "logits/rejected": -1.3829718828201294,
      "logps/chosen": -1.5314788818359375,
      "logps/rejected": -2.899092674255371,
      "loss": 1.5454,
      "nll_loss": 1.5246374607086182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.153147891163826,
      "rewards/margins": 0.1367613971233368,
      "rewards/rejected": -0.289909303188324,
      "step": 1242
    },
    {
      "epoch": 1.9582348305752562,
      "grad_norm": 0.2336142659187317,
      "learning_rate": 5.9198265923881e-09,
      "log_odds_chosen": 1.71607506275177,
      "log_odds_ratio": -0.1803075075149536,
      "logits/chosen": -0.7853384613990784,
      "logits/rejected": -1.4882460832595825,
      "logps/chosen": -1.4375193119049072,
      "logps/rejected": -2.9429540634155273,
      "loss": 1.4549,
      "nll_loss": 1.4368858337402344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14375193417072296,
      "rewards/margins": 0.1505434811115265,
      "rewards/rejected": -0.29429540038108826,
      "step": 1243
    },
    {
      "epoch": 1.9598108747044918,
      "grad_norm": 0.23387569189071655,
      "learning_rate": 5.504723250450593e-09,
      "log_odds_chosen": 1.5739171504974365,
      "log_odds_ratio": -0.19710701704025269,
      "logits/chosen": -0.7720206379890442,
      "logits/rejected": -1.3808279037475586,
      "logps/chosen": -1.4390151500701904,
      "logps/rejected": -2.8034956455230713,
      "loss": 1.4629,
      "nll_loss": 1.4432302713394165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.143901526927948,
      "rewards/margins": 0.13644804060459137,
      "rewards/rejected": -0.2803495526313782,
      "step": 1244
    },
    {
      "epoch": 1.9613869188337274,
      "grad_norm": 0.2759726345539093,
      "learning_rate": 5.1046873676861005e-09,
      "log_odds_chosen": 1.4949547052383423,
      "log_odds_ratio": -0.20694729685783386,
      "logits/chosen": -0.7020338177680969,
      "logits/rejected": -1.4666807651519775,
      "logps/chosen": -1.464300274848938,
      "logps/rejected": -2.76123309135437,
      "loss": 1.4896,
      "nll_loss": 1.4688835144042969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14643001556396484,
      "rewards/margins": 0.12969331443309784,
      "rewards/rejected": -0.2761233150959015,
      "step": 1245
    },
    {
      "epoch": 1.9629629629629628,
      "grad_norm": 0.26726260781288147,
      "learning_rate": 4.719721966174317e-09,
      "log_odds_chosen": 1.6540601253509521,
      "log_odds_ratio": -0.18082070350646973,
      "logits/chosen": -0.801814615726471,
      "logits/rejected": -1.2539857625961304,
      "logps/chosen": -1.3995580673217773,
      "logps/rejected": -2.8313260078430176,
      "loss": 1.4224,
      "nll_loss": 1.404338002204895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1399558186531067,
      "rewards/margins": 0.14317677915096283,
      "rewards/rejected": -0.2831325829029083,
      "step": 1246
    },
    {
      "epoch": 1.9645390070921986,
      "grad_norm": 0.24469448626041412,
      "learning_rate": 4.349829954145123e-09,
      "log_odds_chosen": 1.3933185338974,
      "log_odds_ratio": -0.2400878369808197,
      "logits/chosen": -0.8666211366653442,
      "logits/rejected": -1.1489663124084473,
      "logps/chosen": -1.3813650608062744,
      "logps/rejected": -2.56894588470459,
      "loss": 1.4292,
      "nll_loss": 1.4051640033721924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13813653588294983,
      "rewards/margins": 0.11875806748867035,
      "rewards/rejected": -0.256894588470459,
      "step": 1247
    },
    {
      "epoch": 1.966115051221434,
      "grad_norm": 0.23117229342460632,
      "learning_rate": 3.995014125956153e-09,
      "log_odds_chosen": 1.4433033466339111,
      "log_odds_ratio": -0.22524705529212952,
      "logits/chosen": -0.8159943222999573,
      "logits/rejected": -1.3793911933898926,
      "logps/chosen": -1.400179147720337,
      "logps/rejected": -2.6365504264831543,
      "loss": 1.428,
      "nll_loss": 1.4054628610610962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14001791179180145,
      "rewards/margins": 0.12363710254430771,
      "rewards/rejected": -0.26365500688552856,
      "step": 1248
    },
    {
      "epoch": 1.9676910953506699,
      "grad_norm": 0.2194187194108963,
      "learning_rate": 3.6552771620712573e-09,
      "log_odds_chosen": 1.7505093812942505,
      "log_odds_ratio": -0.16562360525131226,
      "logits/chosen": -0.818230926990509,
      "logits/rejected": -1.5206609964370728,
      "logps/chosen": -1.392747402191162,
      "logps/rejected": -2.9059832096099854,
      "loss": 1.4017,
      "nll_loss": 1.3851619958877563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1392747461795807,
      "rewards/margins": 0.1513236165046692,
      "rewards/rejected": -0.2905983626842499,
      "step": 1249
    },
    {
      "epoch": 1.9692671394799053,
      "grad_norm": 0.24016323685646057,
      "learning_rate": 3.3306216290409637e-09,
      "log_odds_chosen": 1.4799413681030273,
      "log_odds_ratio": -0.20982997119426727,
      "logits/chosen": -0.7829453349113464,
      "logits/rejected": -1.3166208267211914,
      "logps/chosen": -1.2968112230300903,
      "logps/rejected": -2.5387635231018066,
      "loss": 1.3468,
      "nll_loss": 1.3257719278335571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12968112528324127,
      "rewards/margins": 0.12419521063566208,
      "rewards/rejected": -0.25387632846832275,
      "step": 1250
    },
    {
      "epoch": 1.9708431836091411,
      "grad_norm": 0.23585279285907745,
      "learning_rate": 3.021049979482715e-09,
      "log_odds_chosen": 1.368883728981018,
      "log_odds_ratio": -0.23322059214115143,
      "logits/chosen": -0.6684975624084473,
      "logits/rejected": -1.2419157028198242,
      "logps/chosen": -1.3934158086776733,
      "logps/rejected": -2.561020851135254,
      "loss": 1.4269,
      "nll_loss": 1.4035816192626953,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1393415927886963,
      "rewards/margins": 0.11676051467657089,
      "rewards/rejected": -0.2561020851135254,
      "step": 1251
    },
    {
      "epoch": 1.9724192277383765,
      "grad_norm": 0.2372048795223236,
      "learning_rate": 2.72656455206266e-09,
      "log_odds_chosen": 1.3999991416931152,
      "log_odds_ratio": -0.2329872101545334,
      "logits/chosen": -0.7108824253082275,
      "logits/rejected": -1.2188117504119873,
      "logps/chosen": -1.4651814699172974,
      "logps/rejected": -2.6770784854888916,
      "loss": 1.5,
      "nll_loss": 1.4766656160354614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14651814103126526,
      "rewards/margins": 0.12118970602750778,
      "rewards/rejected": -0.26770785450935364,
      "step": 1252
    },
    {
      "epoch": 1.9739952718676124,
      "grad_norm": 0.22314748167991638,
      "learning_rate": 2.447167571477449e-09,
      "log_odds_chosen": 1.475752830505371,
      "log_odds_ratio": -0.20816180109977722,
      "logits/chosen": -0.6688903570175171,
      "logits/rejected": -1.2248951196670532,
      "logps/chosen": -1.4080373048782349,
      "logps/rejected": -2.6693272590637207,
      "loss": 1.4396,
      "nll_loss": 1.418833613395691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.140803724527359,
      "rewards/margins": 0.12612901628017426,
      "rewards/rejected": -0.26693272590637207,
      "step": 1253
    },
    {
      "epoch": 1.9755713159968478,
      "grad_norm": 0.24284829199314117,
      "learning_rate": 2.182861148437798e-09,
      "log_odds_chosen": 1.5073658227920532,
      "log_odds_ratio": -0.22068294882774353,
      "logits/chosen": -0.8059291243553162,
      "logits/rejected": -1.3284012079238892,
      "logps/chosen": -1.5053529739379883,
      "logps/rejected": -2.8241381645202637,
      "loss": 1.5142,
      "nll_loss": 1.4921774864196777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15053530037403107,
      "rewards/margins": 0.13187852501869202,
      "rewards/rejected": -0.2824138104915619,
      "step": 1254
    },
    {
      "epoch": 1.9771473601260836,
      "grad_norm": 0.2299458384513855,
      "learning_rate": 1.933647279652506e-09,
      "log_odds_chosen": 1.740532398223877,
      "log_odds_ratio": -0.20782388746738434,
      "logits/chosen": -0.7796560525894165,
      "logits/rejected": -1.301232099533081,
      "logps/chosen": -1.3972852230072021,
      "logps/rejected": -2.9226598739624023,
      "loss": 1.4454,
      "nll_loss": 1.424637794494629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13972851634025574,
      "rewards/margins": 0.15253746509552002,
      "rewards/rejected": -0.29226598143577576,
      "step": 1255
    },
    {
      "epoch": 1.978723404255319,
      "grad_norm": 0.22599293291568756,
      "learning_rate": 1.6995278478133534e-09,
      "log_odds_chosen": 1.3542404174804688,
      "log_odds_ratio": -0.23507030308246613,
      "logits/chosen": -0.6949554681777954,
      "logits/rejected": -1.2280828952789307,
      "logps/chosen": -1.4115251302719116,
      "logps/rejected": -2.5642495155334473,
      "loss": 1.4667,
      "nll_loss": 1.4432133436203003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411525309085846,
      "rewards/margins": 0.11527244746685028,
      "rewards/rejected": -0.2564249634742737,
      "step": 1256
    },
    {
      "epoch": 1.9802994483845549,
      "grad_norm": 0.2256888747215271,
      "learning_rate": 1.4805046215806693e-09,
      "log_odds_chosen": 1.5666311979293823,
      "log_odds_ratio": -0.19636406004428864,
      "logits/chosen": -0.7716910243034363,
      "logits/rejected": -1.2965235710144043,
      "logps/chosen": -1.3816429376602173,
      "logps/rejected": -2.7271337509155273,
      "loss": 1.4109,
      "nll_loss": 1.3912789821624756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13816428184509277,
      "rewards/margins": 0.1345490962266922,
      "rewards/rejected": -0.27271339297294617,
      "step": 1257
    },
    {
      "epoch": 1.9818754925137902,
      "grad_norm": 0.21410150825977325,
      "learning_rate": 1.2765792555704536e-09,
      "log_odds_chosen": 1.463745355606079,
      "log_odds_ratio": -0.21426446735858917,
      "logits/chosen": -0.7907874584197998,
      "logits/rejected": -1.3679403066635132,
      "logps/chosen": -1.4418962001800537,
      "logps/rejected": -2.701552629470825,
      "loss": 1.4597,
      "nll_loss": 1.4382833242416382,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14418962597846985,
      "rewards/margins": 0.1259656399488449,
      "rewards/rejected": -0.27015528082847595,
      "step": 1258
    },
    {
      "epoch": 1.983451536643026,
      "grad_norm": 0.22319120168685913,
      "learning_rate": 1.0877532903414977e-09,
      "log_odds_chosen": 1.7269924879074097,
      "log_odds_ratio": -0.17321103811264038,
      "logits/chosen": -0.7471919059753418,
      "logits/rejected": -1.2707041501998901,
      "logps/chosen": -1.3436994552612305,
      "logps/rejected": -2.8331375122070312,
      "loss": 1.3804,
      "nll_loss": 1.3630876541137695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13436993956565857,
      "rewards/margins": 0.14894381165504456,
      "rewards/rejected": -0.2833137512207031,
      "step": 1259
    },
    {
      "epoch": 1.9850275807722615,
      "grad_norm": 0.2631131708621979,
      "learning_rate": 9.140281523836168e-10,
      "log_odds_chosen": 1.5748944282531738,
      "log_odds_ratio": -0.19236807525157928,
      "logits/chosen": -0.8308014273643494,
      "logits/rejected": -1.2155208587646484,
      "logps/chosen": -1.4385806322097778,
      "logps/rejected": -2.8007748126983643,
      "loss": 1.4677,
      "nll_loss": 1.4484275579452515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14385807514190674,
      "rewards/margins": 0.13621942698955536,
      "rewards/rejected": -0.2800774872303009,
      "step": 1260
    },
    {
      "epoch": 1.9866036249014973,
      "grad_norm": 0.2249685376882553,
      "learning_rate": 7.554051541074357e-10,
      "log_odds_chosen": 1.5375914573669434,
      "log_odds_ratio": -0.20642447471618652,
      "logits/chosen": -0.759140133857727,
      "logits/rejected": -1.4114692211151123,
      "logps/chosen": -1.3763504028320312,
      "logps/rejected": -2.6906306743621826,
      "loss": 1.4111,
      "nll_loss": 1.3904755115509033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1376350373029709,
      "rewards/margins": 0.13142803311347961,
      "rewards/rejected": -0.2690630853176117,
      "step": 1261
    },
    {
      "epoch": 1.9881796690307327,
      "grad_norm": 0.258484810590744,
      "learning_rate": 6.118854938337304e-10,
      "log_odds_chosen": 1.4732701778411865,
      "log_odds_ratio": -0.21747945249080658,
      "logits/chosen": -0.8093488812446594,
      "logits/rejected": -1.3245649337768555,
      "logps/chosen": -1.408904790878296,
      "logps/rejected": -2.677839994430542,
      "loss": 1.442,
      "nll_loss": 1.4202775955200195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1408904790878296,
      "rewards/margins": 0.12689350545406342,
      "rewards/rejected": -0.2677839994430542,
      "step": 1262
    },
    {
      "epoch": 1.9897557131599686,
      "grad_norm": 0.22418129444122314,
      "learning_rate": 4.834702557852121e-10,
      "log_odds_chosen": 1.5523313283920288,
      "log_odds_ratio": -0.19629479944705963,
      "logits/chosen": -0.79649418592453,
      "logits/rejected": -1.4510045051574707,
      "logps/chosen": -1.4954601526260376,
      "logps/rejected": -2.8531503677368164,
      "loss": 1.5092,
      "nll_loss": 1.4895398616790771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14954602718353271,
      "rewards/margins": 0.1357690393924713,
      "rewards/rejected": -0.28531503677368164,
      "step": 1263
    },
    {
      "epoch": 1.991331757289204,
      "grad_norm": 0.23838064074516296,
      "learning_rate": 3.7016041007742473e-10,
      "log_odds_chosen": 1.7473846673965454,
      "log_odds_ratio": -0.16780352592468262,
      "logits/chosen": -0.7193484902381897,
      "logits/rejected": -1.4361478090286255,
      "logps/chosen": -1.4310880899429321,
      "logps/rejected": -2.956970453262329,
      "loss": 1.4568,
      "nll_loss": 1.440056324005127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1431088149547577,
      "rewards/margins": 0.15258823335170746,
      "rewards/rejected": -0.29569703340530396,
      "step": 1264
    },
    {
      "epoch": 1.9929078014184398,
      "grad_norm": 0.23790115118026733,
      "learning_rate": 2.71956812712304e-10,
      "log_odds_chosen": 1.7546888589859009,
      "log_odds_ratio": -0.16462941467761993,
      "logits/chosen": -0.7946275472640991,
      "logits/rejected": -1.5976415872573853,
      "logps/chosen": -1.4428914785385132,
      "logps/rejected": -2.97890043258667,
      "loss": 1.4754,
      "nll_loss": 1.458935022354126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14428915083408356,
      "rewards/margins": 0.15360090136528015,
      "rewards/rejected": -0.2978900372982025,
      "step": 1265
    },
    {
      "epoch": 1.9944838455476752,
      "grad_norm": 0.23842757940292358,
      "learning_rate": 1.8886020557107308e-10,
      "log_odds_chosen": 1.5966984033584595,
      "log_odds_ratio": -0.19028565287590027,
      "logits/chosen": -0.7216396927833557,
      "logits/rejected": -1.4179062843322754,
      "logps/chosen": -1.4392211437225342,
      "logps/rejected": -2.8244948387145996,
      "loss": 1.4689,
      "nll_loss": 1.449899673461914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1439221054315567,
      "rewards/margins": 0.13852739334106445,
      "rewards/rejected": -0.28244948387145996,
      "step": 1266
    },
    {
      "epoch": 1.996059889676911,
      "grad_norm": 0.2485765665769577,
      "learning_rate": 1.208712164091352e-10,
      "log_odds_chosen": 1.5719382762908936,
      "log_odds_ratio": -0.1944577991962433,
      "logits/chosen": -0.8098716139793396,
      "logits/rejected": -1.3030309677124023,
      "logps/chosen": -1.4441523551940918,
      "logps/rejected": -2.8061628341674805,
      "loss": 1.4646,
      "nll_loss": 1.4451676607131958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1444152295589447,
      "rewards/margins": 0.13620102405548096,
      "rewards/rejected": -0.28061628341674805,
      "step": 1267
    },
    {
      "epoch": 1.9976359338061465,
      "grad_norm": 0.225704163312912,
      "learning_rate": 6.799035885030057e-11,
      "log_odds_chosen": 1.7966450452804565,
      "log_odds_ratio": -0.16431106626987457,
      "logits/chosen": -0.7806457877159119,
      "logits/rejected": -1.3340007066726685,
      "logps/chosen": -1.447324514389038,
      "logps/rejected": -3.0240097045898438,
      "loss": 1.4628,
      "nll_loss": 1.4463653564453125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14473244547843933,
      "rewards/margins": 0.15766854584217072,
      "rewards/rejected": -0.30240100622177124,
      "step": 1268
    },
    {
      "epoch": 1.9992119779353823,
      "grad_norm": 0.2357356995344162,
      "learning_rate": 3.0218032384565774e-11,
      "log_odds_chosen": 1.6102283000946045,
      "log_odds_ratio": -0.18989889323711395,
      "logits/chosen": -0.6828922033309937,
      "logits/rejected": -1.2651770114898682,
      "logps/chosen": -1.337025761604309,
      "logps/rejected": -2.7072224617004395,
      "loss": 1.3853,
      "nll_loss": 1.3663090467453003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1337025761604309,
      "rewards/margins": 0.13701967895030975,
      "rewards/rejected": -0.27072227001190186,
      "step": 1269
    },
    {
      "epoch": 2.0,
      "grad_norm": 0.3162081241607666,
      "learning_rate": 7.554522363895089e-12,
      "log_odds_chosen": 1.6222355365753174,
      "log_odds_ratio": -0.199419766664505,
      "logits/chosen": -0.7464350461959839,
      "logits/rejected": -1.4885425567626953,
      "logps/chosen": -1.3772352933883667,
      "logps/rejected": -2.7769057750701904,
      "loss": 1.4046,
      "nll_loss": 1.3846197128295898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13772353529930115,
      "rewards/margins": 0.13996705412864685,
      "rewards/rejected": -0.277690589427948,
      "step": 1270
    }
  ],
  "logging_steps": 1,
  "max_steps": 1270,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}