{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.7880220646178093,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015760441292356187,
      "grad_norm": 0.7906547784805298,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.391758531332016,
      "log_odds_ratio": -0.534308910369873,
      "logits/chosen": -0.8063042759895325,
      "logits/rejected": 0.05536898970603943,
      "logps/chosen": -2.0363523960113525,
      "logps/rejected": -2.3849008083343506,
      "loss": 2.2935,
      "nll_loss": 2.240062952041626,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20363523066043854,
      "rewards/margins": 0.03485485166311264,
      "rewards/rejected": -0.23849007487297058,
      "step": 1
    },
    {
      "epoch": 0.0031520882584712374,
      "grad_norm": 0.7304520606994629,
      "learning_rate": 3.1496062992125985e-08,
      "log_odds_chosen": 0.3922337293624878,
      "log_odds_ratio": -0.5238180756568909,
      "logits/chosen": -0.7810883522033691,
      "logits/rejected": 0.1618170589208603,
      "logps/chosen": -1.9294850826263428,
      "logps/rejected": -2.2716217041015625,
      "loss": 2.1792,
      "nll_loss": 2.1268293857574463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19294852018356323,
      "rewards/margins": 0.03421364724636078,
      "rewards/rejected": -0.227162167429924,
      "step": 2
    },
    {
      "epoch": 0.004728132387706856,
      "grad_norm": 0.8640011548995972,
      "learning_rate": 6.299212598425197e-08,
      "log_odds_chosen": 0.5660353899002075,
      "log_odds_ratio": -0.4610859751701355,
      "logits/chosen": -0.965283989906311,
      "logits/rejected": -0.0006121229380369186,
      "logps/chosen": -2.0610620975494385,
      "logps/rejected": -2.57210373878479,
      "loss": 2.3037,
      "nll_loss": 2.25754714012146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20610620081424713,
      "rewards/margins": 0.051104187965393066,
      "rewards/rejected": -0.2572104036808014,
      "step": 3
    },
    {
      "epoch": 0.006304176516942475,
      "grad_norm": 0.7450115084648132,
      "learning_rate": 9.448818897637795e-08,
      "log_odds_chosen": 0.5706238746643066,
      "log_odds_ratio": -0.464206337928772,
      "logits/chosen": -0.7551829218864441,
      "logits/rejected": 0.1403137892484665,
      "logps/chosen": -2.06683087348938,
      "logps/rejected": -2.584524631500244,
      "loss": 2.3069,
      "nll_loss": 2.260457754135132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20668308436870575,
      "rewards/margins": 0.05176939070224762,
      "rewards/rejected": -0.25845247507095337,
      "step": 4
    },
    {
      "epoch": 0.007880220646178092,
      "grad_norm": 0.8498793244361877,
      "learning_rate": 1.2598425196850394e-07,
      "log_odds_chosen": 0.7315797805786133,
      "log_odds_ratio": -0.41388875246047974,
      "logits/chosen": -0.8775473833084106,
      "logits/rejected": -0.0007386752404272556,
      "logps/chosen": -1.9901928901672363,
      "logps/rejected": -2.644153594970703,
      "loss": 2.2234,
      "nll_loss": 2.182061195373535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19901929795742035,
      "rewards/margins": 0.06539607048034668,
      "rewards/rejected": -0.26441535353660583,
      "step": 5
    },
    {
      "epoch": 0.009456264775413711,
      "grad_norm": 0.7722200751304626,
      "learning_rate": 1.5748031496062992e-07,
      "log_odds_chosen": 0.4112725853919983,
      "log_odds_ratio": -0.5124160647392273,
      "logits/chosen": -0.7046896815299988,
      "logits/rejected": 0.17564038932323456,
      "logps/chosen": -1.9167629480361938,
      "logps/rejected": -2.2782955169677734,
      "loss": 2.1976,
      "nll_loss": 2.1463887691497803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1916763037443161,
      "rewards/margins": 0.036153268069028854,
      "rewards/rejected": -0.22782957553863525,
      "step": 6
    },
    {
      "epoch": 0.01103230890464933,
      "grad_norm": 0.7802104949951172,
      "learning_rate": 1.889763779527559e-07,
      "log_odds_chosen": 0.5916500091552734,
      "log_odds_ratio": -0.45217183232307434,
      "logits/chosen": -0.7392858266830444,
      "logits/rejected": 0.03766726329922676,
      "logps/chosen": -1.985681176185608,
      "logps/rejected": -2.5173680782318115,
      "loss": 2.2284,
      "nll_loss": 2.1831605434417725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19856813549995422,
      "rewards/margins": 0.05316868796944618,
      "rewards/rejected": -0.2517368197441101,
      "step": 7
    },
    {
      "epoch": 0.01260835303388495,
      "grad_norm": 0.9769444465637207,
      "learning_rate": 2.2047244094488187e-07,
      "log_odds_chosen": 0.8452985882759094,
      "log_odds_ratio": -0.36809873580932617,
      "logits/chosen": -0.8406262397766113,
      "logits/rejected": -0.015037253499031067,
      "logps/chosen": -2.1048223972320557,
      "logps/rejected": -2.8729093074798584,
      "loss": 2.347,
      "nll_loss": 2.3101491928100586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21048225462436676,
      "rewards/margins": 0.07680864632129669,
      "rewards/rejected": -0.28729093074798584,
      "step": 8
    },
    {
      "epoch": 0.014184397163120567,
      "grad_norm": 0.7307494878768921,
      "learning_rate": 2.519685039370079e-07,
      "log_odds_chosen": 0.4397449791431427,
      "log_odds_ratio": -0.5038886070251465,
      "logits/chosen": -0.7611249089241028,
      "logits/rejected": 0.1530451774597168,
      "logps/chosen": -1.9963405132293701,
      "logps/rejected": -2.3867950439453125,
      "loss": 2.2222,
      "nll_loss": 2.1718318462371826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19963406026363373,
      "rewards/margins": 0.03904542699456215,
      "rewards/rejected": -0.23867949843406677,
      "step": 9
    },
    {
      "epoch": 0.015760441292356184,
      "grad_norm": 0.746634304523468,
      "learning_rate": 2.8346456692913386e-07,
      "log_odds_chosen": 0.42592883110046387,
      "log_odds_ratio": -0.5083432793617249,
      "logits/chosen": -0.794756293296814,
      "logits/rejected": -0.07558537274599075,
      "logps/chosen": -2.0108375549316406,
      "logps/rejected": -2.390498161315918,
      "loss": 2.2518,
      "nll_loss": 2.2009661197662354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20108374953269958,
      "rewards/margins": 0.03796607255935669,
      "rewards/rejected": -0.23904980719089508,
      "step": 10
    },
    {
      "epoch": 0.017336485421591805,
      "grad_norm": 0.8190391063690186,
      "learning_rate": 3.1496062992125984e-07,
      "log_odds_chosen": 0.4864140450954437,
      "log_odds_ratio": -0.48338061571121216,
      "logits/chosen": -0.7586538791656494,
      "logits/rejected": 0.0021292148157954216,
      "logps/chosen": -2.1001176834106445,
      "logps/rejected": -2.536212921142578,
      "loss": 2.3376,
      "nll_loss": 2.2892978191375732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2100117951631546,
      "rewards/margins": 0.04360952228307724,
      "rewards/rejected": -0.25362130999565125,
      "step": 11
    },
    {
      "epoch": 0.018912529550827423,
      "grad_norm": 0.9358005523681641,
      "learning_rate": 3.464566929133858e-07,
      "log_odds_chosen": 0.5531883835792542,
      "log_odds_ratio": -0.45853984355926514,
      "logits/chosen": -0.8949778079986572,
      "logits/rejected": 0.16316448152065277,
      "logps/chosen": -2.152214288711548,
      "logps/rejected": -2.653200387954712,
      "loss": 2.3963,
      "nll_loss": 2.350400924682617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21522141993045807,
      "rewards/margins": 0.050098612904548645,
      "rewards/rejected": -0.2653200626373291,
      "step": 12
    },
    {
      "epoch": 0.02048857368006304,
      "grad_norm": 0.9224507212638855,
      "learning_rate": 3.779527559055118e-07,
      "log_odds_chosen": 0.5066958069801331,
      "log_odds_ratio": -0.4827364981174469,
      "logits/chosen": -0.9888880252838135,
      "logits/rejected": 0.2006295621395111,
      "logps/chosen": -2.050516128540039,
      "logps/rejected": -2.5026915073394775,
      "loss": 2.2934,
      "nll_loss": 2.2450859546661377,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20505160093307495,
      "rewards/margins": 0.04521755874156952,
      "rewards/rejected": -0.25026917457580566,
      "step": 13
    },
    {
      "epoch": 0.02206461780929866,
      "grad_norm": 0.7721648216247559,
      "learning_rate": 4.0944881889763777e-07,
      "log_odds_chosen": 0.3940409719944,
      "log_odds_ratio": -0.5303590297698975,
      "logits/chosen": -0.7946900725364685,
      "logits/rejected": -0.17097121477127075,
      "logps/chosen": -2.034684181213379,
      "logps/rejected": -2.387862205505371,
      "loss": 2.275,
      "nll_loss": 2.2220115661621094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2034684270620346,
      "rewards/margins": 0.035317789763212204,
      "rewards/rejected": -0.23878619074821472,
      "step": 14
    },
    {
      "epoch": 0.02364066193853428,
      "grad_norm": 0.8610419631004333,
      "learning_rate": 4.4094488188976375e-07,
      "log_odds_chosen": 0.4232204556465149,
      "log_odds_ratio": -0.5071219205856323,
      "logits/chosen": -0.8932058811187744,
      "logits/rejected": 0.05950244888663292,
      "logps/chosen": -2.0276124477386475,
      "logps/rejected": -2.4058046340942383,
      "loss": 2.2822,
      "nll_loss": 2.23144268989563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20276124775409698,
      "rewards/margins": 0.037819236516952515,
      "rewards/rejected": -0.2405804693698883,
      "step": 15
    },
    {
      "epoch": 0.0252167060677699,
      "grad_norm": 0.8575419783592224,
      "learning_rate": 4.7244094488188973e-07,
      "log_odds_chosen": 0.43113377690315247,
      "log_odds_ratio": -0.5063679814338684,
      "logits/chosen": -0.8862703442573547,
      "logits/rejected": 0.034714244306087494,
      "logps/chosen": -2.03637433052063,
      "logps/rejected": -2.4213576316833496,
      "loss": 2.2825,
      "nll_loss": 2.231827974319458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20363745093345642,
      "rewards/margins": 0.03849831968545914,
      "rewards/rejected": -0.24213576316833496,
      "step": 16
    },
    {
      "epoch": 0.026792750197005517,
      "grad_norm": 0.7937728762626648,
      "learning_rate": 5.039370078740158e-07,
      "log_odds_chosen": 0.642987072467804,
      "log_odds_ratio": -0.4362831711769104,
      "logits/chosen": -0.7214955687522888,
      "logits/rejected": -0.03120867908000946,
      "logps/chosen": -1.9728035926818848,
      "logps/rejected": -2.5497608184814453,
      "loss": 2.2244,
      "nll_loss": 2.1807427406311035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19728036224842072,
      "rewards/margins": 0.057695720344781876,
      "rewards/rejected": -0.2549760937690735,
      "step": 17
    },
    {
      "epoch": 0.028368794326241134,
      "grad_norm": 0.8444339036941528,
      "learning_rate": 5.354330708661418e-07,
      "log_odds_chosen": 0.3567027449607849,
      "log_odds_ratio": -0.5442062020301819,
      "logits/chosen": -0.7939973473548889,
      "logits/rejected": 0.3661736845970154,
      "logps/chosen": -2.0135817527770996,
      "logps/rejected": -2.328972578048706,
      "loss": 2.2728,
      "nll_loss": 2.2184267044067383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20135818421840668,
      "rewards/margins": 0.03153908997774124,
      "rewards/rejected": -0.23289725184440613,
      "step": 18
    },
    {
      "epoch": 0.029944838455476755,
      "grad_norm": 0.7747983336448669,
      "learning_rate": 5.669291338582677e-07,
      "log_odds_chosen": 0.6433313488960266,
      "log_odds_ratio": -0.4306049048900604,
      "logits/chosen": -0.9112986326217651,
      "logits/rejected": -0.013520637527108192,
      "logps/chosen": -1.900106430053711,
      "logps/rejected": -2.468703269958496,
      "loss": 2.1641,
      "nll_loss": 2.1210854053497314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1900106519460678,
      "rewards/margins": 0.056859664618968964,
      "rewards/rejected": -0.24687032401561737,
      "step": 19
    },
    {
      "epoch": 0.03152088258471237,
      "grad_norm": 0.8091546297073364,
      "learning_rate": 5.984251968503937e-07,
      "log_odds_chosen": 0.5987537503242493,
      "log_odds_ratio": -0.4577232897281647,
      "logits/chosen": -0.7413605451583862,
      "logits/rejected": 0.027445267885923386,
      "logps/chosen": -2.060060977935791,
      "logps/rejected": -2.60052490234375,
      "loss": 2.316,
      "nll_loss": 2.2701833248138428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20600609481334686,
      "rewards/margins": 0.0540463849902153,
      "rewards/rejected": -0.26005247235298157,
      "step": 20
    },
    {
      "epoch": 0.03309692671394799,
      "grad_norm": 0.8816735744476318,
      "learning_rate": 6.299212598425197e-07,
      "log_odds_chosen": 0.3807871341705322,
      "log_odds_ratio": -0.5288621187210083,
      "logits/chosen": -0.8584674000740051,
      "logits/rejected": 0.11941666901111603,
      "logps/chosen": -2.117626428604126,
      "logps/rejected": -2.4617509841918945,
      "loss": 2.3647,
      "nll_loss": 2.3118205070495605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21176263689994812,
      "rewards/margins": 0.0344124510884285,
      "rewards/rejected": -0.24617509543895721,
      "step": 21
    },
    {
      "epoch": 0.03467297084318361,
      "grad_norm": 0.8328697681427002,
      "learning_rate": 6.614173228346457e-07,
      "log_odds_chosen": 0.3781239092350006,
      "log_odds_ratio": -0.5352512001991272,
      "logits/chosen": -0.8822906017303467,
      "logits/rejected": 0.2483917474746704,
      "logps/chosen": -2.0555434226989746,
      "logps/rejected": -2.394667625427246,
      "loss": 2.311,
      "nll_loss": 2.257483959197998,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20555433630943298,
      "rewards/margins": 0.03391243517398834,
      "rewards/rejected": -0.23946677148342133,
      "step": 22
    },
    {
      "epoch": 0.036249014972419225,
      "grad_norm": 0.8317603468894958,
      "learning_rate": 6.929133858267716e-07,
      "log_odds_chosen": 0.4104730486869812,
      "log_odds_ratio": -0.5118555426597595,
      "logits/chosen": -0.8775284886360168,
      "logits/rejected": 0.08889060467481613,
      "logps/chosen": -2.0935451984405518,
      "logps/rejected": -2.462343454360962,
      "loss": 2.3226,
      "nll_loss": 2.271423578262329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20935453474521637,
      "rewards/margins": 0.036879826337099075,
      "rewards/rejected": -0.24623435735702515,
      "step": 23
    },
    {
      "epoch": 0.037825059101654845,
      "grad_norm": 0.7823754549026489,
      "learning_rate": 7.244094488188977e-07,
      "log_odds_chosen": 0.44625383615493774,
      "log_odds_ratio": -0.5046113133430481,
      "logits/chosen": -0.5757933259010315,
      "logits/rejected": 0.12505289912223816,
      "logps/chosen": -2.035513162612915,
      "logps/rejected": -2.434288263320923,
      "loss": 2.2836,
      "nll_loss": 2.2331085205078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20355132222175598,
      "rewards/margins": 0.039877522736787796,
      "rewards/rejected": -0.24342885613441467,
      "step": 24
    },
    {
      "epoch": 0.039401103230890466,
      "grad_norm": 0.9246788024902344,
      "learning_rate": 7.559055118110236e-07,
      "log_odds_chosen": 0.375238299369812,
      "log_odds_ratio": -0.5334768891334534,
      "logits/chosen": -0.9162179231643677,
      "logits/rejected": -0.03538450971245766,
      "logps/chosen": -2.0804378986358643,
      "logps/rejected": -2.421753168106079,
      "loss": 2.3239,
      "nll_loss": 2.2705116271972656,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20804381370544434,
      "rewards/margins": 0.03413153439760208,
      "rewards/rejected": -0.24217534065246582,
      "step": 25
    },
    {
      "epoch": 0.04097714736012608,
      "grad_norm": 0.8534817695617676,
      "learning_rate": 7.874015748031496e-07,
      "log_odds_chosen": 0.4714248776435852,
      "log_odds_ratio": -0.49334338307380676,
      "logits/chosen": -1.0039349794387817,
      "logits/rejected": -0.07227999716997147,
      "logps/chosen": -2.0296528339385986,
      "logps/rejected": -2.4532673358917236,
      "loss": 2.2775,
      "nll_loss": 2.228158473968506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20296527445316315,
      "rewards/margins": 0.04236144199967384,
      "rewards/rejected": -0.24532672762870789,
      "step": 26
    },
    {
      "epoch": 0.0425531914893617,
      "grad_norm": 0.8221213221549988,
      "learning_rate": 8.188976377952755e-07,
      "log_odds_chosen": 0.7778643369674683,
      "log_odds_ratio": -0.396297425031662,
      "logits/chosen": -0.8032987713813782,
      "logits/rejected": 0.048319678753614426,
      "logps/chosen": -2.0571279525756836,
      "logps/rejected": -2.763603687286377,
      "loss": 2.306,
      "nll_loss": 2.2663841247558594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20571278035640717,
      "rewards/margins": 0.07064756751060486,
      "rewards/rejected": -0.2763603925704956,
      "step": 27
    },
    {
      "epoch": 0.04412923561859732,
      "grad_norm": 0.8112096786499023,
      "learning_rate": 8.503937007874016e-07,
      "log_odds_chosen": 0.3961740732192993,
      "log_odds_ratio": -0.5245606899261475,
      "logits/chosen": -0.841758668422699,
      "logits/rejected": 0.1445307582616806,
      "logps/chosen": -2.0094542503356934,
      "logps/rejected": -2.3631062507629395,
      "loss": 2.2755,
      "nll_loss": 2.223043918609619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2009454220533371,
      "rewards/margins": 0.03536517173051834,
      "rewards/rejected": -0.23631060123443604,
      "step": 28
    },
    {
      "epoch": 0.045705279747832936,
      "grad_norm": 0.7597041726112366,
      "learning_rate": 8.818897637795275e-07,
      "log_odds_chosen": 0.38566550612449646,
      "log_odds_ratio": -0.5244230031967163,
      "logits/chosen": -0.6991312503814697,
      "logits/rejected": 0.12269733846187592,
      "logps/chosen": -2.1073381900787354,
      "logps/rejected": -2.4572627544403076,
      "loss": 2.3533,
      "nll_loss": 2.3008675575256348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2107338309288025,
      "rewards/margins": 0.034992463886737823,
      "rewards/rejected": -0.24572627246379852,
      "step": 29
    },
    {
      "epoch": 0.04728132387706856,
      "grad_norm": 0.9624651074409485,
      "learning_rate": 9.133858267716535e-07,
      "log_odds_chosen": 0.6133925318717957,
      "log_odds_ratio": -0.4450993537902832,
      "logits/chosen": -0.8754658102989197,
      "logits/rejected": -0.06702499091625214,
      "logps/chosen": -2.0876407623291016,
      "logps/rejected": -2.642871141433716,
      "loss": 2.3645,
      "nll_loss": 2.320007801055908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20876409113407135,
      "rewards/margins": 0.05552302300930023,
      "rewards/rejected": -0.2642871141433716,
      "step": 30
    },
    {
      "epoch": 0.04885736800630418,
      "grad_norm": 0.8790305256843567,
      "learning_rate": 9.448818897637795e-07,
      "log_odds_chosen": 0.661622166633606,
      "log_odds_ratio": -0.4319807291030884,
      "logits/chosen": -0.8294932246208191,
      "logits/rejected": 0.24975837767124176,
      "logps/chosen": -2.129549264907837,
      "logps/rejected": -2.7335667610168457,
      "loss": 2.3622,
      "nll_loss": 2.3190011978149414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21295495331287384,
      "rewards/margins": 0.06040172278881073,
      "rewards/rejected": -0.27335667610168457,
      "step": 31
    },
    {
      "epoch": 0.0504334121355398,
      "grad_norm": 0.7277955412864685,
      "learning_rate": 9.763779527559055e-07,
      "log_odds_chosen": 0.7255255579948425,
      "log_odds_ratio": -0.41055694222450256,
      "logits/chosen": -0.7205650806427002,
      "logits/rejected": -0.08150428533554077,
      "logps/chosen": -1.98775053024292,
      "logps/rejected": -2.6428921222686768,
      "loss": 2.2284,
      "nll_loss": 2.1873605251312256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19877506792545319,
      "rewards/margins": 0.06551413238048553,
      "rewards/rejected": -0.2642892003059387,
      "step": 32
    },
    {
      "epoch": 0.05200945626477541,
      "grad_norm": 0.7199560403823853,
      "learning_rate": 1.0078740157480315e-06,
      "log_odds_chosen": 0.5971649289131165,
      "log_odds_ratio": -0.44201111793518066,
      "logits/chosen": -0.6816457509994507,
      "logits/rejected": 0.10904838144779205,
      "logps/chosen": -2.0162274837493896,
      "logps/rejected": -2.551992893218994,
      "loss": 2.2488,
      "nll_loss": 2.20456862449646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20162276923656464,
      "rewards/margins": 0.0535765141248703,
      "rewards/rejected": -0.25519925355911255,
      "step": 33
    },
    {
      "epoch": 0.05358550039401103,
      "grad_norm": 0.7956497073173523,
      "learning_rate": 1.0393700787401573e-06,
      "log_odds_chosen": 0.3375328779220581,
      "log_odds_ratio": -0.5497519373893738,
      "logits/chosen": -0.6955797672271729,
      "logits/rejected": 0.017697542905807495,
      "logps/chosen": -1.9696956872940063,
      "logps/rejected": -2.263640880584717,
      "loss": 2.2282,
      "nll_loss": 2.173248767852783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19696956872940063,
      "rewards/margins": 0.029394524171948433,
      "rewards/rejected": -0.2263641059398651,
      "step": 34
    },
    {
      "epoch": 0.055161544523246654,
      "grad_norm": 1.0039724111557007,
      "learning_rate": 1.0708661417322836e-06,
      "log_odds_chosen": 0.4184088408946991,
      "log_odds_ratio": -0.5078526735305786,
      "logits/chosen": -0.8881021738052368,
      "logits/rejected": 0.3383480906486511,
      "logps/chosen": -2.167163133621216,
      "logps/rejected": -2.5441415309906006,
      "loss": 2.4142,
      "nll_loss": 2.3633856773376465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21671633422374725,
      "rewards/margins": 0.03769781440496445,
      "rewards/rejected": -0.2544141411781311,
      "step": 35
    },
    {
      "epoch": 0.05673758865248227,
      "grad_norm": 0.8594557642936707,
      "learning_rate": 1.1023622047244094e-06,
      "log_odds_chosen": 0.5742133259773254,
      "log_odds_ratio": -0.4490886628627777,
      "logits/chosen": -0.991584300994873,
      "logits/rejected": 0.18718503415584564,
      "logps/chosen": -1.9992127418518066,
      "logps/rejected": -2.51247239112854,
      "loss": 2.2435,
      "nll_loss": 2.1985414028167725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19992126524448395,
      "rewards/margins": 0.05132598429918289,
      "rewards/rejected": -0.25124722719192505,
      "step": 36
    },
    {
      "epoch": 0.05831363278171789,
      "grad_norm": 0.7524440288543701,
      "learning_rate": 1.1338582677165354e-06,
      "log_odds_chosen": 0.46497225761413574,
      "log_odds_ratio": -0.4982447028160095,
      "logits/chosen": -0.8054354786872864,
      "logits/rejected": 0.2145153284072876,
      "logps/chosen": -1.9772131443023682,
      "logps/rejected": -2.387143135070801,
      "loss": 2.238,
      "nll_loss": 2.188201665878296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19772133231163025,
      "rewards/margins": 0.04099300131201744,
      "rewards/rejected": -0.238714337348938,
      "step": 37
    },
    {
      "epoch": 0.05988967691095351,
      "grad_norm": 0.8709132075309753,
      "learning_rate": 1.1653543307086612e-06,
      "log_odds_chosen": 0.3852269649505615,
      "log_odds_ratio": -0.5274136662483215,
      "logits/chosen": -0.9475684762001038,
      "logits/rejected": -0.014422068372368813,
      "logps/chosen": -1.9959006309509277,
      "logps/rejected": -2.3345746994018555,
      "loss": 2.2597,
      "nll_loss": 2.206918716430664,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1995900422334671,
      "rewards/margins": 0.033867403864860535,
      "rewards/rejected": -0.23345746099948883,
      "step": 38
    },
    {
      "epoch": 0.061465721040189124,
      "grad_norm": 0.9423657059669495,
      "learning_rate": 1.1968503937007875e-06,
      "log_odds_chosen": 0.21351470053195953,
      "log_odds_ratio": -0.6031904816627502,
      "logits/chosen": -0.8634312152862549,
      "logits/rejected": 0.2100798338651657,
      "logps/chosen": -2.113935708999634,
      "logps/rejected": -2.3075613975524902,
      "loss": 2.3813,
      "nll_loss": 2.3209362030029297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2113935798406601,
      "rewards/margins": 0.019362568855285645,
      "rewards/rejected": -0.23075613379478455,
      "step": 39
    },
    {
      "epoch": 0.06304176516942474,
      "grad_norm": 0.739978015422821,
      "learning_rate": 1.2283464566929133e-06,
      "log_odds_chosen": 0.39791059494018555,
      "log_odds_ratio": -0.520595371723175,
      "logits/chosen": -0.8454897403717041,
      "logits/rejected": 0.05343304201960564,
      "logps/chosen": -1.95891535282135,
      "logps/rejected": -2.309115409851074,
      "loss": 2.2211,
      "nll_loss": 2.169041633605957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19589152932167053,
      "rewards/margins": 0.03501999378204346,
      "rewards/rejected": -0.23091153800487518,
      "step": 40
    },
    {
      "epoch": 0.06461780929866036,
      "grad_norm": 0.7263447046279907,
      "learning_rate": 1.2598425196850393e-06,
      "log_odds_chosen": 0.2720071077346802,
      "log_odds_ratio": -0.5738168358802795,
      "logits/chosen": -0.7613966464996338,
      "logits/rejected": 0.1504044085741043,
      "logps/chosen": -2.0094892978668213,
      "logps/rejected": -2.2524259090423584,
      "loss": 2.2512,
      "nll_loss": 2.1937756538391113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20094892382621765,
      "rewards/margins": 0.024293670430779457,
      "rewards/rejected": -0.22524259984493256,
      "step": 41
    },
    {
      "epoch": 0.06619385342789598,
      "grad_norm": 0.8427611589431763,
      "learning_rate": 1.2913385826771652e-06,
      "log_odds_chosen": 0.30227378010749817,
      "log_odds_ratio": -0.5599700808525085,
      "logits/chosen": -0.8331806063652039,
      "logits/rejected": 0.10789404809474945,
      "logps/chosen": -2.01611590385437,
      "logps/rejected": -2.2841808795928955,
      "loss": 2.2696,
      "nll_loss": 2.213594913482666,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20161160826683044,
      "rewards/margins": 0.026806501671671867,
      "rewards/rejected": -0.22841811180114746,
      "step": 42
    },
    {
      "epoch": 0.0677698975571316,
      "grad_norm": 0.7538146376609802,
      "learning_rate": 1.3228346456692914e-06,
      "log_odds_chosen": 0.1851217895746231,
      "log_odds_ratio": -0.6138940453529358,
      "logits/chosen": -0.7334487438201904,
      "logits/rejected": 0.06407423317432404,
      "logps/chosen": -2.0350100994110107,
      "logps/rejected": -2.1991522312164307,
      "loss": 2.2871,
      "nll_loss": 2.2257142066955566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20350100100040436,
      "rewards/margins": 0.016414228826761246,
      "rewards/rejected": -0.2199152410030365,
      "step": 43
    },
    {
      "epoch": 0.06934594168636722,
      "grad_norm": 0.8424365520477295,
      "learning_rate": 1.3543307086614172e-06,
      "log_odds_chosen": 0.5302658677101135,
      "log_odds_ratio": -0.4734542965888977,
      "logits/chosen": -0.8889545202255249,
      "logits/rejected": 0.2788478136062622,
      "logps/chosen": -2.0052120685577393,
      "logps/rejected": -2.481436252593994,
      "loss": 2.2513,
      "nll_loss": 2.203930377960205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20052120089530945,
      "rewards/margins": 0.0476224310696125,
      "rewards/rejected": -0.24814364314079285,
      "step": 44
    },
    {
      "epoch": 0.07092198581560284,
      "grad_norm": 0.7189680933952332,
      "learning_rate": 1.3858267716535433e-06,
      "log_odds_chosen": 0.6273162961006165,
      "log_odds_ratio": -0.44312310218811035,
      "logits/chosen": -0.6784701347351074,
      "logits/rejected": 0.18980826437473297,
      "logps/chosen": -1.957032322883606,
      "logps/rejected": -2.5182816982269287,
      "loss": 2.2283,
      "nll_loss": 2.1839797496795654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19570323824882507,
      "rewards/margins": 0.056124940514564514,
      "rewards/rejected": -0.2518281638622284,
      "step": 45
    },
    {
      "epoch": 0.07249802994483845,
      "grad_norm": 0.9589075446128845,
      "learning_rate": 1.417322834645669e-06,
      "log_odds_chosen": 0.608472466468811,
      "log_odds_ratio": -0.4546305239200592,
      "logits/chosen": -0.9549905061721802,
      "logits/rejected": -0.06440718472003937,
      "logps/chosen": -2.0081634521484375,
      "logps/rejected": -2.555389165878296,
      "loss": 2.2656,
      "nll_loss": 2.220146894454956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.200816348195076,
      "rewards/margins": 0.054722562432289124,
      "rewards/rejected": -0.2555389106273651,
      "step": 46
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 0.796640157699585,
      "learning_rate": 1.4488188976377953e-06,
      "log_odds_chosen": 0.28869107365608215,
      "log_odds_ratio": -0.5726405382156372,
      "logits/chosen": -0.956196129322052,
      "logits/rejected": 0.0430324450135231,
      "logps/chosen": -2.036902904510498,
      "logps/rejected": -2.2967493534088135,
      "loss": 2.286,
      "nll_loss": 2.22874116897583,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2036902904510498,
      "rewards/margins": 0.025984659790992737,
      "rewards/rejected": -0.22967493534088135,
      "step": 47
    },
    {
      "epoch": 0.07565011820330969,
      "grad_norm": 0.9021222591400146,
      "learning_rate": 1.4803149606299211e-06,
      "log_odds_chosen": 0.4264254570007324,
      "log_odds_ratio": -0.5128542184829712,
      "logits/chosen": -0.7777482867240906,
      "logits/rejected": 0.05279793590307236,
      "logps/chosen": -2.0905792713165283,
      "logps/rejected": -2.4763331413269043,
      "loss": 2.3535,
      "nll_loss": 2.302234411239624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20905792713165283,
      "rewards/margins": 0.038575392216444016,
      "rewards/rejected": -0.24763330817222595,
      "step": 48
    },
    {
      "epoch": 0.07722616233254531,
      "grad_norm": 0.721595823764801,
      "learning_rate": 1.5118110236220472e-06,
      "log_odds_chosen": 0.5007736682891846,
      "log_odds_ratio": -0.4825616478919983,
      "logits/chosen": -0.7792184948921204,
      "logits/rejected": 0.3143281936645508,
      "logps/chosen": -1.9676520824432373,
      "logps/rejected": -2.4107041358947754,
      "loss": 2.1893,
      "nll_loss": 2.1410772800445557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19676518440246582,
      "rewards/margins": 0.044305235147476196,
      "rewards/rejected": -0.2410704493522644,
      "step": 49
    },
    {
      "epoch": 0.07880220646178093,
      "grad_norm": 0.7847599387168884,
      "learning_rate": 1.543307086614173e-06,
      "log_odds_chosen": 0.29993894696235657,
      "log_odds_ratio": -0.5612264275550842,
      "logits/chosen": -0.8290573954582214,
      "logits/rejected": 0.06256364285945892,
      "logps/chosen": -2.046567916870117,
      "logps/rejected": -2.3153884410858154,
      "loss": 2.3029,
      "nll_loss": 2.2467591762542725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20465680956840515,
      "rewards/margins": 0.026882071048021317,
      "rewards/rejected": -0.23153886198997498,
      "step": 50
    },
    {
      "epoch": 0.08037825059101655,
      "grad_norm": 0.7472882270812988,
      "learning_rate": 1.5748031496062992e-06,
      "log_odds_chosen": 0.6475747227668762,
      "log_odds_ratio": -0.4279291033744812,
      "logits/chosen": -0.6958625316619873,
      "logits/rejected": 0.19621017575263977,
      "logps/chosen": -1.9286586046218872,
      "logps/rejected": -2.503767728805542,
      "loss": 2.171,
      "nll_loss": 2.1281983852386475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19286584854125977,
      "rewards/margins": 0.05751091241836548,
      "rewards/rejected": -0.25037679076194763,
      "step": 51
    },
    {
      "epoch": 0.08195429472025216,
      "grad_norm": 0.7411985993385315,
      "learning_rate": 1.6062992125984253e-06,
      "log_odds_chosen": 0.5698750615119934,
      "log_odds_ratio": -0.4549402594566345,
      "logits/chosen": -0.6439858078956604,
      "logits/rejected": 0.2690843939781189,
      "logps/chosen": -2.082113027572632,
      "logps/rejected": -2.5969245433807373,
      "loss": 2.3161,
      "nll_loss": 2.2706432342529297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20821130275726318,
      "rewards/margins": 0.05148116499185562,
      "rewards/rejected": -0.2596924602985382,
      "step": 52
    },
    {
      "epoch": 0.08353033884948778,
      "grad_norm": 0.8199878334999084,
      "learning_rate": 1.637795275590551e-06,
      "log_odds_chosen": 0.3187071979045868,
      "log_odds_ratio": -0.5507593154907227,
      "logits/chosen": -0.7804884910583496,
      "logits/rejected": 0.1774444580078125,
      "logps/chosen": -2.027057409286499,
      "logps/rejected": -2.3086814880371094,
      "loss": 2.276,
      "nll_loss": 2.2209625244140625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2027057409286499,
      "rewards/margins": 0.0281623974442482,
      "rewards/rejected": -0.2308681458234787,
      "step": 53
    },
    {
      "epoch": 0.0851063829787234,
      "grad_norm": 0.7717034816741943,
      "learning_rate": 1.6692913385826771e-06,
      "log_odds_chosen": 0.3182855248451233,
      "log_odds_ratio": -0.5493793487548828,
      "logits/chosen": -0.8416004180908203,
      "logits/rejected": 0.0956750363111496,
      "logps/chosen": -1.9605865478515625,
      "logps/rejected": -2.240133047103882,
      "loss": 2.2188,
      "nll_loss": 2.163889169692993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19605866074562073,
      "rewards/margins": 0.027954675257205963,
      "rewards/rejected": -0.2240133434534073,
      "step": 54
    },
    {
      "epoch": 0.08668242710795902,
      "grad_norm": 0.8444288372993469,
      "learning_rate": 1.7007874015748031e-06,
      "log_odds_chosen": 0.4131320118904114,
      "log_odds_ratio": -0.5114343762397766,
      "logits/chosen": -0.7947390675544739,
      "logits/rejected": -0.015921171754598618,
      "logps/chosen": -2.057856559753418,
      "logps/rejected": -2.428257942199707,
      "loss": 2.3167,
      "nll_loss": 2.2655208110809326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20578564703464508,
      "rewards/margins": 0.03704017400741577,
      "rewards/rejected": -0.24282580614089966,
      "step": 55
    },
    {
      "epoch": 0.08825847123719464,
      "grad_norm": 0.9216170907020569,
      "learning_rate": 1.7322834645669292e-06,
      "log_odds_chosen": 0.3599957823753357,
      "log_odds_ratio": -0.5413734316825867,
      "logits/chosen": -0.7153440117835999,
      "logits/rejected": 0.25690826773643494,
      "logps/chosen": -2.1631920337677,
      "logps/rejected": -2.4897618293762207,
      "loss": 2.4079,
      "nll_loss": 2.3537416458129883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21631920337677002,
      "rewards/margins": 0.03265698626637459,
      "rewards/rejected": -0.2489762008190155,
      "step": 56
    },
    {
      "epoch": 0.08983451536643026,
      "grad_norm": 0.698296070098877,
      "learning_rate": 1.763779527559055e-06,
      "log_odds_chosen": 0.37394940853118896,
      "log_odds_ratio": -0.5269332528114319,
      "logits/chosen": -0.7442439198493958,
      "logits/rejected": 0.17622074484825134,
      "logps/chosen": -2.0186767578125,
      "logps/rejected": -2.3512208461761475,
      "loss": 2.2588,
      "nll_loss": 2.2060835361480713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20186766982078552,
      "rewards/margins": 0.033254414796829224,
      "rewards/rejected": -0.23512208461761475,
      "step": 57
    },
    {
      "epoch": 0.09141055949566587,
      "grad_norm": 0.7946895360946655,
      "learning_rate": 1.795275590551181e-06,
      "log_odds_chosen": 0.39631205797195435,
      "log_odds_ratio": -0.5242730975151062,
      "logits/chosen": -0.7613857984542847,
      "logits/rejected": -0.0997920036315918,
      "logps/chosen": -1.9954885244369507,
      "logps/rejected": -2.352520227432251,
      "loss": 2.2451,
      "nll_loss": 2.192626953125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1995488554239273,
      "rewards/margins": 0.035703182220458984,
      "rewards/rejected": -0.2352520227432251,
      "step": 58
    },
    {
      "epoch": 0.09298660362490149,
      "grad_norm": 0.8159598112106323,
      "learning_rate": 1.826771653543307e-06,
      "log_odds_chosen": 0.7107657194137573,
      "log_odds_ratio": -0.4071710407733917,
      "logits/chosen": -0.6927255988121033,
      "logits/rejected": 0.18439871072769165,
      "logps/chosen": -2.0940284729003906,
      "logps/rejected": -2.741525411605835,
      "loss": 2.3155,
      "nll_loss": 2.2747528553009033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20940285921096802,
      "rewards/margins": 0.06474968045949936,
      "rewards/rejected": -0.274152547121048,
      "step": 59
    },
    {
      "epoch": 0.09456264775413711,
      "grad_norm": 0.8400620222091675,
      "learning_rate": 1.858267716535433e-06,
      "log_odds_chosen": 0.515011191368103,
      "log_odds_ratio": -0.4777131676673889,
      "logits/chosen": -0.8022063970565796,
      "logits/rejected": -0.011907435022294521,
      "logps/chosen": -2.0360805988311768,
      "logps/rejected": -2.5011496543884277,
      "loss": 2.2828,
      "nll_loss": 2.2350497245788574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20360803604125977,
      "rewards/margins": 0.04650692641735077,
      "rewards/rejected": -0.25011497735977173,
      "step": 60
    },
    {
      "epoch": 0.09613869188337273,
      "grad_norm": 0.861944317817688,
      "learning_rate": 1.889763779527559e-06,
      "log_odds_chosen": 0.26094532012939453,
      "log_odds_ratio": -0.580884575843811,
      "logits/chosen": -0.7177518010139465,
      "logits/rejected": 0.004429949447512627,
      "logps/chosen": -2.1590168476104736,
      "logps/rejected": -2.3958046436309814,
      "loss": 2.396,
      "nll_loss": 2.337953805923462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2159016877412796,
      "rewards/margins": 0.023678777739405632,
      "rewards/rejected": -0.23958046734333038,
      "step": 61
    },
    {
      "epoch": 0.09771473601260836,
      "grad_norm": 0.9885354042053223,
      "learning_rate": 1.9212598425196847e-06,
      "log_odds_chosen": 0.319558709859848,
      "log_odds_ratio": -0.5569351315498352,
      "logits/chosen": -0.7895075082778931,
      "logits/rejected": 0.13341262936592102,
      "logps/chosen": -2.1451916694641113,
      "logps/rejected": -2.4343841075897217,
      "loss": 2.3904,
      "nll_loss": 2.3347549438476562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21451915800571442,
      "rewards/margins": 0.02891925349831581,
      "rewards/rejected": -0.24343842267990112,
      "step": 62
    },
    {
      "epoch": 0.09929078014184398,
      "grad_norm": 0.7753734588623047,
      "learning_rate": 1.952755905511811e-06,
      "log_odds_chosen": 0.6025581359863281,
      "log_odds_ratio": -0.4435453712940216,
      "logits/chosen": -0.8974717855453491,
      "logits/rejected": -0.006312880665063858,
      "logps/chosen": -1.8660368919372559,
      "logps/rejected": -2.391263961791992,
      "loss": 2.127,
      "nll_loss": 2.0826830863952637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18660369515419006,
      "rewards/margins": 0.05252271145582199,
      "rewards/rejected": -0.23912642896175385,
      "step": 63
    },
    {
      "epoch": 0.1008668242710796,
      "grad_norm": 0.7583571076393127,
      "learning_rate": 1.9842519685039368e-06,
      "log_odds_chosen": 0.7151730060577393,
      "log_odds_ratio": -0.41310709714889526,
      "logits/chosen": -0.8151243925094604,
      "logits/rejected": 0.1741599142551422,
      "logps/chosen": -1.888004183769226,
      "logps/rejected": -2.5221481323242188,
      "loss": 2.1448,
      "nll_loss": 2.1034936904907227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18880042433738708,
      "rewards/margins": 0.06341440230607986,
      "rewards/rejected": -0.25221481919288635,
      "step": 64
    },
    {
      "epoch": 0.1024428684003152,
      "grad_norm": 0.7605916261672974,
      "learning_rate": 2.015748031496063e-06,
      "log_odds_chosen": 0.6391699314117432,
      "log_odds_ratio": -0.4316767752170563,
      "logits/chosen": -0.8075143098831177,
      "logits/rejected": -0.08158548176288605,
      "logps/chosen": -1.988166093826294,
      "logps/rejected": -2.5615224838256836,
      "loss": 2.2364,
      "nll_loss": 2.193208932876587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19881662726402283,
      "rewards/margins": 0.05733564868569374,
      "rewards/rejected": -0.25615230202674866,
      "step": 65
    },
    {
      "epoch": 0.10401891252955082,
      "grad_norm": 0.7569624185562134,
      "learning_rate": 2.047244094488189e-06,
      "log_odds_chosen": 0.47779494524002075,
      "log_odds_ratio": -0.49085569381713867,
      "logits/chosen": -0.7982770204544067,
      "logits/rejected": 0.1348651647567749,
      "logps/chosen": -2.072582721710205,
      "logps/rejected": -2.500241279602051,
      "loss": 2.3299,
      "nll_loss": 2.2808148860931396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20725825428962708,
      "rewards/margins": 0.042765870690345764,
      "rewards/rejected": -0.25002413988113403,
      "step": 66
    },
    {
      "epoch": 0.10559495665878645,
      "grad_norm": 0.792701244354248,
      "learning_rate": 2.0787401574803147e-06,
      "log_odds_chosen": 0.5766542553901672,
      "log_odds_ratio": -0.4502636194229126,
      "logits/chosen": -0.7492872476577759,
      "logits/rejected": 0.21686780452728271,
      "logps/chosen": -2.0231807231903076,
      "logps/rejected": -2.539250135421753,
      "loss": 2.2501,
      "nll_loss": 2.2051100730895996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20231808722019196,
      "rewards/margins": 0.051606953144073486,
      "rewards/rejected": -0.25392502546310425,
      "step": 67
    },
    {
      "epoch": 0.10717100078802207,
      "grad_norm": 0.9331843256950378,
      "learning_rate": 2.1102362204724405e-06,
      "log_odds_chosen": 0.4267440140247345,
      "log_odds_ratio": -0.510626494884491,
      "logits/chosen": -0.8918898105621338,
      "logits/rejected": 0.09724461287260056,
      "logps/chosen": -2.0314688682556152,
      "logps/rejected": -2.413175344467163,
      "loss": 2.3161,
      "nll_loss": 2.2650256156921387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20314690470695496,
      "rewards/margins": 0.038170650601387024,
      "rewards/rejected": -0.24131755530834198,
      "step": 68
    },
    {
      "epoch": 0.10874704491725769,
      "grad_norm": 0.8956534266471863,
      "learning_rate": 2.141732283464567e-06,
      "log_odds_chosen": 0.252351313829422,
      "log_odds_ratio": -0.5821089148521423,
      "logits/chosen": -0.5750194787979126,
      "logits/rejected": 0.10774422436952591,
      "logps/chosen": -2.1035430431365967,
      "logps/rejected": -2.328942060470581,
      "loss": 2.3665,
      "nll_loss": 2.3082704544067383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21035431325435638,
      "rewards/margins": 0.02253989316523075,
      "rewards/rejected": -0.23289422690868378,
      "step": 69
    },
    {
      "epoch": 0.11032308904649331,
      "grad_norm": 0.7740955352783203,
      "learning_rate": 2.173228346456693e-06,
      "log_odds_chosen": 0.4262125790119171,
      "log_odds_ratio": -0.508803129196167,
      "logits/chosen": -0.6928648948669434,
      "logits/rejected": 0.15248773992061615,
      "logps/chosen": -2.0291588306427,
      "logps/rejected": -2.410210132598877,
      "loss": 2.2883,
      "nll_loss": 2.237379312515259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20291589200496674,
      "rewards/margins": 0.03810514509677887,
      "rewards/rejected": -0.2410210371017456,
      "step": 70
    },
    {
      "epoch": 0.11189913317572892,
      "grad_norm": 0.9311802983283997,
      "learning_rate": 2.204724409448819e-06,
      "log_odds_chosen": 0.5261321067810059,
      "log_odds_ratio": -0.4780764877796173,
      "logits/chosen": -0.9807255268096924,
      "logits/rejected": -0.06657220423221588,
      "logps/chosen": -1.992706298828125,
      "logps/rejected": -2.4644320011138916,
      "loss": 2.2402,
      "nll_loss": 2.1923532485961914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19927063584327698,
      "rewards/margins": 0.04717259109020233,
      "rewards/rejected": -0.24644321203231812,
      "step": 71
    },
    {
      "epoch": 0.11347517730496454,
      "grad_norm": 0.801863431930542,
      "learning_rate": 2.2362204724409446e-06,
      "log_odds_chosen": 0.32317402958869934,
      "log_odds_ratio": -0.5580880641937256,
      "logits/chosen": -0.7395951151847839,
      "logits/rejected": 0.14740285277366638,
      "logps/chosen": -2.002899646759033,
      "logps/rejected": -2.290651321411133,
      "loss": 2.2534,
      "nll_loss": 2.1975412368774414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2002899944782257,
      "rewards/margins": 0.028775153681635857,
      "rewards/rejected": -0.2290651500225067,
      "step": 72
    },
    {
      "epoch": 0.11505122143420016,
      "grad_norm": 0.749702513217926,
      "learning_rate": 2.267716535433071e-06,
      "log_odds_chosen": 0.3722783923149109,
      "log_odds_ratio": -0.5272582769393921,
      "logits/chosen": -0.6592772006988525,
      "logits/rejected": 0.012937603518366814,
      "logps/chosen": -2.034029483795166,
      "logps/rejected": -2.3667900562286377,
      "loss": 2.2879,
      "nll_loss": 2.2351927757263184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20340295135974884,
      "rewards/margins": 0.03327607735991478,
      "rewards/rejected": -0.23667901754379272,
      "step": 73
    },
    {
      "epoch": 0.11662726556343578,
      "grad_norm": 0.8620375394821167,
      "learning_rate": 2.2992125984251967e-06,
      "log_odds_chosen": 0.4163767695426941,
      "log_odds_ratio": -0.5100041627883911,
      "logits/chosen": -0.8191778659820557,
      "logits/rejected": 0.17023847997188568,
      "logps/chosen": -1.923947811126709,
      "logps/rejected": -2.286886692047119,
      "loss": 2.1966,
      "nll_loss": 2.14556622505188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19239479303359985,
      "rewards/margins": 0.036293875426054,
      "rewards/rejected": -0.22868867218494415,
      "step": 74
    },
    {
      "epoch": 0.1182033096926714,
      "grad_norm": 0.817757248878479,
      "learning_rate": 2.3307086614173225e-06,
      "log_odds_chosen": 0.5530056953430176,
      "log_odds_ratio": -0.4610140919685364,
      "logits/chosen": -0.7592418193817139,
      "logits/rejected": 0.04180709272623062,
      "logps/chosen": -1.9531089067459106,
      "logps/rejected": -2.4464869499206543,
      "loss": 2.1989,
      "nll_loss": 2.152804136276245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19531089067459106,
      "rewards/margins": 0.049337826669216156,
      "rewards/rejected": -0.24464870989322662,
      "step": 75
    },
    {
      "epoch": 0.11977935382190702,
      "grad_norm": 1.0609755516052246,
      "learning_rate": 2.3622047244094483e-06,
      "log_odds_chosen": 0.5459141135215759,
      "log_odds_ratio": -0.46591970324516296,
      "logits/chosen": -0.6974417567253113,
      "logits/rejected": 0.22949092090129852,
      "logps/chosen": -1.9926183223724365,
      "logps/rejected": -2.4816536903381348,
      "loss": 2.2244,
      "nll_loss": 2.177783250808716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1992618292570114,
      "rewards/margins": 0.04890353977680206,
      "rewards/rejected": -0.24816536903381348,
      "step": 76
    },
    {
      "epoch": 0.12135539795114263,
      "grad_norm": 0.8557410836219788,
      "learning_rate": 2.393700787401575e-06,
      "log_odds_chosen": 0.5039350986480713,
      "log_odds_ratio": -0.48473480343818665,
      "logits/chosen": -0.7636761665344238,
      "logits/rejected": 0.024586625397205353,
      "logps/chosen": -2.024625778198242,
      "logps/rejected": -2.476198673248291,
      "loss": 2.2801,
      "nll_loss": 2.2316083908081055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20246253907680511,
      "rewards/margins": 0.04515731334686279,
      "rewards/rejected": -0.2476198673248291,
      "step": 77
    },
    {
      "epoch": 0.12293144208037825,
      "grad_norm": 0.6954981684684753,
      "learning_rate": 2.425196850393701e-06,
      "log_odds_chosen": 0.40985774993896484,
      "log_odds_ratio": -0.5190044641494751,
      "logits/chosen": -0.5615072250366211,
      "logits/rejected": -0.14762349426746368,
      "logps/chosen": -2.077012538909912,
      "logps/rejected": -2.4454402923583984,
      "loss": 2.3028,
      "nll_loss": 2.2509350776672363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20770126581192017,
      "rewards/margins": 0.03684278577566147,
      "rewards/rejected": -0.24454402923583984,
      "step": 78
    },
    {
      "epoch": 0.12450748620961387,
      "grad_norm": 0.7768843770027161,
      "learning_rate": 2.4566929133858266e-06,
      "log_odds_chosen": 0.5477795600891113,
      "log_odds_ratio": -0.46383628249168396,
      "logits/chosen": -0.6494770646095276,
      "logits/rejected": -0.007308202795684338,
      "logps/chosen": -2.0030148029327393,
      "logps/rejected": -2.4925520420074463,
      "loss": 2.2402,
      "nll_loss": 2.1937737464904785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20030148327350616,
      "rewards/margins": 0.04895370081067085,
      "rewards/rejected": -0.24925518035888672,
      "step": 79
    },
    {
      "epoch": 0.12608353033884948,
      "grad_norm": 0.734984815120697,
      "learning_rate": 2.488188976377953e-06,
      "log_odds_chosen": 0.5732316970825195,
      "log_odds_ratio": -0.4531470239162445,
      "logits/chosen": -0.726293683052063,
      "logits/rejected": 0.02343374863266945,
      "logps/chosen": -1.9289979934692383,
      "logps/rejected": -2.4362032413482666,
      "loss": 2.168,
      "nll_loss": 2.1227097511291504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19289979338645935,
      "rewards/margins": 0.05072051286697388,
      "rewards/rejected": -0.24362030625343323,
      "step": 80
    },
    {
      "epoch": 0.1276595744680851,
      "grad_norm": 0.6569381356239319,
      "learning_rate": 2.5196850393700787e-06,
      "log_odds_chosen": 0.42326849699020386,
      "log_odds_ratio": -0.5137047171592712,
      "logits/chosen": -0.5939977765083313,
      "logits/rejected": 0.033093176782131195,
      "logps/chosen": -1.9879000186920166,
      "logps/rejected": -2.3671982288360596,
      "loss": 2.2174,
      "nll_loss": 2.166071891784668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19878999888896942,
      "rewards/margins": 0.037929829210042953,
      "rewards/rejected": -0.23671981692314148,
      "step": 81
    },
    {
      "epoch": 0.12923561859732072,
      "grad_norm": 0.7049165964126587,
      "learning_rate": 2.5511811023622045e-06,
      "log_odds_chosen": 0.573046088218689,
      "log_odds_ratio": -0.46415337920188904,
      "logits/chosen": -0.6457680463790894,
      "logits/rejected": 0.08115581423044205,
      "logps/chosen": -1.9417915344238281,
      "logps/rejected": -2.458441734313965,
      "loss": 2.1829,
      "nll_loss": 2.136464834213257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19417917728424072,
      "rewards/margins": 0.05166499689221382,
      "rewards/rejected": -0.24584418535232544,
      "step": 82
    },
    {
      "epoch": 0.13081166272655634,
      "grad_norm": 0.7007700800895691,
      "learning_rate": 2.5826771653543303e-06,
      "log_odds_chosen": 0.3587024211883545,
      "log_odds_ratio": -0.5409030914306641,
      "logits/chosen": -0.5476737022399902,
      "logits/rejected": 0.08743837475776672,
      "logps/chosen": -1.971648931503296,
      "logps/rejected": -2.2917253971099854,
      "loss": 2.2082,
      "nll_loss": 2.154064416885376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1971648931503296,
      "rewards/margins": 0.03200764209032059,
      "rewards/rejected": -0.22917252779006958,
      "step": 83
    },
    {
      "epoch": 0.13238770685579196,
      "grad_norm": 0.8176639676094055,
      "learning_rate": 2.6141732283464566e-06,
      "log_odds_chosen": 0.5798341631889343,
      "log_odds_ratio": -0.4628051221370697,
      "logits/chosen": -0.7357879281044006,
      "logits/rejected": 0.003350723534822464,
      "logps/chosen": -1.933185338973999,
      "logps/rejected": -2.4499199390411377,
      "loss": 2.2023,
      "nll_loss": 2.1560540199279785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19331854581832886,
      "rewards/margins": 0.05167344957590103,
      "rewards/rejected": -0.2449919879436493,
      "step": 84
    },
    {
      "epoch": 0.13396375098502758,
      "grad_norm": 0.7031208872795105,
      "learning_rate": 2.645669291338583e-06,
      "log_odds_chosen": 0.5226806998252869,
      "log_odds_ratio": -0.46645060181617737,
      "logits/chosen": -0.5390961170196533,
      "logits/rejected": 0.1369251161813736,
      "logps/chosen": -1.9962283372879028,
      "logps/rejected": -2.4617295265197754,
      "loss": 2.2245,
      "nll_loss": 2.1779024600982666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19962282478809357,
      "rewards/margins": 0.046550117433071136,
      "rewards/rejected": -0.2461729496717453,
      "step": 85
    },
    {
      "epoch": 0.1355397951142632,
      "grad_norm": 0.7993008494377136,
      "learning_rate": 2.6771653543307086e-06,
      "log_odds_chosen": 0.5069440603256226,
      "log_odds_ratio": -0.476991206407547,
      "logits/chosen": -0.791284441947937,
      "logits/rejected": 0.08737476170063019,
      "logps/chosen": -1.9844413995742798,
      "logps/rejected": -2.4346706867218018,
      "loss": 2.2255,
      "nll_loss": 2.1777687072753906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1984441578388214,
      "rewards/margins": 0.04502289742231369,
      "rewards/rejected": -0.2434670478105545,
      "step": 86
    },
    {
      "epoch": 0.13711583924349882,
      "grad_norm": 0.6878076791763306,
      "learning_rate": 2.7086614173228344e-06,
      "log_odds_chosen": 0.3729919195175171,
      "log_odds_ratio": -0.5269061326980591,
      "logits/chosen": -0.5348572134971619,
      "logits/rejected": -0.09278356283903122,
      "logps/chosen": -1.9824546575546265,
      "logps/rejected": -2.310926914215088,
      "loss": 2.2139,
      "nll_loss": 2.1612019538879395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19824546575546265,
      "rewards/margins": 0.03284721076488495,
      "rewards/rejected": -0.2310926914215088,
      "step": 87
    },
    {
      "epoch": 0.13869188337273444,
      "grad_norm": 0.8332554697990417,
      "learning_rate": 2.7401574803149607e-06,
      "log_odds_chosen": 0.5939523577690125,
      "log_odds_ratio": -0.4592173099517822,
      "logits/chosen": -0.8398531079292297,
      "logits/rejected": 0.05735013633966446,
      "logps/chosen": -1.9121273756027222,
      "logps/rejected": -2.4417524337768555,
      "loss": 2.1817,
      "nll_loss": 2.135769844055176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1912127286195755,
      "rewards/margins": 0.05296248942613602,
      "rewards/rejected": -0.24417521059513092,
      "step": 88
    },
    {
      "epoch": 0.14026792750197006,
      "grad_norm": 0.8030682802200317,
      "learning_rate": 2.7716535433070865e-06,
      "log_odds_chosen": 0.5148739218711853,
      "log_odds_ratio": -0.4697887897491455,
      "logits/chosen": -0.6359347701072693,
      "logits/rejected": 0.13956201076507568,
      "logps/chosen": -2.0257599353790283,
      "logps/rejected": -2.486156463623047,
      "loss": 2.2714,
      "nll_loss": 2.224398374557495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20257601141929626,
      "rewards/margins": 0.0460396483540535,
      "rewards/rejected": -0.24861565232276917,
      "step": 89
    },
    {
      "epoch": 0.14184397163120568,
      "grad_norm": 0.7527587413787842,
      "learning_rate": 2.8031496062992123e-06,
      "log_odds_chosen": 0.43857112526893616,
      "log_odds_ratio": -0.5034822225570679,
      "logits/chosen": -0.5429673790931702,
      "logits/rejected": 0.09310950338840485,
      "logps/chosen": -2.072077989578247,
      "logps/rejected": -2.4671554565429688,
      "loss": 2.2851,
      "nll_loss": 2.2347841262817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2072077989578247,
      "rewards/margins": 0.03950772434473038,
      "rewards/rejected": -0.24671553075313568,
      "step": 90
    },
    {
      "epoch": 0.1434200157604413,
      "grad_norm": 0.6531558036804199,
      "learning_rate": 2.834645669291338e-06,
      "log_odds_chosen": 0.47114643454551697,
      "log_odds_ratio": -0.4919435381889343,
      "logits/chosen": -0.511391818523407,
      "logits/rejected": -0.023931853473186493,
      "logps/chosen": -1.860572338104248,
      "logps/rejected": -2.27347731590271,
      "loss": 2.0935,
      "nll_loss": 2.0442581176757812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1860572248697281,
      "rewards/margins": 0.0412905178964138,
      "rewards/rejected": -0.22734776139259338,
      "step": 91
    },
    {
      "epoch": 0.1449960598896769,
      "grad_norm": 0.7162215709686279,
      "learning_rate": 2.8661417322834644e-06,
      "log_odds_chosen": 0.5217415690422058,
      "log_odds_ratio": -0.47228407859802246,
      "logits/chosen": -0.6261119246482849,
      "logits/rejected": 0.10750691592693329,
      "logps/chosen": -1.8966529369354248,
      "logps/rejected": -2.357200860977173,
      "loss": 2.1443,
      "nll_loss": 2.0970218181610107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.189665287733078,
      "rewards/margins": 0.04605480283498764,
      "rewards/rejected": -0.23572009801864624,
      "step": 92
    },
    {
      "epoch": 0.14657210401891252,
      "grad_norm": 0.7588204145431519,
      "learning_rate": 2.8976377952755906e-06,
      "log_odds_chosen": 0.46790874004364014,
      "log_odds_ratio": -0.5142867565155029,
      "logits/chosen": -0.572394073009491,
      "logits/rejected": 0.08447833359241486,
      "logps/chosen": -1.9991455078125,
      "logps/rejected": -2.422028064727783,
      "loss": 2.2352,
      "nll_loss": 2.183756113052368,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19991454482078552,
      "rewards/margins": 0.04228827357292175,
      "rewards/rejected": -0.24220280349254608,
      "step": 93
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 0.698870062828064,
      "learning_rate": 2.9291338582677165e-06,
      "log_odds_chosen": 0.4843592345714569,
      "log_odds_ratio": -0.4909193813800812,
      "logits/chosen": -0.5559593439102173,
      "logits/rejected": -0.03297539800405502,
      "logps/chosen": -1.9764482975006104,
      "logps/rejected": -2.404412269592285,
      "loss": 2.2021,
      "nll_loss": 2.1530561447143555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19764482975006104,
      "rewards/margins": 0.042796410620212555,
      "rewards/rejected": -0.240441232919693,
      "step": 94
    },
    {
      "epoch": 0.14972419227738376,
      "grad_norm": 0.7901385426521301,
      "learning_rate": 2.9606299212598423e-06,
      "log_odds_chosen": 0.568733811378479,
      "log_odds_ratio": -0.459706574678421,
      "logits/chosen": -0.6126303672790527,
      "logits/rejected": 0.1119111031293869,
      "logps/chosen": -1.9837087392807007,
      "logps/rejected": -2.4926552772521973,
      "loss": 2.2244,
      "nll_loss": 2.17838978767395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19837087392807007,
      "rewards/margins": 0.050894659012556076,
      "rewards/rejected": -0.24926555156707764,
      "step": 95
    },
    {
      "epoch": 0.15130023640661938,
      "grad_norm": 0.7014930844306946,
      "learning_rate": 2.9921259842519685e-06,
      "log_odds_chosen": 0.5473185777664185,
      "log_odds_ratio": -0.46086204051971436,
      "logits/chosen": -0.6081118583679199,
      "logits/rejected": -0.03256102278828621,
      "logps/chosen": -1.9777699708938599,
      "logps/rejected": -2.4649531841278076,
      "loss": 2.1907,
      "nll_loss": 2.1445720195770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19777700304985046,
      "rewards/margins": 0.048718295991420746,
      "rewards/rejected": -0.246495321393013,
      "step": 96
    },
    {
      "epoch": 0.152876280535855,
      "grad_norm": 0.6733537316322327,
      "learning_rate": 3.0236220472440943e-06,
      "log_odds_chosen": 0.36231333017349243,
      "log_odds_ratio": -0.529702365398407,
      "logits/chosen": -0.6357975006103516,
      "logits/rejected": 0.13661673665046692,
      "logps/chosen": -1.9484719038009644,
      "logps/rejected": -2.264392137527466,
      "loss": 2.1665,
      "nll_loss": 2.113529682159424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19484718143939972,
      "rewards/margins": 0.03159204497933388,
      "rewards/rejected": -0.2264392226934433,
      "step": 97
    },
    {
      "epoch": 0.15445232466509062,
      "grad_norm": 0.6935497522354126,
      "learning_rate": 3.05511811023622e-06,
      "log_odds_chosen": 0.4252420663833618,
      "log_odds_ratio": -0.5098926424980164,
      "logits/chosen": -0.4698202908039093,
      "logits/rejected": -0.029691245406866074,
      "logps/chosen": -2.0581109523773193,
      "logps/rejected": -2.439418077468872,
      "loss": 2.2929,
      "nll_loss": 2.24191951751709,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20581108331680298,
      "rewards/margins": 0.038130730390548706,
      "rewards/rejected": -0.24394181370735168,
      "step": 98
    },
    {
      "epoch": 0.15602836879432624,
      "grad_norm": 0.753554105758667,
      "learning_rate": 3.086614173228346e-06,
      "log_odds_chosen": 0.2644384205341339,
      "log_odds_ratio": -0.5719925165176392,
      "logits/chosen": -0.7020218372344971,
      "logits/rejected": -0.07810735702514648,
      "logps/chosen": -1.9831633567810059,
      "logps/rejected": -2.2148847579956055,
      "loss": 2.2295,
      "nll_loss": 2.172307252883911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19831633567810059,
      "rewards/margins": 0.023172136396169662,
      "rewards/rejected": -0.22148849070072174,
      "step": 99
    },
    {
      "epoch": 0.15760441292356187,
      "grad_norm": 0.6946485042572021,
      "learning_rate": 3.1181102362204722e-06,
      "log_odds_chosen": 0.3836790919303894,
      "log_odds_ratio": -0.5257760286331177,
      "logits/chosen": -0.6223403215408325,
      "logits/rejected": -0.06376251578330994,
      "logps/chosen": -1.8918278217315674,
      "logps/rejected": -2.230099678039551,
      "loss": 2.1396,
      "nll_loss": 2.0870351791381836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1891828179359436,
      "rewards/margins": 0.03382713720202446,
      "rewards/rejected": -0.22300994396209717,
      "step": 100
    },
    {
      "epoch": 0.15918045705279749,
      "grad_norm": 0.6816413402557373,
      "learning_rate": 3.1496062992125985e-06,
      "log_odds_chosen": 0.44349855184555054,
      "log_odds_ratio": -0.5018309354782104,
      "logits/chosen": -0.5561724901199341,
      "logits/rejected": 0.10826245695352554,
      "logps/chosen": -1.8893253803253174,
      "logps/rejected": -2.280471086502075,
      "loss": 2.1421,
      "nll_loss": 2.0918774604797363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18893250823020935,
      "rewards/margins": 0.03911459445953369,
      "rewards/rejected": -0.22804711759090424,
      "step": 101
    },
    {
      "epoch": 0.1607565011820331,
      "grad_norm": 0.6148979067802429,
      "learning_rate": 3.1811023622047243e-06,
      "log_odds_chosen": 0.4480132758617401,
      "log_odds_ratio": -0.508931577205658,
      "logits/chosen": -0.5319602489471436,
      "logits/rejected": -0.08936317265033722,
      "logps/chosen": -1.8357093334197998,
      "logps/rejected": -2.2293267250061035,
      "loss": 2.0816,
      "nll_loss": 2.0307018756866455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18357093632221222,
      "rewards/margins": 0.03936174511909485,
      "rewards/rejected": -0.22293266654014587,
      "step": 102
    },
    {
      "epoch": 0.16233254531126873,
      "grad_norm": 0.6112706661224365,
      "learning_rate": 3.2125984251968505e-06,
      "log_odds_chosen": 0.5046467185020447,
      "log_odds_ratio": -0.4848136007785797,
      "logits/chosen": -0.4200552999973297,
      "logits/rejected": 0.25700271129608154,
      "logps/chosen": -1.818860411643982,
      "logps/rejected": -2.2571394443511963,
      "loss": 2.0709,
      "nll_loss": 2.022395372390747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18188604712486267,
      "rewards/margins": 0.04382789134979248,
      "rewards/rejected": -0.22571393847465515,
      "step": 103
    },
    {
      "epoch": 0.16390858944050432,
      "grad_norm": 0.704250156879425,
      "learning_rate": 3.2440944881889763e-06,
      "log_odds_chosen": 0.46751153469085693,
      "log_odds_ratio": -0.49732449650764465,
      "logits/chosen": -0.38590556383132935,
      "logits/rejected": 0.3286210894584656,
      "logps/chosen": -1.9726375341415405,
      "logps/rejected": -2.3892126083374023,
      "loss": 2.2131,
      "nll_loss": 2.163343667984009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19726374745368958,
      "rewards/margins": 0.04165750741958618,
      "rewards/rejected": -0.23892125487327576,
      "step": 104
    },
    {
      "epoch": 0.16548463356973994,
      "grad_norm": 0.5806333422660828,
      "learning_rate": 3.275590551181102e-06,
      "log_odds_chosen": 0.5418988466262817,
      "log_odds_ratio": -0.46480318903923035,
      "logits/chosen": -0.5418411493301392,
      "logits/rejected": 0.17040874063968658,
      "logps/chosen": -1.8416911363601685,
      "logps/rejected": -2.3162972927093506,
      "loss": 2.0965,
      "nll_loss": 2.0500617027282715,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18416911363601685,
      "rewards/margins": 0.047460608184337616,
      "rewards/rejected": -0.23162971436977386,
      "step": 105
    },
    {
      "epoch": 0.16706067769897556,
      "grad_norm": 0.6116374135017395,
      "learning_rate": 3.307086614173228e-06,
      "log_odds_chosen": 0.47532159090042114,
      "log_odds_ratio": -0.4957353472709656,
      "logits/chosen": -0.5306176543235779,
      "logits/rejected": 0.20767346024513245,
      "logps/chosen": -1.8645740747451782,
      "logps/rejected": -2.2723007202148438,
      "loss": 2.1024,
      "nll_loss": 2.0528042316436768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18645742535591125,
      "rewards/margins": 0.04077265411615372,
      "rewards/rejected": -0.22723007202148438,
      "step": 106
    },
    {
      "epoch": 0.16863672182821118,
      "grad_norm": 0.6184962391853333,
      "learning_rate": 3.3385826771653542e-06,
      "log_odds_chosen": 0.4586809277534485,
      "log_odds_ratio": -0.5017204284667969,
      "logits/chosen": -0.4147133529186249,
      "logits/rejected": 0.059773970395326614,
      "logps/chosen": -1.9274760484695435,
      "logps/rejected": -2.333932399749756,
      "loss": 2.1614,
      "nll_loss": 2.1112310886383057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19274760782718658,
      "rewards/margins": 0.04064565151929855,
      "rewards/rejected": -0.23339325189590454,
      "step": 107
    },
    {
      "epoch": 0.1702127659574468,
      "grad_norm": 0.5825785398483276,
      "learning_rate": 3.37007874015748e-06,
      "log_odds_chosen": 0.35450294613838196,
      "log_odds_ratio": -0.5409185886383057,
      "logits/chosen": -0.39133238792419434,
      "logits/rejected": 0.04965784400701523,
      "logps/chosen": -1.9083962440490723,
      "logps/rejected": -2.217801809310913,
      "loss": 2.1444,
      "nll_loss": 2.090298652648926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19083963334560394,
      "rewards/margins": 0.030940547585487366,
      "rewards/rejected": -0.22178016602993011,
      "step": 108
    },
    {
      "epoch": 0.17178881008668243,
      "grad_norm": 0.6102772951126099,
      "learning_rate": 3.4015748031496063e-06,
      "log_odds_chosen": 0.48623126745224,
      "log_odds_ratio": -0.4813230633735657,
      "logits/chosen": -0.31981658935546875,
      "logits/rejected": 0.002862483263015747,
      "logps/chosen": -1.9655230045318604,
      "logps/rejected": -2.394603729248047,
      "loss": 2.1966,
      "nll_loss": 2.1484334468841553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1965523064136505,
      "rewards/margins": 0.042908087372779846,
      "rewards/rejected": -0.23946037888526917,
      "step": 109
    },
    {
      "epoch": 0.17336485421591805,
      "grad_norm": 0.595453679561615,
      "learning_rate": 3.433070866141732e-06,
      "log_odds_chosen": 0.5263383388519287,
      "log_odds_ratio": -0.47105205059051514,
      "logits/chosen": -0.42303267121315,
      "logits/rejected": 0.10894276201725006,
      "logps/chosen": -1.8528122901916504,
      "logps/rejected": -2.307560920715332,
      "loss": 2.0631,
      "nll_loss": 2.016003131866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18528123199939728,
      "rewards/margins": 0.045474857091903687,
      "rewards/rejected": -0.23075608909130096,
      "step": 110
    },
    {
      "epoch": 0.17494089834515367,
      "grad_norm": 0.6606065630912781,
      "learning_rate": 3.4645669291338583e-06,
      "log_odds_chosen": 0.5373308062553406,
      "log_odds_ratio": -0.47022971510887146,
      "logits/chosen": -0.39981910586357117,
      "logits/rejected": 0.004297524690628052,
      "logps/chosen": -1.961772084236145,
      "logps/rejected": -2.4356696605682373,
      "loss": 2.1978,
      "nll_loss": 2.150740385055542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19617722928524017,
      "rewards/margins": 0.047389768064022064,
      "rewards/rejected": -0.24356698989868164,
      "step": 111
    },
    {
      "epoch": 0.1765169424743893,
      "grad_norm": 0.603425920009613,
      "learning_rate": 3.496062992125984e-06,
      "log_odds_chosen": 0.3936331868171692,
      "log_odds_ratio": -0.5209735035896301,
      "logits/chosen": -0.33380576968193054,
      "logits/rejected": 0.18296881020069122,
      "logps/chosen": -1.9979010820388794,
      "logps/rejected": -2.344928026199341,
      "loss": 2.2248,
      "nll_loss": 2.1727278232574463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1997901350259781,
      "rewards/margins": 0.034702688455581665,
      "rewards/rejected": -0.23449280858039856,
      "step": 112
    },
    {
      "epoch": 0.1780929866036249,
      "grad_norm": 0.5860929489135742,
      "learning_rate": 3.52755905511811e-06,
      "log_odds_chosen": 0.4042174816131592,
      "log_odds_ratio": -0.5187764763832092,
      "logits/chosen": -0.4759305715560913,
      "logits/rejected": 0.05041307210922241,
      "logps/chosen": -1.8414995670318604,
      "logps/rejected": -2.19472599029541,
      "loss": 2.0856,
      "nll_loss": 2.0337438583374023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18414995074272156,
      "rewards/margins": 0.035322654992341995,
      "rewards/rejected": -0.21947260200977325,
      "step": 113
    },
    {
      "epoch": 0.17966903073286053,
      "grad_norm": 0.5595982074737549,
      "learning_rate": 3.559055118110236e-06,
      "log_odds_chosen": 0.32051122188568115,
      "log_odds_ratio": -0.562548041343689,
      "logits/chosen": -0.22491343319416046,
      "logits/rejected": 0.09666822850704193,
      "logps/chosen": -1.968389630317688,
      "logps/rejected": -2.2551064491271973,
      "loss": 2.2079,
      "nll_loss": 2.151672124862671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19683893024921417,
      "rewards/margins": 0.028671709820628166,
      "rewards/rejected": -0.22551067173480988,
      "step": 114
    },
    {
      "epoch": 0.18124507486209615,
      "grad_norm": 0.5502125024795532,
      "learning_rate": 3.590551181102362e-06,
      "log_odds_chosen": 0.49229851365089417,
      "log_odds_ratio": -0.48761364817619324,
      "logits/chosen": -0.34304749965667725,
      "logits/rejected": -0.03732027858495712,
      "logps/chosen": -1.8795965909957886,
      "logps/rejected": -2.3121144771575928,
      "loss": 2.1061,
      "nll_loss": 2.0573036670684814,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18795965611934662,
      "rewards/margins": 0.043251797556877136,
      "rewards/rejected": -0.23121145367622375,
      "step": 115
    },
    {
      "epoch": 0.18282111899133174,
      "grad_norm": 0.5441986918449402,
      "learning_rate": 3.622047244094488e-06,
      "log_odds_chosen": 0.4177536368370056,
      "log_odds_ratio": -0.5228228569030762,
      "logits/chosen": -0.32923153042793274,
      "logits/rejected": 0.12902945280075073,
      "logps/chosen": -1.9323790073394775,
      "logps/rejected": -2.3019943237304688,
      "loss": 2.1496,
      "nll_loss": 2.097325563430786,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19323790073394775,
      "rewards/margins": 0.03696154057979584,
      "rewards/rejected": -0.2301994264125824,
      "step": 116
    },
    {
      "epoch": 0.18439716312056736,
      "grad_norm": 0.5771306753158569,
      "learning_rate": 3.653543307086614e-06,
      "log_odds_chosen": 0.5892273783683777,
      "log_odds_ratio": -0.45790567994117737,
      "logits/chosen": -0.42785924673080444,
      "logits/rejected": -0.1212269514799118,
      "logps/chosen": -1.8676490783691406,
      "logps/rejected": -2.3864071369171143,
      "loss": 2.106,
      "nll_loss": 2.060208559036255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1867649108171463,
      "rewards/margins": 0.05187581479549408,
      "rewards/rejected": -0.23864072561264038,
      "step": 117
    },
    {
      "epoch": 0.18597320724980299,
      "grad_norm": 0.5304055213928223,
      "learning_rate": 3.68503937007874e-06,
      "log_odds_chosen": 0.3501071035861969,
      "log_odds_ratio": -0.5444656610488892,
      "logits/chosen": -0.19212201237678528,
      "logits/rejected": 0.03166138380765915,
      "logps/chosen": -1.8767951726913452,
      "logps/rejected": -2.1823959350585938,
      "loss": 2.1161,
      "nll_loss": 2.0616323947906494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18767952919006348,
      "rewards/margins": 0.030560053884983063,
      "rewards/rejected": -0.21823956072330475,
      "step": 118
    },
    {
      "epoch": 0.1875492513790386,
      "grad_norm": 0.5213944315910339,
      "learning_rate": 3.716535433070866e-06,
      "log_odds_chosen": 0.3437122404575348,
      "log_odds_ratio": -0.5484852194786072,
      "logits/chosen": -0.2524073123931885,
      "logits/rejected": 0.11574040353298187,
      "logps/chosen": -1.8521205186843872,
      "logps/rejected": -2.152954578399658,
      "loss": 2.1082,
      "nll_loss": 2.0533039569854736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18521204590797424,
      "rewards/margins": 0.03008340485394001,
      "rewards/rejected": -0.2152954488992691,
      "step": 119
    },
    {
      "epoch": 0.18912529550827423,
      "grad_norm": 0.4969903528690338,
      "learning_rate": 3.748031496062992e-06,
      "log_odds_chosen": 0.6459031701087952,
      "log_odds_ratio": -0.4308077096939087,
      "logits/chosen": -0.2889327108860016,
      "logits/rejected": 0.09839168190956116,
      "logps/chosen": -1.7536612749099731,
      "logps/rejected": -2.3134381771087646,
      "loss": 1.9798,
      "nll_loss": 1.9366917610168457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1753661185503006,
      "rewards/margins": 0.05597769469022751,
      "rewards/rejected": -0.2313438206911087,
      "step": 120
    },
    {
      "epoch": 0.19070133963750985,
      "grad_norm": 0.47284096479415894,
      "learning_rate": 3.779527559055118e-06,
      "log_odds_chosen": 0.6493716239929199,
      "log_odds_ratio": -0.4239138066768646,
      "logits/chosen": -0.33733659982681274,
      "logits/rejected": -0.07449808716773987,
      "logps/chosen": -1.8633861541748047,
      "logps/rejected": -2.43552303314209,
      "loss": 2.0796,
      "nll_loss": 2.0372438430786133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1863386034965515,
      "rewards/margins": 0.0572136826813221,
      "rewards/rejected": -0.2435522973537445,
      "step": 121
    },
    {
      "epoch": 0.19227738376674547,
      "grad_norm": 0.5358994603157043,
      "learning_rate": 3.8110236220472436e-06,
      "log_odds_chosen": 0.552227795124054,
      "log_odds_ratio": -0.4722144603729248,
      "logits/chosen": -0.4011126756668091,
      "logits/rejected": 0.0521632581949234,
      "logps/chosen": -1.7913762331008911,
      "logps/rejected": -2.2755613327026367,
      "loss": 2.0092,
      "nll_loss": 1.961932897567749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17913760244846344,
      "rewards/margins": 0.04841852933168411,
      "rewards/rejected": -0.22755613923072815,
      "step": 122
    },
    {
      "epoch": 0.1938534278959811,
      "grad_norm": 0.5210543870925903,
      "learning_rate": 3.8425196850393695e-06,
      "log_odds_chosen": 0.42144662141799927,
      "log_odds_ratio": -0.5061027407646179,
      "logits/chosen": -0.13411694765090942,
      "logits/rejected": 0.00012157857418060303,
      "logps/chosen": -1.8857041597366333,
      "logps/rejected": -2.2534890174865723,
      "loss": 2.1206,
      "nll_loss": 2.0700302124023438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18857041001319885,
      "rewards/margins": 0.036778487265110016,
      "rewards/rejected": -0.22534890472888947,
      "step": 123
    },
    {
      "epoch": 0.1954294720252167,
      "grad_norm": 0.4755237400531769,
      "learning_rate": 3.874015748031496e-06,
      "log_odds_chosen": 0.4865044355392456,
      "log_odds_ratio": -0.49245476722717285,
      "logits/chosen": -0.34655338525772095,
      "logits/rejected": 0.11143852770328522,
      "logps/chosen": -1.7666759490966797,
      "logps/rejected": -2.1775081157684326,
      "loss": 1.9857,
      "nll_loss": 1.9364441633224487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17666758596897125,
      "rewards/margins": 0.04108322411775589,
      "rewards/rejected": -0.21775081753730774,
      "step": 124
    },
    {
      "epoch": 0.19700551615445233,
      "grad_norm": 0.4905242323875427,
      "learning_rate": 3.905511811023622e-06,
      "log_odds_chosen": 0.46543189883232117,
      "log_odds_ratio": -0.49489825963974,
      "logits/chosen": -0.2513599693775177,
      "logits/rejected": -0.10763716697692871,
      "logps/chosen": -1.8007551431655884,
      "logps/rejected": -2.2026453018188477,
      "loss": 2.0412,
      "nll_loss": 1.9916858673095703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1800755262374878,
      "rewards/margins": 0.04018900915980339,
      "rewards/rejected": -0.2202645242214203,
      "step": 125
    },
    {
      "epoch": 0.19858156028368795,
      "grad_norm": 0.5292844772338867,
      "learning_rate": 3.937007874015748e-06,
      "log_odds_chosen": 0.6494875550270081,
      "log_odds_ratio": -0.4264853000640869,
      "logits/chosen": -0.21110114455223083,
      "logits/rejected": 0.15301132202148438,
      "logps/chosen": -1.8184471130371094,
      "logps/rejected": -2.390329599380493,
      "loss": 2.038,
      "nll_loss": 1.995354413986206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18184472620487213,
      "rewards/margins": 0.0571882426738739,
      "rewards/rejected": -0.23903295397758484,
      "step": 126
    },
    {
      "epoch": 0.20015760441292357,
      "grad_norm": 0.5565134882926941,
      "learning_rate": 3.9685039370078736e-06,
      "log_odds_chosen": 0.5321060419082642,
      "log_odds_ratio": -0.47239407896995544,
      "logits/chosen": -0.21071690320968628,
      "logits/rejected": 0.12574365735054016,
      "logps/chosen": -1.849616289138794,
      "logps/rejected": -2.318277597427368,
      "loss": 2.1051,
      "nll_loss": 2.0578956604003906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18496163189411163,
      "rewards/margins": 0.04686610400676727,
      "rewards/rejected": -0.2318277359008789,
      "step": 127
    },
    {
      "epoch": 0.2017336485421592,
      "grad_norm": 0.5013044476509094,
      "learning_rate": 4e-06,
      "log_odds_chosen": 0.3478800356388092,
      "log_odds_ratio": -0.5399872064590454,
      "logits/chosen": -0.08074170351028442,
      "logits/rejected": 0.3015192151069641,
      "logps/chosen": -1.9230244159698486,
      "logps/rejected": -2.2274985313415527,
      "loss": 2.1378,
      "nll_loss": 2.083803653717041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19230243563652039,
      "rewards/margins": 0.030447421595454216,
      "rewards/rejected": -0.22274987399578094,
      "step": 128
    },
    {
      "epoch": 0.2033096926713948,
      "grad_norm": 0.4872080385684967,
      "learning_rate": 3.999992445477635e-06,
      "log_odds_chosen": 0.4767901301383972,
      "log_odds_ratio": -0.49153631925582886,
      "logits/chosen": -0.129595085978508,
      "logits/rejected": 0.0004668300971388817,
      "logps/chosen": -1.9134724140167236,
      "logps/rejected": -2.3344011306762695,
      "loss": 2.1148,
      "nll_loss": 2.065617799758911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.19134724140167236,
      "rewards/margins": 0.04209288954734802,
      "rewards/rejected": -0.2334401160478592,
      "step": 129
    },
    {
      "epoch": 0.2048857368006304,
      "grad_norm": 0.5020763278007507,
      "learning_rate": 3.999969781967615e-06,
      "log_odds_chosen": 0.30211132764816284,
      "log_odds_ratio": -0.5571832060813904,
      "logits/chosen": -0.12993694841861725,
      "logits/rejected": 0.16511544585227966,
      "logps/chosen": -2.020594358444214,
      "logps/rejected": -2.288883686065674,
      "loss": 2.2185,
      "nll_loss": 2.162761688232422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20205941796302795,
      "rewards/margins": 0.026828955858945847,
      "rewards/rejected": -0.2288883775472641,
      "step": 130
    },
    {
      "epoch": 0.20646178092986603,
      "grad_norm": 0.45242053270339966,
      "learning_rate": 3.99993200964115e-06,
      "log_odds_chosen": 0.3914204239845276,
      "log_odds_ratio": -0.5182069540023804,
      "logits/chosen": -0.19754824042320251,
      "logits/rejected": -0.0378432460129261,
      "logps/chosen": -1.8395533561706543,
      "logps/rejected": -2.1777098178863525,
      "loss": 2.0554,
      "nll_loss": 2.00355863571167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1839553415775299,
      "rewards/margins": 0.03381563723087311,
      "rewards/rejected": -0.21777096390724182,
      "step": 131
    },
    {
      "epoch": 0.20803782505910165,
      "grad_norm": 0.5595487952232361,
      "learning_rate": 3.99987912878359e-06,
      "log_odds_chosen": 0.18296487629413605,
      "log_odds_ratio": -0.6110657453536987,
      "logits/chosen": -0.14159467816352844,
      "logits/rejected": 0.2608286738395691,
      "logps/chosen": -1.8113925457000732,
      "logps/rejected": -1.9622738361358643,
      "loss": 2.0762,
      "nll_loss": 2.0150678157806396,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1811392605304718,
      "rewards/margins": 0.015088127925992012,
      "rewards/rejected": -0.19622740149497986,
      "step": 132
    },
    {
      "epoch": 0.20961386918833727,
      "grad_norm": 0.4690943956375122,
      "learning_rate": 3.999811139794429e-06,
      "log_odds_chosen": 0.41473308205604553,
      "log_odds_ratio": -0.5109004378318787,
      "logits/chosen": -0.10903345048427582,
      "logits/rejected": 0.08273053169250488,
      "logps/chosen": -1.8365156650543213,
      "logps/rejected": -2.195512056350708,
      "loss": 2.0443,
      "nll_loss": 1.9931950569152832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18365156650543213,
      "rewards/margins": 0.03589965030550957,
      "rewards/rejected": -0.2195512056350708,
      "step": 133
    },
    {
      "epoch": 0.2111899133175729,
      "grad_norm": 0.49801507592201233,
      "learning_rate": 3.999728043187288e-06,
      "log_odds_chosen": 0.4041176736354828,
      "log_odds_ratio": -0.5218353867530823,
      "logits/chosen": -0.08679277449846268,
      "logits/rejected": 0.06324957311153412,
      "logps/chosen": -1.8879791498184204,
      "logps/rejected": -2.2436025142669678,
      "loss": 2.1203,
      "nll_loss": 2.068152904510498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18879792094230652,
      "rewards/margins": 0.03556232899427414,
      "rewards/rejected": -0.22436025738716125,
      "step": 134
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 0.4822520315647125,
      "learning_rate": 3.999629839589922e-06,
      "log_odds_chosen": 0.25128376483917236,
      "log_odds_ratio": -0.5789471864700317,
      "logits/chosen": -0.040255192667245865,
      "logits/rejected": -0.0998617559671402,
      "logps/chosen": -1.9691245555877686,
      "logps/rejected": -2.189525842666626,
      "loss": 2.1988,
      "nll_loss": 2.140864372253418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1969124674797058,
      "rewards/margins": 0.022040119394659996,
      "rewards/rejected": -0.21895259618759155,
      "step": 135
    },
    {
      "epoch": 0.21434200157604413,
      "grad_norm": 0.46929681301116943,
      "learning_rate": 3.999516529744215e-06,
      "log_odds_chosen": 0.356317937374115,
      "log_odds_ratio": -0.5354443192481995,
      "logits/chosen": 0.03701853007078171,
      "logits/rejected": -0.025361815467476845,
      "logps/chosen": -1.8457211256027222,
      "logps/rejected": -2.154231548309326,
      "loss": 2.0676,
      "nll_loss": 2.0140366554260254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18457213044166565,
      "rewards/margins": 0.03085101954638958,
      "rewards/rejected": -0.215423122048378,
      "step": 136
    },
    {
      "epoch": 0.21591804570527975,
      "grad_norm": 0.45633479952812195,
      "learning_rate": 3.999388114506166e-06,
      "log_odds_chosen": 0.44111165404319763,
      "log_odds_ratio": -0.5032743811607361,
      "logits/chosen": -0.10049010813236237,
      "logits/rejected": -0.09671763330698013,
      "logps/chosen": -1.7441117763519287,
      "logps/rejected": -2.1240317821502686,
      "loss": 1.958,
      "nll_loss": 1.9076509475708008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17441117763519287,
      "rewards/margins": 0.03799201920628548,
      "rewards/rejected": -0.21240317821502686,
      "step": 137
    },
    {
      "epoch": 0.21749408983451538,
      "grad_norm": 0.44847702980041504,
      "learning_rate": 3.999244594845892e-06,
      "log_odds_chosen": 0.30588337779045105,
      "log_odds_ratio": -0.5579502582550049,
      "logits/chosen": -0.06541359424591064,
      "logits/rejected": -0.00460642296820879,
      "logps/chosen": -1.7904387712478638,
      "logps/rejected": -2.0546517372131348,
      "loss": 2.0168,
      "nll_loss": 1.9609698057174683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17904390394687653,
      "rewards/margins": 0.026421282440423965,
      "rewards/rejected": -0.2054651826620102,
      "step": 138
    },
    {
      "epoch": 0.219070133963751,
      "grad_norm": 0.45796382427215576,
      "learning_rate": 3.999085971847616e-06,
      "log_odds_chosen": 0.26914283633232117,
      "log_odds_ratio": -0.5760324597358704,
      "logits/chosen": -0.05198541656136513,
      "logits/rejected": -0.22539588809013367,
      "logps/chosen": -1.850884199142456,
      "logps/rejected": -2.085418462753296,
      "loss": 2.0685,
      "nll_loss": 2.010895252227783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18508842587471008,
      "rewards/margins": 0.02345343679189682,
      "rewards/rejected": -0.2085418552160263,
      "step": 139
    },
    {
      "epoch": 0.22064617809298662,
      "grad_norm": 0.44731172919273376,
      "learning_rate": 3.998912246709658e-06,
      "log_odds_chosen": 0.41634848713874817,
      "log_odds_ratio": -0.5201424360275269,
      "logits/chosen": -0.1276709884405136,
      "logits/rejected": -0.13816407322883606,
      "logps/chosen": -1.74984610080719,
      "logps/rejected": -2.1083877086639404,
      "loss": 1.9674,
      "nll_loss": 1.915358304977417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1749846190214157,
      "rewards/margins": 0.03585416078567505,
      "rewards/rejected": -0.21083875000476837,
      "step": 140
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 0.47976112365722656,
      "learning_rate": 3.9987234207444295e-06,
      "log_odds_chosen": 0.36653023958206177,
      "log_odds_ratio": -0.5325444340705872,
      "logits/chosen": -0.09957035630941391,
      "logits/rejected": 0.037235602736473083,
      "logps/chosen": -1.85134756565094,
      "logps/rejected": -2.1700351238250732,
      "loss": 2.0714,
      "nll_loss": 2.018110752105713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18513476848602295,
      "rewards/margins": 0.031868740916252136,
      "rewards/rejected": -0.21700350940227509,
      "step": 141
    },
    {
      "epoch": 0.22379826635145783,
      "grad_norm": 0.47072869539260864,
      "learning_rate": 3.998519495378419e-06,
      "log_odds_chosen": 0.38448622822761536,
      "log_odds_ratio": -0.524972677230835,
      "logits/chosen": 0.012963086366653442,
      "logits/rejected": 0.13648656010627747,
      "logps/chosen": -1.9236266613006592,
      "logps/rejected": -2.262057304382324,
      "loss": 2.1162,
      "nll_loss": 2.063678503036499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19236266613006592,
      "rewards/margins": 0.03384307771921158,
      "rewards/rejected": -0.2262057363986969,
      "step": 142
    },
    {
      "epoch": 0.22537431048069345,
      "grad_norm": 0.4084656536579132,
      "learning_rate": 3.998300472152187e-06,
      "log_odds_chosen": 0.3563390374183655,
      "log_odds_ratio": -0.539592981338501,
      "logits/chosen": -0.11657491326332092,
      "logits/rejected": -0.21950428187847137,
      "logps/chosen": -1.7084014415740967,
      "logps/rejected": -2.0116360187530518,
      "loss": 1.9309,
      "nll_loss": 1.8769128322601318,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17084015905857086,
      "rewards/margins": 0.030323447659611702,
      "rewards/rejected": -0.20116360485553741,
      "step": 143
    },
    {
      "epoch": 0.22695035460992907,
      "grad_norm": 0.44106900691986084,
      "learning_rate": 3.998066352720347e-06,
      "log_odds_chosen": 0.45243191719055176,
      "log_odds_ratio": -0.49694910645484924,
      "logits/chosen": 0.07859814167022705,
      "logits/rejected": 0.016231544315814972,
      "logps/chosen": -1.8335413932800293,
      "logps/rejected": -2.2273075580596924,
      "loss": 2.0408,
      "nll_loss": 1.9911000728607178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18335412442684174,
      "rewards/margins": 0.0393766388297081,
      "rewards/rejected": -0.22273077070713043,
      "step": 144
    },
    {
      "epoch": 0.2285263987391647,
      "grad_norm": 0.41912582516670227,
      "learning_rate": 3.997817138851562e-06,
      "log_odds_chosen": 0.33265724778175354,
      "log_odds_ratio": -0.5518072843551636,
      "logits/chosen": 0.032988160848617554,
      "logits/rejected": 0.03446170315146446,
      "logps/chosen": -1.9223456382751465,
      "logps/rejected": -2.215817928314209,
      "loss": 2.1297,
      "nll_loss": 2.0745091438293457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1922345608472824,
      "rewards/margins": 0.029347218573093414,
      "rewards/rejected": -0.22158177196979523,
      "step": 145
    },
    {
      "epoch": 0.23010244286840031,
      "grad_norm": 0.40147900581359863,
      "learning_rate": 3.997552832428522e-06,
      "log_odds_chosen": 0.42633017897605896,
      "log_odds_ratio": -0.5056131482124329,
      "logits/chosen": -0.0994889885187149,
      "logits/rejected": 0.03103024512529373,
      "logps/chosen": -1.7114800214767456,
      "logps/rejected": -2.072697639465332,
      "loss": 1.9484,
      "nll_loss": 1.8978841304779053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17114798724651337,
      "rewards/margins": 0.03612176328897476,
      "rewards/rejected": -0.20726974308490753,
      "step": 146
    },
    {
      "epoch": 0.23167848699763594,
      "grad_norm": 0.39530226588249207,
      "learning_rate": 3.9972734354479366e-06,
      "log_odds_chosen": 0.3678266108036041,
      "log_odds_ratio": -0.5362446308135986,
      "logits/chosen": 0.022561486810445786,
      "logits/rejected": -0.250899076461792,
      "logps/chosen": -1.7041471004486084,
      "logps/rejected": -2.0142199993133545,
      "loss": 1.9268,
      "nll_loss": 1.8731917142868042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17041468620300293,
      "rewards/margins": 0.0310073085129261,
      "rewards/rejected": -0.20142200589179993,
      "step": 147
    },
    {
      "epoch": 0.23325453112687156,
      "grad_norm": 0.43584540486335754,
      "learning_rate": 3.996978950020517e-06,
      "log_odds_chosen": 0.23817729949951172,
      "log_odds_ratio": -0.5939024090766907,
      "logits/chosen": 0.11393069475889206,
      "logits/rejected": 0.006526513025164604,
      "logps/chosen": -1.8926572799682617,
      "logps/rejected": -2.095795154571533,
      "loss": 2.1141,
      "nll_loss": 2.0546836853027344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18926572799682617,
      "rewards/margins": 0.020313819870352745,
      "rewards/rejected": -0.20957954227924347,
      "step": 148
    },
    {
      "epoch": 0.23483057525610718,
      "grad_norm": 0.37672045826911926,
      "learning_rate": 3.996669378370959e-06,
      "log_odds_chosen": 0.5706081986427307,
      "log_odds_ratio": -0.4550899863243103,
      "logits/chosen": -0.02084711194038391,
      "logits/rejected": -0.013911699876189232,
      "logps/chosen": -1.6749799251556396,
      "logps/rejected": -2.16387939453125,
      "loss": 1.8911,
      "nll_loss": 1.8456203937530518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16749800741672516,
      "rewards/margins": 0.04888994246721268,
      "rewards/rejected": -0.21638792753219604,
      "step": 149
    },
    {
      "epoch": 0.2364066193853428,
      "grad_norm": 0.5808645486831665,
      "learning_rate": 3.996344722837929e-06,
      "log_odds_chosen": 0.5190019011497498,
      "log_odds_ratio": -0.4721868932247162,
      "logits/chosen": -0.027724824845790863,
      "logits/rejected": -0.011851985938847065,
      "logps/chosen": -1.842341661453247,
      "logps/rejected": -2.2962372303009033,
      "loss": 2.0389,
      "nll_loss": 1.9916990995407104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18423417210578918,
      "rewards/margins": 0.045389577746391296,
      "rewards/rejected": -0.2296237349510193,
      "step": 150
    },
    {
      "epoch": 0.23798266351457842,
      "grad_norm": 0.35176411271095276,
      "learning_rate": 3.996004985874043e-06,
      "log_odds_chosen": 0.2666694223880768,
      "log_odds_ratio": -0.5715771913528442,
      "logits/chosen": 0.09955260902643204,
      "logits/rejected": -0.007417397573590279,
      "logps/chosen": -1.7254502773284912,
      "logps/rejected": -1.9494432210922241,
      "loss": 1.9513,
      "nll_loss": 1.8941357135772705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17254504561424255,
      "rewards/margins": 0.02239927649497986,
      "rewards/rejected": -0.1949443221092224,
      "step": 151
    },
    {
      "epoch": 0.23955870764381404,
      "grad_norm": 0.39265239238739014,
      "learning_rate": 3.995650170045855e-06,
      "log_odds_chosen": 0.31694257259368896,
      "log_odds_ratio": -0.5533034801483154,
      "logits/chosen": 0.014175724238157272,
      "logits/rejected": -0.0333397313952446,
      "logps/chosen": -1.7931270599365234,
      "logps/rejected": -2.063905954360962,
      "loss": 2.0047,
      "nll_loss": 1.9493213891983032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17931270599365234,
      "rewards/margins": 0.027077894657850266,
      "rewards/rejected": -0.2063905894756317,
      "step": 152
    },
    {
      "epoch": 0.24113475177304963,
      "grad_norm": 0.40053820610046387,
      "learning_rate": 3.995280278033825e-06,
      "log_odds_chosen": 0.23943039774894714,
      "log_odds_ratio": -0.589293360710144,
      "logits/chosen": 0.09469472616910934,
      "logits/rejected": 0.06681188195943832,
      "logps/chosen": -1.8935778141021729,
      "logps/rejected": -2.0991814136505127,
      "loss": 2.1063,
      "nll_loss": 2.047382354736328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18935777246952057,
      "rewards/margins": 0.020560378208756447,
      "rewards/rejected": -0.20991815626621246,
      "step": 153
    },
    {
      "epoch": 0.24271079590228525,
      "grad_norm": 0.3706309199333191,
      "learning_rate": 3.994895312632314e-06,
      "log_odds_chosen": 0.5549086928367615,
      "log_odds_ratio": -0.459675669670105,
      "logits/chosen": 0.16506488621234894,
      "logits/rejected": -0.45457974076271057,
      "logps/chosen": -1.7671860456466675,
      "logps/rejected": -2.246805429458618,
      "loss": 1.9762,
      "nll_loss": 1.9302539825439453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17671862244606018,
      "rewards/margins": 0.04796193540096283,
      "rewards/rejected": -0.224680557847023,
      "step": 154
    },
    {
      "epoch": 0.24428684003152087,
      "grad_norm": 0.3580048680305481,
      "learning_rate": 3.994495276749549e-06,
      "log_odds_chosen": 0.505012035369873,
      "log_odds_ratio": -0.4793119430541992,
      "logits/chosen": 0.06876569986343384,
      "logits/rejected": -0.03154408931732178,
      "logps/chosen": -1.786881446838379,
      "logps/rejected": -2.225484609603882,
      "loss": 1.9825,
      "nll_loss": 1.934592843055725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1786881536245346,
      "rewards/margins": 0.04386031627655029,
      "rewards/rejected": -0.2225484549999237,
      "step": 155
    },
    {
      "epoch": 0.2458628841607565,
      "grad_norm": 0.35918235778808594,
      "learning_rate": 3.994080173407612e-06,
      "log_odds_chosen": 0.2776962220668793,
      "log_odds_ratio": -0.5682939291000366,
      "logits/chosen": 0.12636655569076538,
      "logits/rejected": -0.06872869282960892,
      "logps/chosen": -1.819372296333313,
      "logps/rejected": -2.0557971000671387,
      "loss": 2.0238,
      "nll_loss": 1.967012882232666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18193721771240234,
      "rewards/margins": 0.023642467334866524,
      "rewards/rejected": -0.20557968318462372,
      "step": 156
    },
    {
      "epoch": 0.24743892828999212,
      "grad_norm": 0.3515869081020355,
      "learning_rate": 3.993650005742409e-06,
      "log_odds_chosen": 0.345825731754303,
      "log_odds_ratio": -0.5401284694671631,
      "logits/chosen": 0.05361388623714447,
      "logits/rejected": -0.005025242455303669,
      "logps/chosen": -1.7907469272613525,
      "logps/rejected": -2.0871944427490234,
      "loss": 1.9961,
      "nll_loss": 1.9421340227127075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17907467484474182,
      "rewards/margins": 0.029644787311553955,
      "rewards/rejected": -0.20871947705745697,
      "step": 157
    },
    {
      "epoch": 0.24901497241922774,
      "grad_norm": 0.3238402009010315,
      "learning_rate": 3.993204777003652e-06,
      "log_odds_chosen": 0.36766839027404785,
      "log_odds_ratio": -0.531983494758606,
      "logits/chosen": 0.0647420734167099,
      "logits/rejected": -0.17432400584220886,
      "logps/chosen": -1.720491647720337,
      "logps/rejected": -2.0343194007873535,
      "loss": 1.9217,
      "nll_loss": 1.8685407638549805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17204917967319489,
      "rewards/margins": 0.031382784247398376,
      "rewards/rejected": -0.20343194901943207,
      "step": 158
    },
    {
      "epoch": 0.25059101654846333,
      "grad_norm": 0.3346635699272156,
      "learning_rate": 3.992744490554832e-06,
      "log_odds_chosen": 0.35473060607910156,
      "log_odds_ratio": -0.537135899066925,
      "logits/chosen": 0.2205273061990738,
      "logits/rejected": -0.0024103261530399323,
      "logps/chosen": -1.7656519412994385,
      "logps/rejected": -2.068941116333008,
      "loss": 1.9832,
      "nll_loss": 1.9294898509979248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17656521499156952,
      "rewards/margins": 0.030328912660479546,
      "rewards/rejected": -0.20689409971237183,
      "step": 159
    },
    {
      "epoch": 0.25216706067769895,
      "grad_norm": 0.3138229250907898,
      "learning_rate": 3.992269149873192e-06,
      "log_odds_chosen": 0.34384748339653015,
      "log_odds_ratio": -0.5416197776794434,
      "logits/chosen": 0.021261127665638924,
      "logits/rejected": -0.09686248004436493,
      "logps/chosen": -1.7365537881851196,
      "logps/rejected": -2.0252068042755127,
      "loss": 1.9407,
      "nll_loss": 1.8865716457366943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17365536093711853,
      "rewards/margins": 0.028865298256278038,
      "rewards/rejected": -0.20252066850662231,
      "step": 160
    },
    {
      "epoch": 0.25374310480693457,
      "grad_norm": 0.3120760917663574,
      "learning_rate": 3.991778758549705e-06,
      "log_odds_chosen": 0.21335069835186005,
      "log_odds_ratio": -0.5993782877922058,
      "logits/chosen": 0.2368992120027542,
      "logits/rejected": 0.06489282846450806,
      "logps/chosen": -1.8601230382919312,
      "logps/rejected": -2.0463929176330566,
      "loss": 2.0466,
      "nll_loss": 1.9866821765899658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18601229786872864,
      "rewards/margins": 0.0186269860714674,
      "rewards/rejected": -0.2046392858028412,
      "step": 161
    },
    {
      "epoch": 0.2553191489361702,
      "grad_norm": 0.3198974132537842,
      "learning_rate": 3.9912733202890415e-06,
      "log_odds_chosen": 0.3527311384677887,
      "log_odds_ratio": -0.53630530834198,
      "logits/chosen": 0.10290198773145676,
      "logits/rejected": -0.1413993537425995,
      "logps/chosen": -1.7850815057754517,
      "logps/rejected": -2.086127519607544,
      "loss": 1.9862,
      "nll_loss": 1.9325822591781616,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17850814759731293,
      "rewards/margins": 0.03010460175573826,
      "rewards/rejected": -0.20861275494098663,
      "step": 162
    },
    {
      "epoch": 0.2568951930654058,
      "grad_norm": 0.3027884066104889,
      "learning_rate": 3.990752838909548e-06,
      "log_odds_chosen": 0.21511955559253693,
      "log_odds_ratio": -0.5959004759788513,
      "logits/chosen": 0.2015780210494995,
      "logits/rejected": -0.022823316976428032,
      "logps/chosen": -1.8808540105819702,
      "logps/rejected": -2.0652594566345215,
      "loss": 2.072,
      "nll_loss": 2.012399911880493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1880854070186615,
      "rewards/margins": 0.01844053901731968,
      "rewards/rejected": -0.20652592182159424,
      "step": 163
    },
    {
      "epoch": 0.25847123719464143,
      "grad_norm": 0.3271465599536896,
      "learning_rate": 3.990217318343213e-06,
      "log_odds_chosen": 0.5507936477661133,
      "log_odds_ratio": -0.45798903703689575,
      "logits/chosen": 0.1652592420578003,
      "logits/rejected": -0.24176833033561707,
      "logps/chosen": -1.9504748582839966,
      "logps/rejected": -2.439309597015381,
      "loss": 2.1296,
      "nll_loss": 2.083810567855835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1950474977493286,
      "rewards/margins": 0.04888347536325455,
      "rewards/rejected": -0.24393096566200256,
      "step": 164
    },
    {
      "epoch": 0.26004728132387706,
      "grad_norm": 0.31373828649520874,
      "learning_rate": 3.989666762635637e-06,
      "log_odds_chosen": 0.3172238767147064,
      "log_odds_ratio": -0.5503073334693909,
      "logits/chosen": 0.15342219173908234,
      "logits/rejected": -0.018054338172078133,
      "logps/chosen": -1.7960243225097656,
      "logps/rejected": -2.067051887512207,
      "loss": 2.0099,
      "nll_loss": 1.9548213481903076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17960244417190552,
      "rewards/margins": 0.027102762833237648,
      "rewards/rejected": -0.20670519769191742,
      "step": 165
    },
    {
      "epoch": 0.2616233254531127,
      "grad_norm": 0.30167585611343384,
      "learning_rate": 3.9891011759460056e-06,
      "log_odds_chosen": 0.3845152258872986,
      "log_odds_ratio": -0.5250376462936401,
      "logits/chosen": 0.14441440999507904,
      "logits/rejected": -0.09336265176534653,
      "logps/chosen": -1.7806988954544067,
      "logps/rejected": -2.10469913482666,
      "loss": 1.9846,
      "nll_loss": 1.9321050643920898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17806988954544067,
      "rewards/margins": 0.03240001201629639,
      "rewards/rejected": -0.21046990156173706,
      "step": 166
    },
    {
      "epoch": 0.2631993695823483,
      "grad_norm": 0.2798842787742615,
      "learning_rate": 3.988520562547057e-06,
      "log_odds_chosen": 0.4200190603733063,
      "log_odds_ratio": -0.5160478949546814,
      "logits/chosen": 0.15987613797187805,
      "logits/rejected": -0.1669200360774994,
      "logps/chosen": -1.7405807971954346,
      "logps/rejected": -2.096102714538574,
      "loss": 1.9268,
      "nll_loss": 1.875168800354004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17405809462070465,
      "rewards/margins": 0.03555220365524292,
      "rewards/rejected": -0.20961029827594757,
      "step": 167
    },
    {
      "epoch": 0.2647754137115839,
      "grad_norm": 0.30115410685539246,
      "learning_rate": 3.987924926825047e-06,
      "log_odds_chosen": 0.44322502613067627,
      "log_odds_ratio": -0.5085245370864868,
      "logits/chosen": 0.1368703693151474,
      "logits/rejected": -0.08649970591068268,
      "logps/chosen": -1.8128654956817627,
      "logps/rejected": -2.1969082355499268,
      "loss": 2.0227,
      "nll_loss": 1.971897006034851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18128655850887299,
      "rewards/margins": 0.03840426355600357,
      "rewards/rejected": -0.21969082951545715,
      "step": 168
    },
    {
      "epoch": 0.26635145784081954,
      "grad_norm": 0.29241764545440674,
      "learning_rate": 3.98731427327972e-06,
      "log_odds_chosen": 0.4207753539085388,
      "log_odds_ratio": -0.5057156085968018,
      "logits/chosen": 0.1068802922964096,
      "logits/rejected": -0.30606210231781006,
      "logps/chosen": -1.747009515762329,
      "logps/rejected": -2.1049232482910156,
      "loss": 1.9335,
      "nll_loss": 1.8828821182250977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17470093071460724,
      "rewards/margins": 0.035791389644145966,
      "rewards/rejected": -0.2104923129081726,
      "step": 169
    },
    {
      "epoch": 0.26792750197005516,
      "grad_norm": 0.28724029660224915,
      "learning_rate": 3.986688606524273e-06,
      "log_odds_chosen": 0.35461530089378357,
      "log_odds_ratio": -0.5425211787223816,
      "logits/chosen": 0.2177436798810959,
      "logits/rejected": -0.5054653882980347,
      "logps/chosen": -1.7733808755874634,
      "logps/rejected": -2.080212354660034,
      "loss": 1.9646,
      "nll_loss": 1.9103646278381348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17733806371688843,
      "rewards/margins": 0.030683180317282677,
      "rewards/rejected": -0.20802125334739685,
      "step": 170
    },
    {
      "epoch": 0.2695035460992908,
      "grad_norm": 0.2893814444541931,
      "learning_rate": 3.986047931285315e-06,
      "log_odds_chosen": 0.5224654078483582,
      "log_odds_ratio": -0.4700799584388733,
      "logits/chosen": 0.10108586400747299,
      "logits/rejected": -0.44279980659484863,
      "logps/chosen": -1.6865884065628052,
      "logps/rejected": -2.128582239151001,
      "loss": 1.8737,
      "nll_loss": 1.8267263174057007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16865883767604828,
      "rewards/margins": 0.0441993847489357,
      "rewards/rejected": -0.21285822987556458,
      "step": 171
    },
    {
      "epoch": 0.2710795902285264,
      "grad_norm": 0.2736685574054718,
      "learning_rate": 3.985392252402847e-06,
      "log_odds_chosen": 0.25993895530700684,
      "log_odds_ratio": -0.5742053389549255,
      "logits/chosen": 0.0892399251461029,
      "logits/rejected": -0.24680867791175842,
      "logps/chosen": -1.6766496896743774,
      "logps/rejected": -1.8926042318344116,
      "loss": 1.8747,
      "nll_loss": 1.8173226118087769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16766498982906342,
      "rewards/margins": 0.02159545198082924,
      "rewards/rejected": -0.18926043808460236,
      "step": 172
    },
    {
      "epoch": 0.272655634357762,
      "grad_norm": 0.289073646068573,
      "learning_rate": 3.984721574830206e-06,
      "log_odds_chosen": 0.30355799198150635,
      "log_odds_ratio": -0.5568473935127258,
      "logits/chosen": 0.1579122394323349,
      "logits/rejected": -0.27320054173469543,
      "logps/chosen": -1.78755521774292,
      "logps/rejected": -2.046647548675537,
      "loss": 1.9764,
      "nll_loss": 1.9206898212432861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1787555068731308,
      "rewards/margins": 0.02590925060212612,
      "rewards/rejected": -0.20466476678848267,
      "step": 173
    },
    {
      "epoch": 0.27423167848699764,
      "grad_norm": 0.2663167715072632,
      "learning_rate": 3.984035903634041e-06,
      "log_odds_chosen": 0.30917906761169434,
      "log_odds_ratio": -0.5573617815971375,
      "logits/chosen": 0.2104271948337555,
      "logits/rejected": -0.3466285467147827,
      "logps/chosen": -1.7617706060409546,
      "logps/rejected": -2.023843288421631,
      "loss": 1.9372,
      "nll_loss": 1.8814386129379272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17617705464363098,
      "rewards/margins": 0.026207279413938522,
      "rewards/rejected": -0.2023843377828598,
      "step": 174
    },
    {
      "epoch": 0.27580772261623326,
      "grad_norm": 0.2792463004589081,
      "learning_rate": 3.983335243994273e-06,
      "log_odds_chosen": 0.4087638258934021,
      "log_odds_ratio": -0.5167474746704102,
      "logits/chosen": 0.21963684260845184,
      "logits/rejected": -0.2987426817417145,
      "logps/chosen": -1.7534160614013672,
      "logps/rejected": -2.1018359661102295,
      "loss": 1.9423,
      "nll_loss": 1.8906673192977905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17534160614013672,
      "rewards/margins": 0.03484198823571205,
      "rewards/rejected": -0.21018359065055847,
      "step": 175
    },
    {
      "epoch": 0.2773837667454689,
      "grad_norm": 0.30124711990356445,
      "learning_rate": 3.982619601204049e-06,
      "log_odds_chosen": 0.3245881497859955,
      "log_odds_ratio": -0.5491042137145996,
      "logits/chosen": 0.1435730904340744,
      "logits/rejected": -0.3272641599178314,
      "logps/chosen": -1.8128660917282104,
      "logps/rejected": -2.0914556980133057,
      "loss": 2.001,
      "nll_loss": 1.9461297988891602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18128660321235657,
      "rewards/margins": 0.027858952060341835,
      "rewards/rejected": -0.20914556086063385,
      "step": 176
    },
    {
      "epoch": 0.2789598108747045,
      "grad_norm": 0.2849816083908081,
      "learning_rate": 3.9818889806697085e-06,
      "log_odds_chosen": 0.3977530002593994,
      "log_odds_ratio": -0.5233108401298523,
      "logits/chosen": 0.2636592984199524,
      "logits/rejected": -0.3068069815635681,
      "logps/chosen": -1.8528499603271484,
      "logps/rejected": -2.1990413665771484,
      "loss": 2.0173,
      "nll_loss": 1.964940071105957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18528500199317932,
      "rewards/margins": 0.03461914509534836,
      "rewards/rejected": -0.2199041247367859,
      "step": 177
    },
    {
      "epoch": 0.2805358550039401,
      "grad_norm": 0.27957502007484436,
      "learning_rate": 3.98114338791074e-06,
      "log_odds_chosen": 0.49340713024139404,
      "log_odds_ratio": -0.4914700388908386,
      "logits/chosen": 0.13450200855731964,
      "logits/rejected": -0.340486079454422,
      "logps/chosen": -1.7227312326431274,
      "logps/rejected": -2.1473944187164307,
      "loss": 1.915,
      "nll_loss": 1.865891933441162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17227312922477722,
      "rewards/margins": 0.04246632009744644,
      "rewards/rejected": -0.21473942697048187,
      "step": 178
    },
    {
      "epoch": 0.28211189913317575,
      "grad_norm": 0.2686019837856293,
      "learning_rate": 3.980382828559742e-06,
      "log_odds_chosen": 0.39518705010414124,
      "log_odds_ratio": -0.5202585458755493,
      "logits/chosen": 0.14406242966651917,
      "logits/rejected": -0.3577483594417572,
      "logps/chosen": -1.7618299722671509,
      "logps/rejected": -2.098731279373169,
      "loss": 1.9315,
      "nll_loss": 1.8795194625854492,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17618300020694733,
      "rewards/margins": 0.033690109848976135,
      "rewards/rejected": -0.20987312495708466,
      "step": 179
    },
    {
      "epoch": 0.28368794326241137,
      "grad_norm": 0.28153085708618164,
      "learning_rate": 3.9796073083623774e-06,
      "log_odds_chosen": 0.3710815906524658,
      "log_odds_ratio": -0.5394359230995178,
      "logits/chosen": 0.16021382808685303,
      "logits/rejected": -0.4299715459346771,
      "logps/chosen": -1.836319088935852,
      "logps/rejected": -2.1562142372131348,
      "loss": 2.0036,
      "nll_loss": 1.9496216773986816,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18363192677497864,
      "rewards/margins": 0.031989507377147675,
      "rewards/rejected": -0.21562142670154572,
      "step": 180
    },
    {
      "epoch": 0.285263987391647,
      "grad_norm": 0.2878672182559967,
      "learning_rate": 3.978816833177329e-06,
      "log_odds_chosen": 0.3781207203865051,
      "log_odds_ratio": -0.5273905992507935,
      "logits/chosen": 0.2487175166606903,
      "logits/rejected": -0.232444629073143,
      "logps/chosen": -1.759475588798523,
      "logps/rejected": -2.082449436187744,
      "loss": 1.9353,
      "nll_loss": 1.8825562000274658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17594757676124573,
      "rewards/margins": 0.03229738026857376,
      "rewards/rejected": -0.2082449495792389,
      "step": 181
    },
    {
      "epoch": 0.2868400315208826,
      "grad_norm": 0.28296828269958496,
      "learning_rate": 3.978011408976261e-06,
      "log_odds_chosen": 0.24777647852897644,
      "log_odds_ratio": -0.5824975371360779,
      "logits/chosen": 0.20205409824848175,
      "logits/rejected": -0.42155444622039795,
      "logps/chosen": -1.77714204788208,
      "logps/rejected": -1.985145092010498,
      "loss": 1.9669,
      "nll_loss": 1.9086382389068604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17771419882774353,
      "rewards/margins": 0.020800314843654633,
      "rewards/rejected": -0.19851452112197876,
      "step": 182
    },
    {
      "epoch": 0.28841607565011823,
      "grad_norm": 0.29371410608291626,
      "learning_rate": 3.9771910418437674e-06,
      "log_odds_chosen": 0.32709380984306335,
      "log_odds_ratio": -0.544011116027832,
      "logits/chosen": 0.20650595426559448,
      "logits/rejected": -0.11976965516805649,
      "logps/chosen": -1.8121384382247925,
      "logps/rejected": -2.093899726867676,
      "loss": 1.9856,
      "nll_loss": 1.9311554431915283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1812138557434082,
      "rewards/margins": 0.028176117688417435,
      "rewards/rejected": -0.20938995480537415,
      "step": 183
    },
    {
      "epoch": 0.2899921197793538,
      "grad_norm": 0.28328511118888855,
      "learning_rate": 3.976355737977332e-06,
      "log_odds_chosen": 0.39620110392570496,
      "log_odds_ratio": -0.5227944254875183,
      "logits/chosen": 0.21991243958473206,
      "logits/rejected": -0.3309701681137085,
      "logps/chosen": -1.7577619552612305,
      "logps/rejected": -2.0953731536865234,
      "loss": 1.939,
      "nll_loss": 1.8866724967956543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1757761836051941,
      "rewards/margins": 0.03376111388206482,
      "rewards/rejected": -0.2095372974872589,
      "step": 184
    },
    {
      "epoch": 0.2915681639085894,
      "grad_norm": 0.2596893608570099,
      "learning_rate": 3.975505503687274e-06,
      "log_odds_chosen": 0.3184034824371338,
      "log_odds_ratio": -0.5498912930488586,
      "logits/chosen": 0.26362285017967224,
      "logits/rejected": -0.1583450883626938,
      "logps/chosen": -1.7432103157043457,
      "logps/rejected": -2.0135068893432617,
      "loss": 1.9195,
      "nll_loss": 1.864485263824463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1743210256099701,
      "rewards/margins": 0.027029668912291527,
      "rewards/rejected": -0.20135070383548737,
      "step": 185
    },
    {
      "epoch": 0.29314420803782504,
      "grad_norm": 0.25459468364715576,
      "learning_rate": 3.974640345396708e-06,
      "log_odds_chosen": 0.4846782386302948,
      "log_odds_ratio": -0.4870211184024811,
      "logits/chosen": 0.1184130385518074,
      "logits/rejected": -0.6584069132804871,
      "logps/chosen": -1.7136808633804321,
      "logps/rejected": -2.1256215572357178,
      "loss": 1.8877,
      "nll_loss": 1.8389503955841064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1713680922985077,
      "rewards/margins": 0.041194066405296326,
      "rewards/rejected": -0.21256215870380402,
      "step": 186
    },
    {
      "epoch": 0.29472025216706066,
      "grad_norm": 0.2585580050945282,
      "learning_rate": 3.9737602696414925e-06,
      "log_odds_chosen": 0.17746226489543915,
      "log_odds_ratio": -0.6186371445655823,
      "logits/chosen": 0.2338634729385376,
      "logits/rejected": -0.4608267545700073,
      "logps/chosen": -1.804581880569458,
      "logps/rejected": -1.954529881477356,
      "loss": 1.9595,
      "nll_loss": 1.897607445716858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1804581880569458,
      "rewards/margins": 0.014994792640209198,
      "rewards/rejected": -0.1954529881477356,
      "step": 187
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 0.2511560618877411,
      "learning_rate": 3.972865283070179e-06,
      "log_odds_chosen": 0.40657275915145874,
      "log_odds_ratio": -0.5161192417144775,
      "logits/chosen": 0.18840941786766052,
      "logits/rejected": -0.6009777188301086,
      "logps/chosen": -1.7036168575286865,
      "logps/rejected": -2.0506670475006104,
      "loss": 1.8859,
      "nll_loss": 1.8342941999435425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1703616976737976,
      "rewards/margins": 0.0347050204873085,
      "rewards/rejected": -0.2050667107105255,
      "step": 188
    },
    {
      "epoch": 0.2978723404255319,
      "grad_norm": 0.26080071926116943,
      "learning_rate": 3.971955392443965e-06,
      "log_odds_chosen": 0.3762245774269104,
      "log_odds_ratio": -0.528282642364502,
      "logits/chosen": 0.2339557558298111,
      "logits/rejected": -0.35206758975982666,
      "logps/chosen": -1.735405683517456,
      "logps/rejected": -2.0585732460021973,
      "loss": 1.905,
      "nll_loss": 1.852203607559204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17354057729244232,
      "rewards/margins": 0.03231675922870636,
      "rewards/rejected": -0.2058573216199875,
      "step": 189
    },
    {
      "epoch": 0.2994483845547675,
      "grad_norm": 0.26392868161201477,
      "learning_rate": 3.971030604636637e-06,
      "log_odds_chosen": 0.2363440990447998,
      "log_odds_ratio": -0.5862751007080078,
      "logits/chosen": 0.2692638039588928,
      "logits/rejected": -0.36195865273475647,
      "logps/chosen": -1.7536004781723022,
      "logps/rejected": -1.9518376588821411,
      "loss": 1.9225,
      "nll_loss": 1.8638556003570557,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1753600686788559,
      "rewards/margins": 0.019823705777525902,
      "rewards/rejected": -0.19518375396728516,
      "step": 190
    },
    {
      "epoch": 0.30102442868400314,
      "grad_norm": 0.2797010540962219,
      "learning_rate": 3.970090926634526e-06,
      "log_odds_chosen": 0.31876862049102783,
      "log_odds_ratio": -0.5496628284454346,
      "logits/chosen": 0.33125385642051697,
      "logits/rejected": -0.578338623046875,
      "logps/chosen": -1.8113794326782227,
      "logps/rejected": -2.0837459564208984,
      "loss": 1.9864,
      "nll_loss": 1.931410789489746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18113794922828674,
      "rewards/margins": 0.027236640453338623,
      "rewards/rejected": -0.20837458968162537,
      "step": 191
    },
    {
      "epoch": 0.30260047281323876,
      "grad_norm": 0.2825038731098175,
      "learning_rate": 3.9691363655364526e-06,
      "log_odds_chosen": 0.3615967631340027,
      "log_odds_ratio": -0.5366310477256775,
      "logits/chosen": 0.23022601008415222,
      "logits/rejected": -0.39873024821281433,
      "logps/chosen": -1.787469744682312,
      "logps/rejected": -2.097710609436035,
      "loss": 1.9624,
      "nll_loss": 1.9087271690368652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1787469983100891,
      "rewards/margins": 0.031024089083075523,
      "rewards/rejected": -0.209771066904068,
      "step": 192
    },
    {
      "epoch": 0.3041765169424744,
      "grad_norm": 0.26274222135543823,
      "learning_rate": 3.968166928553666e-06,
      "log_odds_chosen": 0.3207397162914276,
      "log_odds_ratio": -0.5503286123275757,
      "logits/chosen": 0.2751014530658722,
      "logits/rejected": -0.5721142888069153,
      "logps/chosen": -1.731804370880127,
      "logps/rejected": -2.001689910888672,
      "loss": 1.9222,
      "nll_loss": 1.8671228885650635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1731804460287094,
      "rewards/margins": 0.026988547295331955,
      "rewards/rejected": -0.20016898214817047,
      "step": 193
    },
    {
      "epoch": 0.30575256107171,
      "grad_norm": 0.26265692710876465,
      "learning_rate": 3.967182623009804e-06,
      "log_odds_chosen": 0.4414345324039459,
      "log_odds_ratio": -0.5021454691886902,
      "logits/chosen": 0.22120928764343262,
      "logits/rejected": -0.43845516443252563,
      "logps/chosen": -1.760418176651001,
      "logps/rejected": -2.14136004447937,
      "loss": 1.9192,
      "nll_loss": 1.8690216541290283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.176041841506958,
      "rewards/margins": 0.0380941778421402,
      "rewards/rejected": -0.2141360193490982,
      "step": 194
    },
    {
      "epoch": 0.3073286052009456,
      "grad_norm": 0.30240899324417114,
      "learning_rate": 3.966183456340821e-06,
      "log_odds_chosen": 0.21254514157772064,
      "log_odds_ratio": -0.5945923328399658,
      "logits/chosen": 0.27597901225090027,
      "logits/rejected": -0.4168132543563843,
      "logps/chosen": -1.7859089374542236,
      "logps/rejected": -1.9644159078598022,
      "loss": 1.96,
      "nll_loss": 1.9005576372146606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17859089374542236,
      "rewards/margins": 0.017850710079073906,
      "rewards/rejected": -0.19644160568714142,
      "step": 195
    },
    {
      "epoch": 0.30890464933018125,
      "grad_norm": 0.28170743584632874,
      "learning_rate": 3.965169436094947e-06,
      "log_odds_chosen": 0.38190627098083496,
      "log_odds_ratio": -0.5301341414451599,
      "logits/chosen": 0.2661678194999695,
      "logits/rejected": -0.5325687527656555,
      "logps/chosen": -1.6825157403945923,
      "logps/rejected": -2.00687313079834,
      "loss": 1.8654,
      "nll_loss": 1.8123762607574463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16825160384178162,
      "rewards/margins": 0.03243573009967804,
      "rewards/rejected": -0.20068733394145966,
      "step": 196
    },
    {
      "epoch": 0.31048069345941687,
      "grad_norm": 0.2710944414138794,
      "learning_rate": 3.964140569932618e-06,
      "log_odds_chosen": 0.3068499267101288,
      "log_odds_ratio": -0.5730952620506287,
      "logits/chosen": 0.2248048484325409,
      "logits/rejected": -0.7354637980461121,
      "logps/chosen": -1.806632161140442,
      "logps/rejected": -2.067098379135132,
      "loss": 1.9701,
      "nll_loss": 1.912825345993042,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18066321313381195,
      "rewards/margins": 0.026046616956591606,
      "rewards/rejected": -0.2067098468542099,
      "step": 197
    },
    {
      "epoch": 0.3120567375886525,
      "grad_norm": 0.31186628341674805,
      "learning_rate": 3.9630968656264285e-06,
      "log_odds_chosen": 0.44008272886276245,
      "log_odds_ratio": -0.503641664981842,
      "logits/chosen": 0.24553100764751434,
      "logits/rejected": -0.38101547956466675,
      "logps/chosen": -1.7257674932479858,
      "logps/rejected": -2.101093292236328,
      "loss": 1.8853,
      "nll_loss": 1.8349802494049072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17257672548294067,
      "rewards/margins": 0.03753257542848587,
      "rewards/rejected": -0.21010930836200714,
      "step": 198
    },
    {
      "epoch": 0.3136327817178881,
      "grad_norm": 0.2788161337375641,
      "learning_rate": 3.962038331061065e-06,
      "log_odds_chosen": 0.17320865392684937,
      "log_odds_ratio": -0.622861385345459,
      "logits/chosen": 0.321949303150177,
      "logits/rejected": -0.5195744037628174,
      "logps/chosen": -1.7807538509368896,
      "logps/rejected": -1.9327521324157715,
      "loss": 1.9626,
      "nll_loss": 1.900266408920288,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1780754029750824,
      "rewards/margins": 0.015199808403849602,
      "rewards/rejected": -0.19327519834041595,
      "step": 199
    },
    {
      "epoch": 0.31520882584712373,
      "grad_norm": 0.27439412474632263,
      "learning_rate": 3.96096497423325e-06,
      "log_odds_chosen": 0.2597573399543762,
      "log_odds_ratio": -0.5759460926055908,
      "logits/chosen": 0.36237332224845886,
      "logits/rejected": -0.16011810302734375,
      "logps/chosen": -1.6894915103912354,
      "logps/rejected": -1.9068094491958618,
      "loss": 1.8906,
      "nll_loss": 1.8330268859863281,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.168949156999588,
      "rewards/margins": 0.021731778979301453,
      "rewards/rejected": -0.19068092107772827,
      "step": 200
    },
    {
      "epoch": 0.31678486997635935,
      "grad_norm": 0.25059738755226135,
      "learning_rate": 3.959876803251684e-06,
      "log_odds_chosen": 0.39733871817588806,
      "log_odds_ratio": -0.5207474231719971,
      "logits/chosen": 0.21183565258979797,
      "logits/rejected": -0.4376421570777893,
      "logps/chosen": -1.7023674249649048,
      "logps/rejected": -2.038724184036255,
      "loss": 1.8566,
      "nll_loss": 1.8045035600662231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1702367514371872,
      "rewards/margins": 0.033635687083005905,
      "rewards/rejected": -0.2038724273443222,
      "step": 201
    },
    {
      "epoch": 0.31836091410559497,
      "grad_norm": 0.25566282868385315,
      "learning_rate": 3.958773826336977e-06,
      "log_odds_chosen": 0.2807323634624481,
      "log_odds_ratio": -0.564405083656311,
      "logits/chosen": 0.31915491819381714,
      "logits/rejected": -0.293272465467453,
      "logps/chosen": -1.7425774335861206,
      "logps/rejected": -1.9793710708618164,
      "loss": 1.9173,
      "nll_loss": 1.8608673810958862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1742577701807022,
      "rewards/margins": 0.0236793402582407,
      "rewards/rejected": -0.19793710112571716,
      "step": 202
    },
    {
      "epoch": 0.3199369582348306,
      "grad_norm": 0.26755860447883606,
      "learning_rate": 3.957656051821592e-06,
      "log_odds_chosen": 0.2489960789680481,
      "log_odds_ratio": -0.5815601348876953,
      "logits/chosen": 0.3447924852371216,
      "logits/rejected": -0.6091679930686951,
      "logps/chosen": -1.7780265808105469,
      "logps/rejected": -1.9914788007736206,
      "loss": 1.9267,
      "nll_loss": 1.868532419204712,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1778026670217514,
      "rewards/margins": 0.02134522795677185,
      "rewards/rejected": -0.19914788007736206,
      "step": 203
    },
    {
      "epoch": 0.3215130023640662,
      "grad_norm": 0.27641505002975464,
      "learning_rate": 3.956523488149783e-06,
      "log_odds_chosen": 0.31930261850357056,
      "log_odds_ratio": -0.5506942272186279,
      "logits/chosen": 0.3657802641391754,
      "logits/rejected": -0.375068724155426,
      "logps/chosen": -1.8819518089294434,
      "logps/rejected": -2.158445358276367,
      "loss": 2.0406,
      "nll_loss": 1.9855563640594482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18819519877433777,
      "rewards/margins": 0.02764933556318283,
      "rewards/rejected": -0.2158445119857788,
      "step": 204
    },
    {
      "epoch": 0.32308904649330183,
      "grad_norm": 0.27057579159736633,
      "learning_rate": 3.9553761438775285e-06,
      "log_odds_chosen": 0.5523542165756226,
      "log_odds_ratio": -0.4604877531528473,
      "logits/chosen": 0.22423981130123138,
      "logits/rejected": -0.40424689650535583,
      "logps/chosen": -1.6142833232879639,
      "logps/rejected": -2.081634521484375,
      "loss": 1.7752,
      "nll_loss": 1.7291845083236694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1614283323287964,
      "rewards/margins": 0.04673512279987335,
      "rewards/rejected": -0.20816347002983093,
      "step": 205
    },
    {
      "epoch": 0.32466509062253746,
      "grad_norm": 0.24270053207874298,
      "learning_rate": 3.954214027672465e-06,
      "log_odds_chosen": 0.49918606877326965,
      "log_odds_ratio": -0.48198771476745605,
      "logits/chosen": 0.22353754937648773,
      "logits/rejected": -0.8031458854675293,
      "logps/chosen": -1.7024301290512085,
      "logps/rejected": -2.131436347961426,
      "loss": 1.8545,
      "nll_loss": 1.8062725067138672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1702430099248886,
      "rewards/margins": 0.04290061816573143,
      "rewards/rejected": -0.21314363181591034,
      "step": 206
    },
    {
      "epoch": 0.3262411347517731,
      "grad_norm": 0.2744021713733673,
      "learning_rate": 3.953037148313825e-06,
      "log_odds_chosen": 0.2930293679237366,
      "log_odds_ratio": -0.5630696415901184,
      "logits/chosen": 0.3285022974014282,
      "logits/rejected": -0.46074235439300537,
      "logps/chosen": -1.6958521604537964,
      "logps/rejected": -1.9453150033950806,
      "loss": 1.8815,
      "nll_loss": 1.8252310752868652,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1695852279663086,
      "rewards/margins": 0.024946292862296104,
      "rewards/rejected": -0.19453151524066925,
      "step": 207
    },
    {
      "epoch": 0.32781717888100864,
      "grad_norm": 0.28513532876968384,
      "learning_rate": 3.951845514692371e-06,
      "log_odds_chosen": 0.3969423770904541,
      "log_odds_ratio": -0.5238114595413208,
      "logits/chosen": 0.25184959173202515,
      "logits/rejected": -0.3663902282714844,
      "logps/chosen": -1.7550077438354492,
      "logps/rejected": -2.095698833465576,
      "loss": 1.9278,
      "nll_loss": 1.875378131866455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1755007952451706,
      "rewards/margins": 0.03406910225749016,
      "rewards/rejected": -0.20956988632678986,
      "step": 208
    },
    {
      "epoch": 0.32939322301024426,
      "grad_norm": 0.2841278910636902,
      "learning_rate": 3.950639135810325e-06,
      "log_odds_chosen": 0.32432445883750916,
      "log_odds_ratio": -0.5494241118431091,
      "logits/chosen": 0.3648757338523865,
      "logits/rejected": -0.3396746516227722,
      "logps/chosen": -1.87786066532135,
      "logps/rejected": -2.157219171524048,
      "loss": 2.0224,
      "nll_loss": 1.967415452003479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1877860575914383,
      "rewards/margins": 0.027935855090618134,
      "rewards/rejected": -0.21572190523147583,
      "step": 209
    },
    {
      "epoch": 0.3309692671394799,
      "grad_norm": 0.2702895998954773,
      "learning_rate": 3.9494180207813044e-06,
      "log_odds_chosen": 0.25352904200553894,
      "log_odds_ratio": -0.5853363275527954,
      "logits/chosen": 0.3584989011287689,
      "logits/rejected": -0.6828628182411194,
      "logps/chosen": -1.7445666790008545,
      "logps/rejected": -1.9578887224197388,
      "loss": 1.9155,
      "nll_loss": 1.8569811582565308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17445670068264008,
      "rewards/margins": 0.02133220247924328,
      "rewards/rejected": -0.1957888901233673,
      "step": 210
    },
    {
      "epoch": 0.3325453112687155,
      "grad_norm": 0.2695715129375458,
      "learning_rate": 3.948182178830249e-06,
      "log_odds_chosen": 0.3178305923938751,
      "log_odds_ratio": -0.5667225122451782,
      "logits/chosen": 0.3227792978286743,
      "logits/rejected": -0.824029266834259,
      "logps/chosen": -1.7716305255889893,
      "logps/rejected": -2.047945737838745,
      "loss": 1.9257,
      "nll_loss": 1.869060754776001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17716306447982788,
      "rewards/margins": 0.027631523087620735,
      "rewards/rejected": -0.20479457080364227,
      "step": 211
    },
    {
      "epoch": 0.3341213553979511,
      "grad_norm": 0.29492443799972534,
      "learning_rate": 3.9469316192933545e-06,
      "log_odds_chosen": 0.5440890192985535,
      "log_odds_ratio": -0.4644208550453186,
      "logits/chosen": 0.2088673710823059,
      "logits/rejected": -0.4241965413093567,
      "logps/chosen": -1.629172444343567,
      "logps/rejected": -2.0890142917633057,
      "loss": 1.798,
      "nll_loss": 1.751549482345581,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16291725635528564,
      "rewards/margins": 0.0459841787815094,
      "rewards/rejected": -0.20890143513679504,
      "step": 212
    },
    {
      "epoch": 0.33569739952718675,
      "grad_norm": 0.26417070627212524,
      "learning_rate": 3.945666351618001e-06,
      "log_odds_chosen": 0.28261005878448486,
      "log_odds_ratio": -0.5651870369911194,
      "logits/chosen": 0.3082667589187622,
      "logits/rejected": -0.4787600636482239,
      "logps/chosen": -1.6829228401184082,
      "logps/rejected": -1.9201278686523438,
      "loss": 1.8496,
      "nll_loss": 1.7930916547775269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16829228401184082,
      "rewards/margins": 0.02372048795223236,
      "rewards/rejected": -0.19201278686523438,
      "step": 213
    },
    {
      "epoch": 0.33727344365642237,
      "grad_norm": 0.2696978449821472,
      "learning_rate": 3.9443863853626825e-06,
      "log_odds_chosen": 0.3259792923927307,
      "log_odds_ratio": -0.5532336831092834,
      "logits/chosen": 0.34843507409095764,
      "logits/rejected": -0.5844205021858215,
      "logps/chosen": -1.6465122699737549,
      "logps/rejected": -1.9122709035873413,
      "loss": 1.8357,
      "nll_loss": 1.7804213762283325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16465122997760773,
      "rewards/margins": 0.02657586894929409,
      "rewards/rejected": -0.19122707843780518,
      "step": 214
    },
    {
      "epoch": 0.338849487785658,
      "grad_norm": 0.2630347013473511,
      "learning_rate": 3.943091730196931e-06,
      "log_odds_chosen": 0.2849031388759613,
      "log_odds_ratio": -0.5690271258354187,
      "logits/chosen": 0.39357054233551025,
      "logits/rejected": -0.36922866106033325,
      "logps/chosen": -1.6450810432434082,
      "logps/rejected": -1.8856078386306763,
      "loss": 1.8206,
      "nll_loss": 1.7637048959732056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16450810432434082,
      "rewards/margins": 0.024052688851952553,
      "rewards/rejected": -0.18856079876422882,
      "step": 215
    },
    {
      "epoch": 0.3404255319148936,
      "grad_norm": 0.2966042160987854,
      "learning_rate": 3.941782395901249e-06,
      "log_odds_chosen": 0.35482144355773926,
      "log_odds_ratio": -0.5378621220588684,
      "logits/chosen": 0.27690693736076355,
      "logits/rejected": -0.5179506540298462,
      "logps/chosen": -1.7256001234054565,
      "logps/rejected": -2.026144027709961,
      "loss": 1.9001,
      "nll_loss": 1.846313714981079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17256002128124237,
      "rewards/margins": 0.030054409056901932,
      "rewards/rejected": -0.2026144117116928,
      "step": 216
    },
    {
      "epoch": 0.34200157604412923,
      "grad_norm": 0.271468847990036,
      "learning_rate": 3.940458392367032e-06,
      "log_odds_chosen": 0.32546529173851013,
      "log_odds_ratio": -0.5480217933654785,
      "logits/chosen": 0.30692675709724426,
      "logits/rejected": -0.7998979091644287,
      "logps/chosen": -1.775266170501709,
      "logps/rejected": -2.052006959915161,
      "loss": 1.9171,
      "nll_loss": 1.862282633781433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17752662301063538,
      "rewards/margins": 0.02767406962811947,
      "rewards/rejected": -0.2052006870508194,
      "step": 217
    },
    {
      "epoch": 0.34357762017336485,
      "grad_norm": 0.27401667833328247,
      "learning_rate": 3.939119729596493e-06,
      "log_odds_chosen": 0.3677333891391754,
      "log_odds_ratio": -0.5295292735099792,
      "logits/chosen": 0.36077094078063965,
      "logits/rejected": -0.6209971904754639,
      "logps/chosen": -1.7778162956237793,
      "logps/rejected": -2.092672109603882,
      "loss": 1.9304,
      "nll_loss": 1.8774876594543457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1777816265821457,
      "rewards/margins": 0.031485576182603836,
      "rewards/rejected": -0.20926719903945923,
      "step": 218
    },
    {
      "epoch": 0.34515366430260047,
      "grad_norm": 0.26384812593460083,
      "learning_rate": 3.93776641770259e-06,
      "log_odds_chosen": 0.2971005141735077,
      "log_odds_ratio": -0.562300980091095,
      "logits/chosen": 0.3340204954147339,
      "logits/rejected": -0.6690645217895508,
      "logps/chosen": -1.745128870010376,
      "logps/rejected": -1.998238205909729,
      "loss": 1.8973,
      "nll_loss": 1.841081142425537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17451287806034088,
      "rewards/margins": 0.02531094290316105,
      "rewards/rejected": -0.19982382655143738,
      "step": 219
    },
    {
      "epoch": 0.3467297084318361,
      "grad_norm": 0.2645641565322876,
      "learning_rate": 3.93639846690895e-06,
      "log_odds_chosen": 0.43852394819259644,
      "log_odds_ratio": -0.5020437240600586,
      "logits/chosen": 0.3577096164226532,
      "logits/rejected": -0.8902232646942139,
      "logps/chosen": -1.7646898031234741,
      "logps/rejected": -2.1421992778778076,
      "loss": 1.9106,
      "nll_loss": 1.8603509664535522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17646898329257965,
      "rewards/margins": 0.0377509668469429,
      "rewards/rejected": -0.21421995759010315,
      "step": 220
    },
    {
      "epoch": 0.3483057525610717,
      "grad_norm": 0.26245489716529846,
      "learning_rate": 3.9350158875497855e-06,
      "log_odds_chosen": 0.5075678825378418,
      "log_odds_ratio": -0.47529348731040955,
      "logits/chosen": 0.36207839846611023,
      "logits/rejected": -0.7290895581245422,
      "logps/chosen": -1.6745859384536743,
      "logps/rejected": -2.103607654571533,
      "loss": 1.8345,
      "nll_loss": 1.7869353294372559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16745859384536743,
      "rewards/margins": 0.04290216416120529,
      "rewards/rejected": -0.21036075055599213,
      "step": 221
    },
    {
      "epoch": 0.34988179669030733,
      "grad_norm": 0.25845250487327576,
      "learning_rate": 3.933618690069824e-06,
      "log_odds_chosen": 0.46901246905326843,
      "log_odds_ratio": -0.4901207983493805,
      "logits/chosen": 0.22216065227985382,
      "logits/rejected": -0.667863667011261,
      "logps/chosen": -1.6221312284469604,
      "logps/rejected": -2.011317491531372,
      "loss": 1.7945,
      "nll_loss": 1.7455064058303833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16221313178539276,
      "rewards/margins": 0.0389186255633831,
      "rewards/rejected": -0.20113176107406616,
      "step": 222
    },
    {
      "epoch": 0.35145784081954295,
      "grad_norm": 0.26442599296569824,
      "learning_rate": 3.932206885024226e-06,
      "log_odds_chosen": 0.2337113320827484,
      "log_odds_ratio": -0.5857836604118347,
      "logits/chosen": 0.27757957577705383,
      "logits/rejected": -0.6246755123138428,
      "logps/chosen": -1.7342665195465088,
      "logps/rejected": -1.9312242269515991,
      "loss": 1.8784,
      "nll_loss": 1.8198186159133911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17342665791511536,
      "rewards/margins": 0.019695764407515526,
      "rewards/rejected": -0.19312243163585663,
      "step": 223
    },
    {
      "epoch": 0.3530338849487786,
      "grad_norm": 0.24192552268505096,
      "learning_rate": 3.930780483078502e-06,
      "log_odds_chosen": 0.3615175485610962,
      "log_odds_ratio": -0.5337653160095215,
      "logits/chosen": 0.24594762921333313,
      "logits/rejected": -0.6662352085113525,
      "logps/chosen": -1.6547470092773438,
      "logps/rejected": -1.9560520648956299,
      "loss": 1.8248,
      "nll_loss": 1.7714567184448242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16547471284866333,
      "rewards/margins": 0.03013051114976406,
      "rewards/rejected": -0.19560521841049194,
      "step": 224
    },
    {
      "epoch": 0.3546099290780142,
      "grad_norm": 0.2694976329803467,
      "learning_rate": 3.92933949500844e-06,
      "log_odds_chosen": 0.3964681923389435,
      "log_odds_ratio": -0.5309619307518005,
      "logits/chosen": 0.34370604157447815,
      "logits/rejected": -0.6380555033683777,
      "logps/chosen": -1.6309007406234741,
      "logps/rejected": -1.9560682773590088,
      "loss": 1.8016,
      "nll_loss": 1.7484827041625977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1630900502204895,
      "rewards/margins": 0.03251676633954048,
      "rewards/rejected": -0.19560682773590088,
      "step": 225
    },
    {
      "epoch": 0.3561859732072498,
      "grad_norm": 0.256059855222702,
      "learning_rate": 3.9278839317000155e-06,
      "log_odds_chosen": 0.4125687777996063,
      "log_odds_ratio": -0.513237714767456,
      "logits/chosen": 0.3524473309516907,
      "logits/rejected": -0.6497681736946106,
      "logps/chosen": -1.6583887338638306,
      "logps/rejected": -2.0051815509796143,
      "loss": 1.8159,
      "nll_loss": 1.7646095752716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16583886742591858,
      "rewards/margins": 0.03467928618192673,
      "rewards/rejected": -0.2005181461572647,
      "step": 226
    },
    {
      "epoch": 0.35776201733648544,
      "grad_norm": 0.2596084177494049,
      "learning_rate": 3.926413804149314e-06,
      "log_odds_chosen": 0.1289360225200653,
      "log_odds_ratio": -0.6390258073806763,
      "logits/chosen": 0.33115288615226746,
      "logits/rejected": -0.6374678611755371,
      "logps/chosen": -1.7433453798294067,
      "logps/rejected": -1.851163625717163,
      "loss": 1.899,
      "nll_loss": 1.8351025581359863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1743345409631729,
      "rewards/margins": 0.010781830176711082,
      "rewards/rejected": -0.18511638045310974,
      "step": 227
    },
    {
      "epoch": 0.35933806146572106,
      "grad_norm": 0.26425355672836304,
      "learning_rate": 3.924929123462447e-06,
      "log_odds_chosen": 0.2934200167655945,
      "log_odds_ratio": -0.5636212229728699,
      "logits/chosen": 0.387963205575943,
      "logits/rejected": -0.7065924406051636,
      "logps/chosen": -1.7703977823257446,
      "logps/rejected": -2.0198216438293457,
      "loss": 1.9186,
      "nll_loss": 1.8622859716415405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17703978717327118,
      "rewards/margins": 0.024942385032773018,
      "rewards/rejected": -0.20198215544223785,
      "step": 228
    },
    {
      "epoch": 0.3609141055949567,
      "grad_norm": 0.27313873171806335,
      "learning_rate": 3.923429900855468e-06,
      "log_odds_chosen": 0.21642562747001648,
      "log_odds_ratio": -0.5981402397155762,
      "logits/chosen": 0.3475606143474579,
      "logits/rejected": -0.4283681809902191,
      "logps/chosen": -1.8448374271392822,
      "logps/rejected": -2.0277938842773438,
      "loss": 1.9955,
      "nll_loss": 1.9356396198272705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18448375165462494,
      "rewards/margins": 0.01829562336206436,
      "rewards/rejected": -0.2027793526649475,
      "step": 229
    },
    {
      "epoch": 0.3624901497241923,
      "grad_norm": 0.24974492192268372,
      "learning_rate": 3.921916147654287e-06,
      "log_odds_chosen": 0.4641319811344147,
      "log_odds_ratio": -0.4937167465686798,
      "logits/chosen": 0.2954743504524231,
      "logits/rejected": -0.7521487474441528,
      "logps/chosen": -1.6764249801635742,
      "logps/rejected": -2.066699743270874,
      "loss": 1.8146,
      "nll_loss": 1.7652264833450317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1676425188779831,
      "rewards/margins": 0.03902747482061386,
      "rewards/rejected": -0.20666998624801636,
      "step": 230
    },
    {
      "epoch": 0.3640661938534279,
      "grad_norm": 0.2573936879634857,
      "learning_rate": 3.920387875294588e-06,
      "log_odds_chosen": 0.30775612592697144,
      "log_odds_ratio": -0.5542665719985962,
      "logits/chosen": 0.32833412289619446,
      "logits/rejected": -0.5938704609870911,
      "logps/chosen": -1.6527678966522217,
      "logps/rejected": -1.90794038772583,
      "loss": 1.8093,
      "nll_loss": 1.753854751586914,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16527679562568665,
      "rewards/margins": 0.02551724575459957,
      "rewards/rejected": -0.19079405069351196,
      "step": 231
    },
    {
      "epoch": 0.3656422379826635,
      "grad_norm": 0.25298821926116943,
      "learning_rate": 3.918845095321737e-06,
      "log_odds_chosen": 0.28933507204055786,
      "log_odds_ratio": -0.5692440271377563,
      "logits/chosen": 0.2770174443721771,
      "logits/rejected": -0.8114198446273804,
      "logps/chosen": -1.6806954145431519,
      "logps/rejected": -1.9158856868743896,
      "loss": 1.829,
      "nll_loss": 1.7720324993133545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16806954145431519,
      "rewards/margins": 0.023519039154052734,
      "rewards/rejected": -0.19158856570720673,
      "step": 232
    },
    {
      "epoch": 0.3672182821118991,
      "grad_norm": 0.2757298946380615,
      "learning_rate": 3.9172878193907004e-06,
      "log_odds_chosen": 0.32267412543296814,
      "log_odds_ratio": -0.5543497204780579,
      "logits/chosen": 0.23460762202739716,
      "logits/rejected": -0.6888258457183838,
      "logps/chosen": -1.7551041841506958,
      "logps/rejected": -2.0310540199279785,
      "loss": 1.9168,
      "nll_loss": 1.8614091873168945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17551042139530182,
      "rewards/margins": 0.027594992890954018,
      "rewards/rejected": -0.20310541987419128,
      "step": 233
    },
    {
      "epoch": 0.36879432624113473,
      "grad_norm": 0.25875324010849,
      "learning_rate": 3.915716059265955e-06,
      "log_odds_chosen": 0.34329843521118164,
      "log_odds_ratio": -0.5443562269210815,
      "logits/chosen": 0.3021478056907654,
      "logits/rejected": -0.7619258761405945,
      "logps/chosen": -1.7081949710845947,
      "logps/rejected": -1.9994511604309082,
      "loss": 1.8579,
      "nll_loss": 1.8034164905548096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.170819491147995,
      "rewards/margins": 0.029125628992915154,
      "rewards/rejected": -0.1999451220035553,
      "step": 234
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 0.23391850292682648,
      "learning_rate": 3.9141298268213966e-06,
      "log_odds_chosen": 0.4332599639892578,
      "log_odds_ratio": -0.5040322542190552,
      "logits/chosen": 0.3331410884857178,
      "logits/rejected": -0.6711893677711487,
      "logps/chosen": -1.6272982358932495,
      "logps/rejected": -1.991321325302124,
      "loss": 1.7828,
      "nll_loss": 1.7323882579803467,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16272982954978943,
      "rewards/margins": 0.03640230745077133,
      "rewards/rejected": -0.19913212954998016,
      "step": 235
    },
    {
      "epoch": 0.37194641449960597,
      "grad_norm": 0.28272461891174316,
      "learning_rate": 3.912529134040255e-06,
      "log_odds_chosen": 0.2862742841243744,
      "log_odds_ratio": -0.5635257959365845,
      "logits/chosen": 0.3295111358165741,
      "logits/rejected": -0.5666621327400208,
      "logps/chosen": -1.7123397588729858,
      "logps/rejected": -1.9539713859558105,
      "loss": 1.874,
      "nll_loss": 1.8176579475402832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17123398184776306,
      "rewards/margins": 0.02416316047310829,
      "rewards/rejected": -0.19539715349674225,
      "step": 236
    },
    {
      "epoch": 0.3735224586288416,
      "grad_norm": 0.27505528926849365,
      "learning_rate": 3.910913993014998e-06,
      "log_odds_chosen": 0.45064306259155273,
      "log_odds_ratio": -0.4959080219268799,
      "logits/chosen": 0.41635861992836,
      "logits/rejected": -0.7518233060836792,
      "logps/chosen": -1.7062500715255737,
      "logps/rejected": -2.0893027782440186,
      "loss": 1.854,
      "nll_loss": 1.8043806552886963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1706250011920929,
      "rewards/margins": 0.03830527514219284,
      "rewards/rejected": -0.20893028378486633,
      "step": 237
    },
    {
      "epoch": 0.3750985027580772,
      "grad_norm": 0.26204821467399597,
      "learning_rate": 3.909284415947246e-06,
      "log_odds_chosen": 0.42578941583633423,
      "log_odds_ratio": -0.5093416571617126,
      "logits/chosen": 0.37128570675849915,
      "logits/rejected": -0.8889274597167969,
      "logps/chosen": -1.7555906772613525,
      "logps/rejected": -2.1172666549682617,
      "loss": 1.8899,
      "nll_loss": 1.8389294147491455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17555907368659973,
      "rewards/margins": 0.036167606711387634,
      "rewards/rejected": -0.21172668039798737,
      "step": 238
    },
    {
      "epoch": 0.37667454688731283,
      "grad_norm": 0.2497677356004715,
      "learning_rate": 3.907640415147674e-06,
      "log_odds_chosen": 0.47326362133026123,
      "log_odds_ratio": -0.4884757101535797,
      "logits/chosen": 0.25646257400512695,
      "logits/rejected": -0.9685801863670349,
      "logps/chosen": -1.6502771377563477,
      "logps/rejected": -2.047640800476074,
      "loss": 1.8045,
      "nll_loss": 1.75564706325531,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16502772271633148,
      "rewards/margins": 0.03973636403679848,
      "rewards/rejected": -0.20476408302783966,
      "step": 239
    },
    {
      "epoch": 0.37825059101654845,
      "grad_norm": 0.2736426293849945,
      "learning_rate": 3.905982003035924e-06,
      "log_odds_chosen": 0.38030922412872314,
      "log_odds_ratio": -0.5255998373031616,
      "logits/chosen": 0.28299060463905334,
      "logits/rejected": -0.8722066283226013,
      "logps/chosen": -1.7557415962219238,
      "logps/rejected": -2.0799028873443604,
      "loss": 1.893,
      "nll_loss": 1.8404154777526855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1755741536617279,
      "rewards/margins": 0.03241611272096634,
      "rewards/rejected": -0.20799028873443604,
      "step": 240
    },
    {
      "epoch": 0.3798266351457841,
      "grad_norm": 0.2504989504814148,
      "learning_rate": 3.904309192140506e-06,
      "log_odds_chosen": 0.2628810703754425,
      "log_odds_ratio": -0.5786678194999695,
      "logits/chosen": 0.300167053937912,
      "logits/rejected": -0.4999735355377197,
      "logps/chosen": -1.6737117767333984,
      "logps/rejected": -1.8940290212631226,
      "loss": 1.8283,
      "nll_loss": 1.7704728841781616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16737116873264313,
      "rewards/margins": 0.022031733766198158,
      "rewards/rejected": -0.18940290808677673,
      "step": 241
    },
    {
      "epoch": 0.3814026792750197,
      "grad_norm": 0.2762889564037323,
      "learning_rate": 3.90262199509871e-06,
      "log_odds_chosen": 0.2762555181980133,
      "log_odds_ratio": -0.5677258968353271,
      "logits/chosen": 0.2766878604888916,
      "logits/rejected": -0.591003954410553,
      "logps/chosen": -1.704395055770874,
      "logps/rejected": -1.93495512008667,
      "loss": 1.8625,
      "nll_loss": 1.80568265914917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17043951153755188,
      "rewards/margins": 0.02305598370730877,
      "rewards/rejected": -0.1934954971075058,
      "step": 242
    },
    {
      "epoch": 0.3829787234042553,
      "grad_norm": 0.25611984729766846,
      "learning_rate": 3.900920424656501e-06,
      "log_odds_chosen": 0.41975897550582886,
      "log_odds_ratio": -0.5121146440505981,
      "logits/chosen": 0.2905770540237427,
      "logits/rejected": -0.7793818712234497,
      "logps/chosen": -1.61533784866333,
      "logps/rejected": -1.9650779962539673,
      "loss": 1.7628,
      "nll_loss": 1.7115776538848877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16153380274772644,
      "rewards/margins": 0.034974001348018646,
      "rewards/rejected": -0.1965077966451645,
      "step": 243
    },
    {
      "epoch": 0.38455476753349094,
      "grad_norm": 0.273844838142395,
      "learning_rate": 3.899204493668432e-06,
      "log_odds_chosen": 0.3490802049636841,
      "log_odds_ratio": -0.5446145534515381,
      "logits/chosen": 0.3348943293094635,
      "logits/rejected": -0.6913176774978638,
      "logps/chosen": -1.7777903079986572,
      "logps/rejected": -2.080195903778076,
      "loss": 1.916,
      "nll_loss": 1.861492395401001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17777903378009796,
      "rewards/margins": 0.030240550637245178,
      "rewards/rejected": -0.20801956951618195,
      "step": 244
    },
    {
      "epoch": 0.38613081166272656,
      "grad_norm": 0.2658459544181824,
      "learning_rate": 3.897474215097542e-06,
      "log_odds_chosen": 0.34318360686302185,
      "log_odds_ratio": -0.5412995219230652,
      "logits/chosen": 0.3551873564720154,
      "logits/rejected": -0.9330862760543823,
      "logps/chosen": -1.7187716960906982,
      "logps/rejected": -2.0109448432922363,
      "loss": 1.8774,
      "nll_loss": 1.8233082294464111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1718771755695343,
      "rewards/margins": 0.029217295348644257,
      "rewards/rejected": -0.20109446346759796,
      "step": 245
    },
    {
      "epoch": 0.3877068557919622,
      "grad_norm": 0.24852316081523895,
      "learning_rate": 3.8957296020152596e-06,
      "log_odds_chosen": 0.45998305082321167,
      "log_odds_ratio": -0.4943844676017761,
      "logits/chosen": 0.3586815297603607,
      "logits/rejected": -0.8210923671722412,
      "logps/chosen": -1.6819372177124023,
      "logps/rejected": -2.0723345279693604,
      "loss": 1.8356,
      "nll_loss": 1.786157488822937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1681937277317047,
      "rewards/margins": 0.039039719849824905,
      "rewards/rejected": -0.20723344385623932,
      "step": 246
    },
    {
      "epoch": 0.3892828999211978,
      "grad_norm": 0.28672662377357483,
      "learning_rate": 3.893970667601303e-06,
      "log_odds_chosen": 0.2993201017379761,
      "log_odds_ratio": -0.5586628913879395,
      "logits/chosen": 0.24899174273014069,
      "logits/rejected": -1.121972680091858,
      "logps/chosen": -1.7663580179214478,
      "logps/rejected": -2.020313024520874,
      "loss": 1.9069,
      "nll_loss": 1.8509916067123413,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17663580179214478,
      "rewards/margins": 0.025395508855581284,
      "rewards/rejected": -0.20203131437301636,
      "step": 247
    },
    {
      "epoch": 0.3908589440504334,
      "grad_norm": 0.26399871706962585,
      "learning_rate": 3.892197425143581e-06,
      "log_odds_chosen": 0.46287333965301514,
      "log_odds_ratio": -0.49190735816955566,
      "logits/chosen": 0.30056482553482056,
      "logits/rejected": -0.6871494650840759,
      "logps/chosen": -1.6935536861419678,
      "logps/rejected": -2.085163116455078,
      "loss": 1.8295,
      "nll_loss": 1.7803454399108887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1693553626537323,
      "rewards/margins": 0.03916092962026596,
      "rewards/rejected": -0.20851629972457886,
      "step": 248
    },
    {
      "epoch": 0.39243498817966904,
      "grad_norm": 0.2822303771972656,
      "learning_rate": 3.890409888038094e-06,
      "log_odds_chosen": 0.3473043441772461,
      "log_odds_ratio": -0.5403538942337036,
      "logits/chosen": 0.332716166973114,
      "logits/rejected": -0.7040198445320129,
      "logps/chosen": -1.7375996112823486,
      "logps/rejected": -2.0354490280151367,
      "loss": 1.8869,
      "nll_loss": 1.8328940868377686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17375998198986053,
      "rewards/margins": 0.02978491596877575,
      "rewards/rejected": -0.20354489982128143,
      "step": 249
    },
    {
      "epoch": 0.39401103230890466,
      "grad_norm": 0.2671235203742981,
      "learning_rate": 3.888608069788831e-06,
      "log_odds_chosen": 0.3373025357723236,
      "log_odds_ratio": -0.5513738393783569,
      "logits/chosen": 0.2397560477256775,
      "logits/rejected": -0.565992534160614,
      "logps/chosen": -1.6891679763793945,
      "logps/rejected": -1.9773098230361938,
      "loss": 1.8412,
      "nll_loss": 1.7861071825027466,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16891679167747498,
      "rewards/margins": 0.028814200311899185,
      "rewards/rejected": -0.19773098826408386,
      "step": 250
    },
    {
      "epoch": 0.3955870764381403,
      "grad_norm": 0.24620778858661652,
      "learning_rate": 3.8867919840076685e-06,
      "log_odds_chosen": 0.4103567898273468,
      "log_odds_ratio": -0.5125883221626282,
      "logits/chosen": 0.26780739426612854,
      "logits/rejected": -0.8729146122932434,
      "logps/chosen": -1.618111252784729,
      "logps/rejected": -1.9599648714065552,
      "loss": 1.7577,
      "nll_loss": 1.7063930034637451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16181112825870514,
      "rewards/margins": 0.03418537974357605,
      "rewards/rejected": -0.1959965080022812,
      "step": 251
    },
    {
      "epoch": 0.3971631205673759,
      "grad_norm": 0.2638973593711853,
      "learning_rate": 3.884961644414267e-06,
      "log_odds_chosen": 0.33575767278671265,
      "log_odds_ratio": -0.5410860776901245,
      "logits/chosen": 0.2888558804988861,
      "logits/rejected": -0.4153730273246765,
      "logps/chosen": -1.7437500953674316,
      "logps/rejected": -2.0283775329589844,
      "loss": 1.865,
      "nll_loss": 1.810841679573059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1743750125169754,
      "rewards/margins": 0.028462760150432587,
      "rewards/rejected": -0.2028377652168274,
      "step": 252
    },
    {
      "epoch": 0.3987391646966115,
      "grad_norm": 0.2482423335313797,
      "learning_rate": 3.883117064835967e-06,
      "log_odds_chosen": 0.38616234064102173,
      "log_odds_ratio": -0.5299303531646729,
      "logits/chosen": 0.2521415948867798,
      "logits/rejected": -0.7721424102783203,
      "logps/chosen": -1.6814165115356445,
      "logps/rejected": -2.0103795528411865,
      "loss": 1.8339,
      "nll_loss": 1.7809207439422607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1681416630744934,
      "rewards/margins": 0.03289632126688957,
      "rewards/rejected": -0.20103797316551208,
      "step": 253
    },
    {
      "epoch": 0.40031520882584715,
      "grad_norm": 0.2557568848133087,
      "learning_rate": 3.881258259207688e-06,
      "log_odds_chosen": 0.32933154702186584,
      "log_odds_ratio": -0.5472507476806641,
      "logits/chosen": 0.34836918115615845,
      "logits/rejected": -0.8260299563407898,
      "logps/chosen": -1.6516460180282593,
      "logps/rejected": -1.9261865615844727,
      "loss": 1.7824,
      "nll_loss": 1.727644920349121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16516461968421936,
      "rewards/margins": 0.027454044669866562,
      "rewards/rejected": -0.19261866807937622,
      "step": 254
    },
    {
      "epoch": 0.40189125295508277,
      "grad_norm": 0.2454916387796402,
      "learning_rate": 3.8793852415718165e-06,
      "log_odds_chosen": 0.41223955154418945,
      "log_odds_ratio": -0.5096971988677979,
      "logits/chosen": 0.26183220744132996,
      "logits/rejected": -1.1104363203048706,
      "logps/chosen": -1.5731486082077026,
      "logps/rejected": -1.911667823791504,
      "loss": 1.7079,
      "nll_loss": 1.656911015510559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15731485188007355,
      "rewards/margins": 0.03385192155838013,
      "rewards/rejected": -0.19116678833961487,
      "step": 255
    },
    {
      "epoch": 0.4034672970843184,
      "grad_norm": 0.24788597226142883,
      "learning_rate": 3.877498026078107e-06,
      "log_odds_chosen": 0.2555736005306244,
      "log_odds_ratio": -0.5793567299842834,
      "logits/chosen": 0.2708052396774292,
      "logits/rejected": -0.9535562992095947,
      "logps/chosen": -1.718558430671692,
      "logps/rejected": -1.936058759689331,
      "loss": 1.8553,
      "nll_loss": 1.7973601818084717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1718558371067047,
      "rewards/margins": 0.021750036627054214,
      "rewards/rejected": -0.19360588490962982,
      "step": 256
    },
    {
      "epoch": 0.40504334121355395,
      "grad_norm": 0.2589392364025116,
      "learning_rate": 3.875596626983573e-06,
      "log_odds_chosen": 0.469348669052124,
      "log_odds_ratio": -0.4969593286514282,
      "logits/chosen": 0.29150527715682983,
      "logits/rejected": -0.6516475677490234,
      "logps/chosen": -1.6384902000427246,
      "logps/rejected": -2.030492067337036,
      "loss": 1.7747,
      "nll_loss": 1.7249572277069092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16384902596473694,
      "rewards/margins": 0.039200179278850555,
      "rewards/rejected": -0.2030492126941681,
      "step": 257
    },
    {
      "epoch": 0.4066193853427896,
      "grad_norm": 0.2652013599872589,
      "learning_rate": 3.873681058652374e-06,
      "log_odds_chosen": 0.2649344503879547,
      "log_odds_ratio": -0.5769127607345581,
      "logits/chosen": 0.26984673738479614,
      "logits/rejected": -0.7516025304794312,
      "logps/chosen": -1.7576013803482056,
      "logps/rejected": -1.9836013317108154,
      "loss": 1.8938,
      "nll_loss": 1.836103081703186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1757601499557495,
      "rewards/margins": 0.02259998768568039,
      "rewards/rejected": -0.1983601450920105,
      "step": 258
    },
    {
      "epoch": 0.4081954294720252,
      "grad_norm": 0.275820255279541,
      "learning_rate": 3.871751335555715e-06,
      "log_odds_chosen": 0.26289334893226624,
      "log_odds_ratio": -0.5722215175628662,
      "logits/chosen": 0.25413140654563904,
      "logits/rejected": -0.9038801193237305,
      "logps/chosen": -1.6938437223434448,
      "logps/rejected": -1.9135998487472534,
      "loss": 1.8231,
      "nll_loss": 1.7658631801605225,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16938436031341553,
      "rewards/margins": 0.021975621581077576,
      "rewards/rejected": -0.1913599818944931,
      "step": 259
    },
    {
      "epoch": 0.4097714736012608,
      "grad_norm": 0.24297621846199036,
      "learning_rate": 3.869807472271731e-06,
      "log_odds_chosen": 0.33671316504478455,
      "log_odds_ratio": -0.5421686172485352,
      "logits/chosen": 0.2816615402698517,
      "logits/rejected": -0.782688558101654,
      "logps/chosen": -1.6605300903320312,
      "logps/rejected": -1.943679928779602,
      "loss": 1.8234,
      "nll_loss": 1.7691376209259033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16605301201343536,
      "rewards/margins": 0.02831498347222805,
      "rewards/rejected": -0.19436800479888916,
      "step": 260
    },
    {
      "epoch": 0.41134751773049644,
      "grad_norm": 0.24730239808559418,
      "learning_rate": 3.8678494834853826e-06,
      "log_odds_chosen": 0.3380075693130493,
      "log_odds_ratio": -0.5455616116523743,
      "logits/chosen": 0.2523096799850464,
      "logits/rejected": -0.7331975698471069,
      "logps/chosen": -1.6633615493774414,
      "logps/rejected": -1.9441428184509277,
      "loss": 1.7975,
      "nll_loss": 1.7429437637329102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16633614897727966,
      "rewards/margins": 0.02807813137769699,
      "rewards/rejected": -0.19441428780555725,
      "step": 261
    },
    {
      "epoch": 0.41292356185973206,
      "grad_norm": 0.27988502383232117,
      "learning_rate": 3.865877383988339e-06,
      "log_odds_chosen": 0.4063892066478729,
      "log_odds_ratio": -0.5170100331306458,
      "logits/chosen": 0.25469520688056946,
      "logits/rejected": -0.7429967522621155,
      "logps/chosen": -1.6950594186782837,
      "logps/rejected": -2.035081624984741,
      "loss": 1.8501,
      "nll_loss": 1.7983577251434326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16950593888759613,
      "rewards/margins": 0.03400222212076187,
      "rewards/rejected": -0.2035081684589386,
      "step": 262
    },
    {
      "epoch": 0.4144996059889677,
      "grad_norm": 0.25723162293434143,
      "learning_rate": 3.863891188678869e-06,
      "log_odds_chosen": 0.14911779761314392,
      "log_odds_ratio": -0.631981372833252,
      "logits/chosen": 0.2207110971212387,
      "logits/rejected": -0.883880615234375,
      "logps/chosen": -1.6087576150894165,
      "logps/rejected": -1.7343672513961792,
      "loss": 1.7568,
      "nll_loss": 1.6936231851577759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16087576746940613,
      "rewards/margins": 0.012560954317450523,
      "rewards/rejected": -0.1734367161989212,
      "step": 263
    },
    {
      "epoch": 0.4160756501182033,
      "grad_norm": 0.25149017572402954,
      "learning_rate": 3.8618909125617305e-06,
      "log_odds_chosen": 0.5187549591064453,
      "log_odds_ratio": -0.47057828307151794,
      "logits/chosen": 0.2168709635734558,
      "logits/rejected": -1.114577293395996,
      "logps/chosen": -1.5999822616577148,
      "logps/rejected": -2.0340919494628906,
      "loss": 1.7533,
      "nll_loss": 1.7062718868255615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15999823808670044,
      "rewards/margins": 0.04341099411249161,
      "rewards/rejected": -0.20340920984745026,
      "step": 264
    },
    {
      "epoch": 0.4176516942474389,
      "grad_norm": 0.2453727126121521,
      "learning_rate": 3.859876570748054e-06,
      "log_odds_chosen": 0.3410520851612091,
      "log_odds_ratio": -0.5392724871635437,
      "logits/chosen": 0.2857251465320587,
      "logits/rejected": -0.38999515771865845,
      "logps/chosen": -1.6569011211395264,
      "logps/rejected": -1.9416050910949707,
      "loss": 1.8086,
      "nll_loss": 1.7547087669372559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1656901240348816,
      "rewards/margins": 0.028470391407608986,
      "rewards/rejected": -0.19416050612926483,
      "step": 265
    },
    {
      "epoch": 0.41922773837667454,
      "grad_norm": 0.24315941333770752,
      "learning_rate": 3.857848178455231e-06,
      "log_odds_chosen": 0.10583052039146423,
      "log_odds_ratio": -0.6480942964553833,
      "logits/chosen": 0.2967842221260071,
      "logits/rejected": -1.1144427061080933,
      "logps/chosen": -1.6708816289901733,
      "logps/rejected": -1.7641648054122925,
      "loss": 1.8376,
      "nll_loss": 1.7727646827697754,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16708815097808838,
      "rewards/margins": 0.009328329935669899,
      "rewards/rejected": -0.17641650140285492,
      "step": 266
    },
    {
      "epoch": 0.42080378250591016,
      "grad_norm": 0.27732643485069275,
      "learning_rate": 3.855805751006794e-06,
      "log_odds_chosen": 0.4604467749595642,
      "log_odds_ratio": -0.498542845249176,
      "logits/chosen": 0.2691567838191986,
      "logits/rejected": -1.1059002876281738,
      "logps/chosen": -1.6944466829299927,
      "logps/rejected": -2.0833187103271484,
      "loss": 1.8215,
      "nll_loss": 1.7716130018234253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16944465041160583,
      "rewards/margins": 0.03888721019029617,
      "rewards/rejected": -0.2083318680524826,
      "step": 267
    },
    {
      "epoch": 0.4223798266351458,
      "grad_norm": 0.23589597642421722,
      "learning_rate": 3.853749303832308e-06,
      "log_odds_chosen": 0.33512479066848755,
      "log_odds_ratio": -0.5445858836174011,
      "logits/chosen": 0.24153344333171844,
      "logits/rejected": -0.648094117641449,
      "logps/chosen": -1.6203473806381226,
      "logps/rejected": -1.897982120513916,
      "loss": 1.761,
      "nll_loss": 1.706520438194275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16203473508358002,
      "rewards/margins": 0.027763469144701958,
      "rewards/rejected": -0.18979820609092712,
      "step": 268
    },
    {
      "epoch": 0.4239558707643814,
      "grad_norm": 0.2540602385997772,
      "learning_rate": 3.8516788524672495e-06,
      "log_odds_chosen": 0.20173487067222595,
      "log_odds_ratio": -0.6037977933883667,
      "logits/chosen": 0.27147728204727173,
      "logits/rejected": -0.9890564680099487,
      "logps/chosen": -1.69952392578125,
      "logps/rejected": -1.8702551126480103,
      "loss": 1.8336,
      "nll_loss": 1.773256540298462,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.169952392578125,
      "rewards/margins": 0.01707313396036625,
      "rewards/rejected": -0.1870255172252655,
      "step": 269
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 0.24478159844875336,
      "learning_rate": 3.849594412552889e-06,
      "log_odds_chosen": 0.33746102452278137,
      "log_odds_ratio": -0.5440058708190918,
      "logits/chosen": 0.285639226436615,
      "logits/rejected": -0.7275075912475586,
      "logps/chosen": -1.6040332317352295,
      "logps/rejected": -1.8771222829818726,
      "loss": 1.7444,
      "nll_loss": 1.689996361732483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1604033261537552,
      "rewards/margins": 0.027308890596032143,
      "rewards/rejected": -0.18771222233772278,
      "step": 270
    },
    {
      "epoch": 0.42710795902285265,
      "grad_norm": 0.25568917393684387,
      "learning_rate": 3.847495999836175e-06,
      "log_odds_chosen": 0.3471507430076599,
      "log_odds_ratio": -0.5386791229248047,
      "logits/chosen": 0.3089812994003296,
      "logits/rejected": -0.9828994870185852,
      "logps/chosen": -1.6793549060821533,
      "logps/rejected": -1.9712432622909546,
      "loss": 1.8059,
      "nll_loss": 1.7519892454147339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16793550550937653,
      "rewards/margins": 0.02918882854282856,
      "rewards/rejected": -0.19712434709072113,
      "step": 271
    },
    {
      "epoch": 0.42868400315208827,
      "grad_norm": 0.254783034324646,
      "learning_rate": 3.845383630169613e-06,
      "log_odds_chosen": 0.31106436252593994,
      "log_odds_ratio": -0.5561890602111816,
      "logits/chosen": 0.16303634643554688,
      "logits/rejected": -1.0483474731445312,
      "logps/chosen": -1.5915002822875977,
      "logps/rejected": -1.8480899333953857,
      "loss": 1.7205,
      "nll_loss": 1.6648414134979248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15915003418922424,
      "rewards/margins": 0.02565896324813366,
      "rewards/rejected": -0.18480899930000305,
      "step": 272
    },
    {
      "epoch": 0.4302600472813239,
      "grad_norm": 0.25443676114082336,
      "learning_rate": 3.843257319511147e-06,
      "log_odds_chosen": 0.54793381690979,
      "log_odds_ratio": -0.4633035957813263,
      "logits/chosen": 0.21883374452590942,
      "logits/rejected": -1.1158220767974854,
      "logps/chosen": -1.5903308391571045,
      "logps/rejected": -2.0446715354919434,
      "loss": 1.7182,
      "nll_loss": 1.6718711853027344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15903308987617493,
      "rewards/margins": 0.04543406888842583,
      "rewards/rejected": -0.20446714758872986,
      "step": 273
    },
    {
      "epoch": 0.4318360914105595,
      "grad_norm": 0.25186702609062195,
      "learning_rate": 3.841117083924039e-06,
      "log_odds_chosen": 0.3146308958530426,
      "log_odds_ratio": -0.5538010597229004,
      "logits/chosen": 0.20665690302848816,
      "logits/rejected": -1.150301218032837,
      "logps/chosen": -1.6276448965072632,
      "logps/rejected": -1.8898921012878418,
      "loss": 1.7612,
      "nll_loss": 1.705790400505066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16276448965072632,
      "rewards/margins": 0.026224708184599876,
      "rewards/rejected": -0.18898919224739075,
      "step": 274
    },
    {
      "epoch": 0.43341213553979513,
      "grad_norm": 0.2522598206996918,
      "learning_rate": 3.838962939576746e-06,
      "log_odds_chosen": 0.41350072622299194,
      "log_odds_ratio": -0.5153959393501282,
      "logits/chosen": 0.17553453147411346,
      "logits/rejected": -0.9353335499763489,
      "logps/chosen": -1.5784237384796143,
      "logps/rejected": -1.9216094017028809,
      "loss": 1.7319,
      "nll_loss": 1.6803406476974487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15784236788749695,
      "rewards/margins": 0.03431856259703636,
      "rewards/rejected": -0.1921609491109848,
      "step": 275
    },
    {
      "epoch": 0.43498817966903075,
      "grad_norm": 0.2677028477191925,
      "learning_rate": 3.8367949027427985e-06,
      "log_odds_chosen": 0.2886348366737366,
      "log_odds_ratio": -0.5637444257736206,
      "logits/chosen": 0.23681315779685974,
      "logits/rejected": -0.8251351714134216,
      "logps/chosen": -1.6966907978057861,
      "logps/rejected": -1.9407154321670532,
      "loss": 1.8551,
      "nll_loss": 1.798759937286377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16966909170150757,
      "rewards/margins": 0.024402447044849396,
      "rewards/rejected": -0.19407154619693756,
      "step": 276
    },
    {
      "epoch": 0.43656422379826637,
      "grad_norm": 0.27280256152153015,
      "learning_rate": 3.834612989800681e-06,
      "log_odds_chosen": 0.5323548316955566,
      "log_odds_ratio": -0.46610027551651,
      "logits/chosen": 0.25051310658454895,
      "logits/rejected": -1.3698272705078125,
      "logps/chosen": -1.680455207824707,
      "logps/rejected": -2.1312544345855713,
      "loss": 1.7988,
      "nll_loss": 1.7521436214447021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1680455207824707,
      "rewards/margins": 0.04507990926504135,
      "rewards/rejected": -0.21312543749809265,
      "step": 277
    },
    {
      "epoch": 0.438140267927502,
      "grad_norm": 0.2634829878807068,
      "learning_rate": 3.832417217233703e-06,
      "log_odds_chosen": 0.29642635583877563,
      "log_odds_ratio": -0.5586500763893127,
      "logits/chosen": 0.21414227783679962,
      "logits/rejected": -0.7982608675956726,
      "logps/chosen": -1.6701531410217285,
      "logps/rejected": -1.9189419746398926,
      "loss": 1.8078,
      "nll_loss": 1.751927375793457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16701531410217285,
      "rewards/margins": 0.024878891184926033,
      "rewards/rejected": -0.19189421832561493,
      "step": 278
    },
    {
      "epoch": 0.4397163120567376,
      "grad_norm": 0.2531651556491852,
      "learning_rate": 3.8302076016298775e-06,
      "log_odds_chosen": 0.4342229664325714,
      "log_odds_ratio": -0.5029668211936951,
      "logits/chosen": 0.1577448695898056,
      "logits/rejected": -1.0718286037445068,
      "logps/chosen": -1.6032099723815918,
      "logps/rejected": -1.9638495445251465,
      "loss": 1.7458,
      "nll_loss": 1.695469617843628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16032099723815918,
      "rewards/margins": 0.03606396168470383,
      "rewards/rejected": -0.1963849514722824,
      "step": 279
    },
    {
      "epoch": 0.44129235618597323,
      "grad_norm": 0.2585320472717285,
      "learning_rate": 3.827984159681796e-06,
      "log_odds_chosen": 0.33446642756462097,
      "log_odds_ratio": -0.5452247262001038,
      "logits/chosen": 0.14251501858234406,
      "logits/rejected": -0.8923399448394775,
      "logps/chosen": -1.654819130897522,
      "logps/rejected": -1.932909369468689,
      "loss": 1.7837,
      "nll_loss": 1.7291784286499023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16548191010951996,
      "rewards/margins": 0.027809027582406998,
      "rewards/rejected": -0.19329093396663666,
      "step": 280
    },
    {
      "epoch": 0.4428684003152088,
      "grad_norm": 0.2518828809261322,
      "learning_rate": 3.825746908186498e-06,
      "log_odds_chosen": 0.38528937101364136,
      "log_odds_ratio": -0.5269155502319336,
      "logits/chosen": 0.2786559760570526,
      "logits/rejected": -1.1318894624710083,
      "logps/chosen": -1.6995317935943604,
      "logps/rejected": -2.024996519088745,
      "loss": 1.8116,
      "nll_loss": 1.7589478492736816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16995318233966827,
      "rewards/margins": 0.03254646807909012,
      "rewards/rejected": -0.2024996429681778,
      "step": 281
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 0.26809749007225037,
      "learning_rate": 3.823495864045352e-06,
      "log_odds_chosen": 0.4371758699417114,
      "log_odds_ratio": -0.5007396936416626,
      "logits/chosen": 0.2518642842769623,
      "logits/rejected": -0.5491166114807129,
      "logps/chosen": -1.6472457647323608,
      "logps/rejected": -2.0112357139587402,
      "loss": 1.7796,
      "nll_loss": 1.7295327186584473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16472457349300385,
      "rewards/margins": 0.036398995667696,
      "rewards/rejected": -0.20112356543540955,
      "step": 282
    },
    {
      "epoch": 0.44602048857368004,
      "grad_norm": 0.2323625683784485,
      "learning_rate": 3.8212310442639205e-06,
      "log_odds_chosen": 0.32891276478767395,
      "log_odds_ratio": -0.5458555221557617,
      "logits/chosen": 0.24299903213977814,
      "logits/rejected": -0.7525103092193604,
      "logps/chosen": -1.592462182044983,
      "logps/rejected": -1.8642407655715942,
      "loss": 1.738,
      "nll_loss": 1.6833724975585938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15924622118473053,
      "rewards/margins": 0.027177870273590088,
      "rewards/rejected": -0.18642409145832062,
      "step": 283
    },
    {
      "epoch": 0.44759653270291566,
      "grad_norm": 0.2419874519109726,
      "learning_rate": 3.8189524659518355e-06,
      "log_odds_chosen": 0.39278745651245117,
      "log_odds_ratio": -0.5193477869033813,
      "logits/chosen": 0.31300458312034607,
      "logits/rejected": -0.9649338722229004,
      "logps/chosen": -1.664646029472351,
      "logps/rejected": -1.992173194885254,
      "loss": 1.7913,
      "nll_loss": 1.7394092082977295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16646461188793182,
      "rewards/margins": 0.03275272995233536,
      "rewards/rejected": -0.19921733438968658,
      "step": 284
    },
    {
      "epoch": 0.4491725768321513,
      "grad_norm": 0.2631385326385498,
      "learning_rate": 3.816660146322667e-06,
      "log_odds_chosen": 0.3710939884185791,
      "log_odds_ratio": -0.5282797813415527,
      "logits/chosen": 0.22600415349006653,
      "logits/rejected": -0.7380872964859009,
      "logps/chosen": -1.5571084022521973,
      "logps/rejected": -1.8607244491577148,
      "loss": 1.6912,
      "nll_loss": 1.6383720636367798,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557108461856842,
      "rewards/margins": 0.030361609533429146,
      "rewards/rejected": -0.1860724538564682,
      "step": 285
    },
    {
      "epoch": 0.4507486209613869,
      "grad_norm": 0.2479519248008728,
      "learning_rate": 3.814354102693797e-06,
      "log_odds_chosen": 0.3791089355945587,
      "log_odds_ratio": -0.5239448547363281,
      "logits/chosen": 0.2559822201728821,
      "logits/rejected": -1.0194425582885742,
      "logps/chosen": -1.6287118196487427,
      "logps/rejected": -1.9415102005004883,
      "loss": 1.7601,
      "nll_loss": 1.7077348232269287,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16287118196487427,
      "rewards/margins": 0.03127983957529068,
      "rewards/rejected": -0.19415102899074554,
      "step": 286
    },
    {
      "epoch": 0.4523246650906225,
      "grad_norm": 0.2415776550769806,
      "learning_rate": 3.8120343524862814e-06,
      "log_odds_chosen": 0.3282337784767151,
      "log_odds_ratio": -0.5441159009933472,
      "logits/chosen": 0.22718586027622223,
      "logits/rejected": -0.8667283058166504,
      "logps/chosen": -1.669337272644043,
      "logps/rejected": -1.9410953521728516,
      "loss": 1.7842,
      "nll_loss": 1.7298314571380615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16693371534347534,
      "rewards/margins": 0.027175823226571083,
      "rewards/rejected": -0.19410954415798187,
      "step": 287
    },
    {
      "epoch": 0.45390070921985815,
      "grad_norm": 0.23657608032226562,
      "learning_rate": 3.809700913224726e-06,
      "log_odds_chosen": 0.37087053060531616,
      "log_odds_ratio": -0.5310470461845398,
      "logits/chosen": 0.19090989232063293,
      "logits/rejected": -1.3478600978851318,
      "logps/chosen": -1.5922802686691284,
      "logps/rejected": -1.897598147392273,
      "loss": 1.7178,
      "nll_loss": 1.6647306680679321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15922804176807404,
      "rewards/margins": 0.030531786382198334,
      "rewards/rejected": -0.18975980579853058,
      "step": 288
    },
    {
      "epoch": 0.45547675334909377,
      "grad_norm": 0.24203184247016907,
      "learning_rate": 3.8073538025371494e-06,
      "log_odds_chosen": 0.46555230021476746,
      "log_odds_ratio": -0.49043652415275574,
      "logits/chosen": 0.30645501613616943,
      "logits/rejected": -1.1749683618545532,
      "logps/chosen": -1.568337321281433,
      "logps/rejected": -1.9523917436599731,
      "loss": 1.7066,
      "nll_loss": 1.6575335264205933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1568337380886078,
      "rewards/margins": 0.03840543329715729,
      "rewards/rejected": -0.19523918628692627,
      "step": 289
    },
    {
      "epoch": 0.4570527974783294,
      "grad_norm": 0.2672315537929535,
      "learning_rate": 3.804993038154852e-06,
      "log_odds_chosen": 0.34897854924201965,
      "log_odds_ratio": -0.541498601436615,
      "logits/chosen": 0.2300131767988205,
      "logits/rejected": -1.0130763053894043,
      "logps/chosen": -1.6003525257110596,
      "logps/rejected": -1.8910918235778809,
      "loss": 1.7494,
      "nll_loss": 1.6952693462371826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16003525257110596,
      "rewards/margins": 0.02907394804060459,
      "rewards/rejected": -0.189109206199646,
      "step": 290
    },
    {
      "epoch": 0.458628841607565,
      "grad_norm": 0.2524731755256653,
      "learning_rate": 3.8026186379122816e-06,
      "log_odds_chosen": 0.2566876709461212,
      "log_odds_ratio": -0.5760392546653748,
      "logits/chosen": 0.19924253225326538,
      "logits/rejected": -1.0389641523361206,
      "logps/chosen": -1.6778925657272339,
      "logps/rejected": -1.891493797302246,
      "loss": 1.8,
      "nll_loss": 1.7424429655075073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1677892506122589,
      "rewards/margins": 0.021360116079449654,
      "rewards/rejected": -0.1891493797302246,
      "step": 291
    },
    {
      "epoch": 0.46020488573680063,
      "grad_norm": 0.24778525531291962,
      "learning_rate": 3.8002306197468983e-06,
      "log_odds_chosen": 0.17079176008701324,
      "log_odds_ratio": -0.613702654838562,
      "logits/chosen": 0.19200792908668518,
      "logits/rejected": -1.0982946157455444,
      "logps/chosen": -1.6556106805801392,
      "logps/rejected": -1.795432686805725,
      "loss": 1.7958,
      "nll_loss": 1.734451174736023,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16556105017662048,
      "rewards/margins": 0.01398220844566822,
      "rewards/rejected": -0.17954328656196594,
      "step": 292
    },
    {
      "epoch": 0.46178092986603625,
      "grad_norm": 0.22741490602493286,
      "learning_rate": 3.7978290016990367e-06,
      "log_odds_chosen": 0.48492002487182617,
      "log_odds_ratio": -0.4839008152484894,
      "logits/chosen": 0.17027397453784943,
      "logits/rejected": -1.0292561054229736,
      "logps/chosen": -1.6551365852355957,
      "logps/rejected": -2.065377712249756,
      "loss": 1.7785,
      "nll_loss": 1.7300668954849243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16551366448402405,
      "rewards/margins": 0.0410241037607193,
      "rewards/rejected": -0.20653776824474335,
      "step": 293
    },
    {
      "epoch": 0.46335697399527187,
      "grad_norm": 0.2619555592536926,
      "learning_rate": 3.795413801911776e-06,
      "log_odds_chosen": 0.54489666223526,
      "log_odds_ratio": -0.46421873569488525,
      "logits/chosen": 0.15529164671897888,
      "logits/rejected": -1.1627665758132935,
      "logps/chosen": -1.5853267908096313,
      "logps/rejected": -2.041583299636841,
      "loss": 1.7285,
      "nll_loss": 1.6820851564407349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15853269398212433,
      "rewards/margins": 0.045625634491443634,
      "rewards/rejected": -0.20415832102298737,
      "step": 294
    },
    {
      "epoch": 0.4649330181245075,
      "grad_norm": 0.2782156765460968,
      "learning_rate": 3.7929850386307965e-06,
      "log_odds_chosen": 0.323818564414978,
      "log_odds_ratio": -0.5496729612350464,
      "logits/chosen": 0.2609451711177826,
      "logits/rejected": -0.9133318662643433,
      "logps/chosen": -1.7928597927093506,
      "logps/rejected": -2.07020902633667,
      "loss": 1.9066,
      "nll_loss": 1.851614236831665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17928597331047058,
      "rewards/margins": 0.027734942734241486,
      "rewards/rejected": -0.20702092349529266,
      "step": 295
    },
    {
      "epoch": 0.4665090622537431,
      "grad_norm": 0.2524731755256653,
      "learning_rate": 3.790542730204245e-06,
      "log_odds_chosen": 0.38338762521743774,
      "log_odds_ratio": -0.5222858190536499,
      "logits/chosen": 0.2131703644990921,
      "logits/rejected": -1.1159170866012573,
      "logps/chosen": -1.7220946550369263,
      "logps/rejected": -2.048020124435425,
      "loss": 1.8309,
      "nll_loss": 1.7786983251571655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1722094714641571,
      "rewards/margins": 0.03259255737066269,
      "rewards/rejected": -0.2048020213842392,
      "step": 296
    },
    {
      "epoch": 0.46808510638297873,
      "grad_norm": 0.2552836835384369,
      "learning_rate": 3.7880868950825935e-06,
      "log_odds_chosen": 0.3997831344604492,
      "log_odds_ratio": -0.513701856136322,
      "logits/chosen": 0.1777328997850418,
      "logits/rejected": -1.2488939762115479,
      "logps/chosen": -1.6790422201156616,
      "logps/rejected": -2.0147104263305664,
      "loss": 1.7804,
      "nll_loss": 1.7290263175964355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16790422797203064,
      "rewards/margins": 0.03356681764125824,
      "rewards/rejected": -0.20147106051445007,
      "step": 297
    },
    {
      "epoch": 0.46966115051221435,
      "grad_norm": 0.2645845115184784,
      "learning_rate": 3.7856175518185058e-06,
      "log_odds_chosen": 0.39919474720954895,
      "log_odds_ratio": -0.519536554813385,
      "logits/chosen": 0.04900103062391281,
      "logits/rejected": -1.2728874683380127,
      "logps/chosen": -1.6557228565216064,
      "logps/rejected": -1.9891810417175293,
      "loss": 1.7843,
      "nll_loss": 1.7323014736175537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16557228565216064,
      "rewards/margins": 0.033345822244882584,
      "rewards/rejected": -0.19891811907291412,
      "step": 298
    },
    {
      "epoch": 0.47123719464145,
      "grad_norm": 0.25159481167793274,
      "learning_rate": 3.7831347190666883e-06,
      "log_odds_chosen": 0.4714580476284027,
      "log_odds_ratio": -0.49071937799453735,
      "logits/chosen": 0.12509863078594208,
      "logits/rejected": -1.183892846107483,
      "logps/chosen": -1.6513078212738037,
      "logps/rejected": -2.048064708709717,
      "loss": 1.7823,
      "nll_loss": 1.7332583665847778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16513077914714813,
      "rewards/margins": 0.03967569023370743,
      "rewards/rejected": -0.20480646193027496,
      "step": 299
    },
    {
      "epoch": 0.4728132387706856,
      "grad_norm": 0.25482556223869324,
      "learning_rate": 3.780638415583759e-06,
      "log_odds_chosen": 0.27312323451042175,
      "log_odds_ratio": -0.5755019187927246,
      "logits/chosen": 0.24975763261318207,
      "logits/rejected": -0.9788644909858704,
      "logps/chosen": -1.6345561742782593,
      "logps/rejected": -1.8644256591796875,
      "loss": 1.7747,
      "nll_loss": 1.7171214818954468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16345562040805817,
      "rewards/margins": 0.022986939176917076,
      "rewards/rejected": -0.1864425539970398,
      "step": 300
    },
    {
      "epoch": 0.4743892828999212,
      "grad_norm": 0.25458070635795593,
      "learning_rate": 3.7781286602280967e-06,
      "log_odds_chosen": 0.16361647844314575,
      "log_odds_ratio": -0.6239433288574219,
      "logits/chosen": 0.1774168461561203,
      "logits/rejected": -1.2051948308944702,
      "logps/chosen": -1.616410493850708,
      "logps/rejected": -1.745263934135437,
      "loss": 1.7654,
      "nll_loss": 1.702994465827942,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16164104640483856,
      "rewards/margins": 0.012885356321930885,
      "rewards/rejected": -0.1745263934135437,
      "step": 301
    },
    {
      "epoch": 0.47596532702915684,
      "grad_norm": 0.23742206394672394,
      "learning_rate": 3.7756054719597044e-06,
      "log_odds_chosen": 0.2543914318084717,
      "log_odds_ratio": -0.59993976354599,
      "logits/chosen": 0.06885389983654022,
      "logits/rejected": -1.19141685962677,
      "logps/chosen": -1.6490998268127441,
      "logps/rejected": -1.869534969329834,
      "loss": 1.7688,
      "nll_loss": 1.7087764739990234,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1649099886417389,
      "rewards/margins": 0.02204350382089615,
      "rewards/rejected": -0.18695348501205444,
      "step": 302
    },
    {
      "epoch": 0.47754137115839246,
      "grad_norm": 0.22604241967201233,
      "learning_rate": 3.773068869840066e-06,
      "log_odds_chosen": 0.19994798302650452,
      "log_odds_ratio": -0.6074256300926208,
      "logits/chosen": 0.28619179129600525,
      "logits/rejected": -1.2311829328536987,
      "logps/chosen": -1.6744049787521362,
      "logps/rejected": -1.838005542755127,
      "loss": 1.7948,
      "nll_loss": 1.7340208292007446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1674405187368393,
      "rewards/margins": 0.016360044479370117,
      "rewards/rejected": -0.1838005632162094,
      "step": 303
    },
    {
      "epoch": 0.4791174152876281,
      "grad_norm": 0.25295767188072205,
      "learning_rate": 3.770518873031997e-06,
      "log_odds_chosen": 0.4454966187477112,
      "log_odds_ratio": -0.4978073537349701,
      "logits/chosen": 0.06649523228406906,
      "logits/rejected": -0.8774638772010803,
      "logps/chosen": -1.5495182275772095,
      "logps/rejected": -1.9150521755218506,
      "loss": 1.6921,
      "nll_loss": 1.6423286199569702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15495182573795319,
      "rewards/margins": 0.03655339777469635,
      "rewards/rejected": -0.19150523841381073,
      "step": 304
    },
    {
      "epoch": 0.4806934594168637,
      "grad_norm": 0.26729831099510193,
      "learning_rate": 3.7679555007995065e-06,
      "log_odds_chosen": 0.39848801493644714,
      "log_odds_ratio": -0.5314606428146362,
      "logits/chosen": 0.18635906279087067,
      "logits/rejected": -1.1083000898361206,
      "logps/chosen": -1.671202301979065,
      "logps/rejected": -2.0063977241516113,
      "loss": 1.8023,
      "nll_loss": 1.7491679191589355,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16712023317813873,
      "rewards/margins": 0.033519547432661057,
      "rewards/rejected": -0.2006397843360901,
      "step": 305
    },
    {
      "epoch": 0.48226950354609927,
      "grad_norm": 0.25258681178092957,
      "learning_rate": 3.7653787725076464e-06,
      "log_odds_chosen": 0.25875869393348694,
      "log_odds_ratio": -0.5776093602180481,
      "logits/chosen": 0.18513011932373047,
      "logits/rejected": -1.0163553953170776,
      "logps/chosen": -1.6290266513824463,
      "logps/rejected": -1.844577670097351,
      "loss": 1.7547,
      "nll_loss": 1.696903109550476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16290265321731567,
      "rewards/margins": 0.021555107086896896,
      "rewards/rejected": -0.18445777893066406,
      "step": 306
    },
    {
      "epoch": 0.4838455476753349,
      "grad_norm": 0.26785412430763245,
      "learning_rate": 3.7627887076223685e-06,
      "log_odds_chosen": 0.36140918731689453,
      "log_odds_ratio": -0.5329678058624268,
      "logits/chosen": 0.2078455090522766,
      "logits/rejected": -0.8232326507568359,
      "logps/chosen": -1.6960173845291138,
      "logps/rejected": -1.9999104738235474,
      "loss": 1.8036,
      "nll_loss": 1.7503188848495483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16960173845291138,
      "rewards/margins": 0.03038930706679821,
      "rewards/rejected": -0.19999106228351593,
      "step": 307
    },
    {
      "epoch": 0.4854215918045705,
      "grad_norm": 0.22155308723449707,
      "learning_rate": 3.7601853257103765e-06,
      "log_odds_chosen": 0.21894103288650513,
      "log_odds_ratio": -0.5963100790977478,
      "logits/chosen": 0.103122279047966,
      "logits/rejected": -1.134813904762268,
      "logps/chosen": -1.5974401235580444,
      "logps/rejected": -1.7806645631790161,
      "loss": 1.7083,
      "nll_loss": 1.6486657857894897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1597440242767334,
      "rewards/margins": 0.018322426825761795,
      "rewards/rejected": -0.1780664473772049,
      "step": 308
    },
    {
      "epoch": 0.48699763593380613,
      "grad_norm": 0.22800187766551971,
      "learning_rate": 3.7575686464389767e-06,
      "log_odds_chosen": 0.34263283014297485,
      "log_odds_ratio": -0.5459380745887756,
      "logits/chosen": 0.1387854516506195,
      "logits/rejected": -1.1660175323486328,
      "logps/chosen": -1.5741075277328491,
      "logps/rejected": -1.856614351272583,
      "loss": 1.7095,
      "nll_loss": 1.6548840999603271,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15741074085235596,
      "rewards/margins": 0.02825068309903145,
      "rewards/rejected": -0.1856614202260971,
      "step": 309
    },
    {
      "epoch": 0.48857368006304175,
      "grad_norm": 0.22426706552505493,
      "learning_rate": 3.7549386895759315e-06,
      "log_odds_chosen": 0.37626713514328003,
      "log_odds_ratio": -0.5249163508415222,
      "logits/chosen": 0.11323234438896179,
      "logits/rejected": -1.1059436798095703,
      "logps/chosen": -1.5328314304351807,
      "logps/rejected": -1.8388895988464355,
      "loss": 1.656,
      "nll_loss": 1.6035394668579102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15328313410282135,
      "rewards/margins": 0.030605828389525414,
      "rewards/rejected": -0.18388895690441132,
      "step": 310
    },
    {
      "epoch": 0.49014972419227737,
      "grad_norm": 0.21954846382141113,
      "learning_rate": 3.7522954749893086e-06,
      "log_odds_chosen": 0.5949737429618835,
      "log_odds_ratio": -0.45127981901168823,
      "logits/chosen": 0.03048582188785076,
      "logits/rejected": -1.2382416725158691,
      "logps/chosen": -1.5583537817001343,
      "logps/rejected": -2.053128719329834,
      "loss": 1.6949,
      "nll_loss": 1.6497327089309692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1558353751897812,
      "rewards/margins": 0.049477506428956985,
      "rewards/rejected": -0.20531287789344788,
      "step": 311
    },
    {
      "epoch": 0.491725768321513,
      "grad_norm": 0.24479185044765472,
      "learning_rate": 3.749639022647332e-06,
      "log_odds_chosen": 0.3550751507282257,
      "log_odds_ratio": -0.535753071308136,
      "logits/chosen": 0.09609371423721313,
      "logits/rejected": -1.0945216417312622,
      "logps/chosen": -1.6470805406570435,
      "logps/rejected": -1.9446109533309937,
      "loss": 1.7724,
      "nll_loss": 1.7188349962234497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16470806300640106,
      "rewards/margins": 0.029753029346466064,
      "rewards/rejected": -0.19446110725402832,
      "step": 312
    },
    {
      "epoch": 0.4933018124507486,
      "grad_norm": 0.21343177556991577,
      "learning_rate": 3.7469693526182304e-06,
      "log_odds_chosen": 0.507236659526825,
      "log_odds_ratio": -0.4877496659755707,
      "logits/chosen": 0.05133073404431343,
      "logits/rejected": -1.505554437637329,
      "logps/chosen": -1.5942163467407227,
      "logps/rejected": -2.0157418251037598,
      "loss": 1.7043,
      "nll_loss": 1.6554750204086304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1594216376543045,
      "rewards/margins": 0.04215255007147789,
      "rewards/rejected": -0.2015742063522339,
      "step": 313
    },
    {
      "epoch": 0.49487785657998423,
      "grad_norm": 0.22851817309856415,
      "learning_rate": 3.744286485070085e-06,
      "log_odds_chosen": 0.5064301490783691,
      "log_odds_ratio": -0.4742722511291504,
      "logits/chosen": 0.0719347670674324,
      "logits/rejected": -1.217496633529663,
      "logps/chosen": -1.5423535108566284,
      "logps/rejected": -1.9602735042572021,
      "loss": 1.6577,
      "nll_loss": 1.6102428436279297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1542353630065918,
      "rewards/margins": 0.041791994124650955,
      "rewards/rejected": -0.19602736830711365,
      "step": 314
    },
    {
      "epoch": 0.49645390070921985,
      "grad_norm": 0.2481304556131363,
      "learning_rate": 3.7415904402706795e-06,
      "log_odds_chosen": 0.41338393092155457,
      "log_odds_ratio": -0.5167226791381836,
      "logits/chosen": 0.10534465312957764,
      "logits/rejected": -1.2418715953826904,
      "logps/chosen": -1.742213249206543,
      "logps/rejected": -2.095057964324951,
      "loss": 1.8526,
      "nll_loss": 1.8009579181671143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17422132194042206,
      "rewards/margins": 0.03528450429439545,
      "rewards/rejected": -0.2095058262348175,
      "step": 315
    },
    {
      "epoch": 0.4980299448384555,
      "grad_norm": 0.22114680707454681,
      "learning_rate": 3.7388812385873435e-06,
      "log_odds_chosen": 0.35585927963256836,
      "log_odds_ratio": -0.5332568883895874,
      "logits/chosen": 0.07114619016647339,
      "logits/rejected": -1.2164775133132935,
      "logps/chosen": -1.577876091003418,
      "logps/rejected": -1.8701857328414917,
      "loss": 1.6942,
      "nll_loss": 1.6408575773239136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15778760612010956,
      "rewards/margins": 0.029230959713459015,
      "rewards/rejected": -0.18701857328414917,
      "step": 316
    },
    {
      "epoch": 0.4996059889676911,
      "grad_norm": 0.24303023517131805,
      "learning_rate": 3.7361589004868033e-06,
      "log_odds_chosen": 0.4969717562198639,
      "log_odds_ratio": -0.4814871549606323,
      "logits/chosen": 0.17601065337657928,
      "logits/rejected": -1.2875845432281494,
      "logps/chosen": -1.6603622436523438,
      "logps/rejected": -2.0823590755462646,
      "loss": 1.7619,
      "nll_loss": 1.7137192487716675,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1660362333059311,
      "rewards/margins": 0.04219968616962433,
      "rewards/rejected": -0.20823588967323303,
      "step": 317
    },
    {
      "epoch": 0.5011820330969267,
      "grad_norm": 0.22220100462436676,
      "learning_rate": 3.733423446535022e-06,
      "log_odds_chosen": 0.5156369209289551,
      "log_odds_ratio": -0.4770677089691162,
      "logits/chosen": 0.1485612392425537,
      "logits/rejected": -1.373117446899414,
      "logps/chosen": -1.6504617929458618,
      "logps/rejected": -2.0905652046203613,
      "loss": 1.7536,
      "nll_loss": 1.705846905708313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16504618525505066,
      "rewards/margins": 0.04401035234332085,
      "rewards/rejected": -0.2090565413236618,
      "step": 318
    },
    {
      "epoch": 0.5027580772261623,
      "grad_norm": 0.21169239282608032,
      "learning_rate": 3.7306748973970476e-06,
      "log_odds_chosen": 0.3826340436935425,
      "log_odds_ratio": -0.529701292514801,
      "logits/chosen": 0.07297363877296448,
      "logits/rejected": -1.5029369592666626,
      "logps/chosen": -1.5217207670211792,
      "logps/rejected": -1.831828236579895,
      "loss": 1.6678,
      "nll_loss": 1.6148674488067627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15217207372188568,
      "rewards/margins": 0.031010746955871582,
      "rewards/rejected": -0.18318283557891846,
      "step": 319
    },
    {
      "epoch": 0.5043341213553979,
      "grad_norm": 0.21747496724128723,
      "learning_rate": 3.7279132738368564e-06,
      "log_odds_chosen": 0.18434260785579681,
      "log_odds_ratio": -0.6090730428695679,
      "logits/chosen": 0.13470004498958588,
      "logits/rejected": -1.289181113243103,
      "logps/chosen": -1.7176737785339355,
      "logps/rejected": -1.8703434467315674,
      "loss": 1.8286,
      "nll_loss": 1.7676782608032227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17176738381385803,
      "rewards/margins": 0.015266966074705124,
      "rewards/rejected": -0.18703433871269226,
      "step": 320
    },
    {
      "epoch": 0.5059101654846335,
      "grad_norm": 0.22482265532016754,
      "learning_rate": 3.725138596717195e-06,
      "log_odds_chosen": 0.4353731870651245,
      "log_odds_ratio": -0.5029094815254211,
      "logits/chosen": 0.05846483260393143,
      "logits/rejected": -1.3245011568069458,
      "logps/chosen": -1.5730060338974,
      "logps/rejected": -1.933168888092041,
      "loss": 1.6842,
      "nll_loss": 1.6339199542999268,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15730062127113342,
      "rewards/margins": 0.03601628169417381,
      "rewards/rejected": -0.19331689178943634,
      "step": 321
    },
    {
      "epoch": 0.5074862096138691,
      "grad_norm": 0.23347902297973633,
      "learning_rate": 3.7223508869994244e-06,
      "log_odds_chosen": 0.4694395065307617,
      "log_odds_ratio": -0.4926353096961975,
      "logits/chosen": 0.08627428859472275,
      "logits/rejected": -1.3811330795288086,
      "logps/chosen": -1.6119287014007568,
      "logps/rejected": -2.0031211376190186,
      "loss": 1.7265,
      "nll_loss": 1.6772775650024414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1611928790807724,
      "rewards/margins": 0.03911925479769707,
      "rewards/rejected": -0.20031213760375977,
      "step": 322
    },
    {
      "epoch": 0.5090622537431048,
      "grad_norm": 0.24362680315971375,
      "learning_rate": 3.7195501657433594e-06,
      "log_odds_chosen": 0.3797358572483063,
      "log_odds_ratio": -0.5255128741264343,
      "logits/chosen": 0.03265100345015526,
      "logits/rejected": -1.3387643098831177,
      "logps/chosen": -1.6188209056854248,
      "logps/rejected": -1.9326517581939697,
      "loss": 1.7415,
      "nll_loss": 1.6889467239379883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16188210248947144,
      "rewards/margins": 0.03138309717178345,
      "rewards/rejected": -0.19326519966125488,
      "step": 323
    },
    {
      "epoch": 0.5106382978723404,
      "grad_norm": 0.21358193457126617,
      "learning_rate": 3.716736454107111e-06,
      "log_odds_chosen": 0.5095394253730774,
      "log_odds_ratio": -0.4752754867076874,
      "logits/chosen": 0.04177181422710419,
      "logits/rejected": -1.0887494087219238,
      "logps/chosen": -1.5133510828018188,
      "logps/rejected": -1.9286960363388062,
      "loss": 1.6277,
      "nll_loss": 1.5801358222961426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15133510529994965,
      "rewards/margins": 0.04153449088335037,
      "rewards/rejected": -0.19286958873271942,
      "step": 324
    },
    {
      "epoch": 0.512214342001576,
      "grad_norm": 0.23502080142498016,
      "learning_rate": 3.7139097733469277e-06,
      "log_odds_chosen": 0.5284742712974548,
      "log_odds_ratio": -0.4676551818847656,
      "logits/chosen": 0.05847911164164543,
      "logits/rejected": -1.3190383911132812,
      "logps/chosen": -1.5592632293701172,
      "logps/rejected": -1.9967985153198242,
      "loss": 1.677,
      "nll_loss": 1.6302053928375244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15592631697654724,
      "rewards/margins": 0.04375353828072548,
      "rewards/rejected": -0.19967986643314362,
      "step": 325
    },
    {
      "epoch": 0.5137903861308116,
      "grad_norm": 0.23256242275238037,
      "learning_rate": 3.711070144817032e-06,
      "log_odds_chosen": 0.4173673391342163,
      "log_odds_ratio": -0.5119081735610962,
      "logits/chosen": 0.047142788767814636,
      "logits/rejected": -1.1630587577819824,
      "logps/chosen": -1.6483253240585327,
      "logps/rejected": -2.0011093616485596,
      "loss": 1.7695,
      "nll_loss": 1.718288779258728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16483254730701447,
      "rewards/margins": 0.035278379917144775,
      "rewards/rejected": -0.20011092722415924,
      "step": 326
    },
    {
      "epoch": 0.5153664302600472,
      "grad_norm": 0.23888525366783142,
      "learning_rate": 3.708217589969461e-06,
      "log_odds_chosen": 0.5111429691314697,
      "log_odds_ratio": -0.4780721664428711,
      "logits/chosen": 0.0068323384039103985,
      "logits/rejected": -1.3913313150405884,
      "logps/chosen": -1.621565818786621,
      "logps/rejected": -2.0499422550201416,
      "loss": 1.7462,
      "nll_loss": 1.6983516216278076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1621565967798233,
      "rewards/margins": 0.04283764958381653,
      "rewards/rejected": -0.20499424636363983,
      "step": 327
    },
    {
      "epoch": 0.5169424743892829,
      "grad_norm": 0.23694045841693878,
      "learning_rate": 3.705352130353904e-06,
      "log_odds_chosen": 0.45277297496795654,
      "log_odds_ratio": -0.5033365488052368,
      "logits/chosen": 0.07880112528800964,
      "logits/rejected": -1.3383313417434692,
      "logps/chosen": -1.539336085319519,
      "logps/rejected": -1.9113584756851196,
      "loss": 1.6711,
      "nll_loss": 1.6207761764526367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15393361449241638,
      "rewards/margins": 0.03720225393772125,
      "rewards/rejected": -0.19113586843013763,
      "step": 328
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 0.22416533529758453,
      "learning_rate": 3.7024737876175404e-06,
      "log_odds_chosen": 0.4236825108528137,
      "log_odds_ratio": -0.5054708123207092,
      "logits/chosen": 0.021605733782052994,
      "logits/rejected": -1.1384456157684326,
      "logps/chosen": -1.5932283401489258,
      "logps/rejected": -1.9415353536605835,
      "loss": 1.7001,
      "nll_loss": 1.6495964527130127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1593228429555893,
      "rewards/margins": 0.03483070060610771,
      "rewards/rejected": -0.1941535323858261,
      "step": 329
    },
    {
      "epoch": 0.5200945626477541,
      "grad_norm": 0.23309147357940674,
      "learning_rate": 3.699582583504874e-06,
      "log_odds_chosen": 0.2958032786846161,
      "log_odds_ratio": -0.5629885196685791,
      "logits/chosen": 0.07190592586994171,
      "logits/rejected": -1.3420836925506592,
      "logps/chosen": -1.7306642532348633,
      "logps/rejected": -1.9801539182662964,
      "loss": 1.8318,
      "nll_loss": 1.7754981517791748,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1730664223432541,
      "rewards/margins": 0.02494896575808525,
      "rewards/rejected": -0.19801540672779083,
      "step": 330
    },
    {
      "epoch": 0.5216706067769897,
      "grad_norm": 0.21672366559505463,
      "learning_rate": 3.696678539857571e-06,
      "log_odds_chosen": 0.38817960023880005,
      "log_odds_ratio": -0.5281521677970886,
      "logits/chosen": 0.02449723891913891,
      "logits/rejected": -1.3222875595092773,
      "logps/chosen": -1.7110434770584106,
      "logps/rejected": -2.0422046184539795,
      "loss": 1.8193,
      "nll_loss": 1.766467571258545,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17110435664653778,
      "rewards/margins": 0.03311609849333763,
      "rewards/rejected": -0.20422044396400452,
      "step": 331
    },
    {
      "epoch": 0.5232466509062254,
      "grad_norm": 0.22159616649150848,
      "learning_rate": 3.6937616786142956e-06,
      "log_odds_chosen": 0.5169373750686646,
      "log_odds_ratio": -0.47212016582489014,
      "logits/chosen": 0.048412833362817764,
      "logits/rejected": -1.0862215757369995,
      "logps/chosen": -1.5307966470718384,
      "logps/rejected": -1.9572421312332153,
      "loss": 1.6629,
      "nll_loss": 1.6156394481658936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15307967364788055,
      "rewards/margins": 0.04264454543590546,
      "rewards/rejected": -0.1957242339849472,
      "step": 332
    },
    {
      "epoch": 0.524822695035461,
      "grad_norm": 0.19886189699172974,
      "learning_rate": 3.6908320218105393e-06,
      "log_odds_chosen": 0.32757923007011414,
      "log_odds_ratio": -0.5524694919586182,
      "logits/chosen": 0.028493206948041916,
      "logits/rejected": -1.2768640518188477,
      "logps/chosen": -1.5223734378814697,
      "logps/rejected": -1.792266845703125,
      "loss": 1.637,
      "nll_loss": 1.5817255973815918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15223734080791473,
      "rewards/margins": 0.02698933705687523,
      "rewards/rejected": -0.17922668159008026,
      "step": 333
    },
    {
      "epoch": 0.5263987391646966,
      "grad_norm": 0.1958778351545334,
      "learning_rate": 3.6878895915784607e-06,
      "log_odds_chosen": 0.4811401963233948,
      "log_odds_ratio": -0.4867932200431824,
      "logits/chosen": 0.06704150140285492,
      "logits/rejected": -1.2202537059783936,
      "logps/chosen": -1.4886175394058228,
      "logps/rejected": -1.8793421983718872,
      "loss": 1.5997,
      "nll_loss": 1.5510001182556152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14886175096035004,
      "rewards/margins": 0.039072465151548386,
      "rewards/rejected": -0.18793421983718872,
      "step": 334
    },
    {
      "epoch": 0.5279747832939322,
      "grad_norm": 0.2485465407371521,
      "learning_rate": 3.6849344101467147e-06,
      "log_odds_chosen": 0.393343985080719,
      "log_odds_ratio": -0.5225814580917358,
      "logits/chosen": 0.059318412095308304,
      "logits/rejected": -1.4150105714797974,
      "logps/chosen": -1.5716626644134521,
      "logps/rejected": -1.8979530334472656,
      "loss": 1.6926,
      "nll_loss": 1.6403700113296509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1571662873029709,
      "rewards/margins": 0.032629020512104034,
      "rewards/rejected": -0.18979531526565552,
      "step": 335
    },
    {
      "epoch": 0.5295508274231678,
      "grad_norm": 0.2148926705121994,
      "learning_rate": 3.6819664998402857e-06,
      "log_odds_chosen": 0.3840746283531189,
      "log_odds_ratio": -0.5230731964111328,
      "logits/chosen": 0.03537122905254364,
      "logits/rejected": -1.249131441116333,
      "logps/chosen": -1.6128778457641602,
      "logps/rejected": -1.9333324432373047,
      "loss": 1.7263,
      "nll_loss": 1.6739463806152344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16128778457641602,
      "rewards/margins": 0.03204546496272087,
      "rewards/rejected": -0.19333325326442719,
      "step": 336
    },
    {
      "epoch": 0.5311268715524035,
      "grad_norm": 0.22502197325229645,
      "learning_rate": 3.6789858830803186e-06,
      "log_odds_chosen": 0.4095141291618347,
      "log_odds_ratio": -0.5146152973175049,
      "logits/chosen": 0.054265327751636505,
      "logits/rejected": -1.2500349283218384,
      "logps/chosen": -1.6632660627365112,
      "logps/rejected": -2.0055625438690186,
      "loss": 1.7668,
      "nll_loss": 1.7153334617614746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1663266271352768,
      "rewards/margins": 0.03422961384057999,
      "rewards/rejected": -0.20055623352527618,
      "step": 337
    },
    {
      "epoch": 0.5327029156816391,
      "grad_norm": 0.21749760210514069,
      "learning_rate": 3.6759925823839486e-06,
      "log_odds_chosen": 0.3231702446937561,
      "log_odds_ratio": -0.549367368221283,
      "logits/chosen": 0.019689474254846573,
      "logits/rejected": -1.1351220607757568,
      "logps/chosen": -1.5444843769073486,
      "logps/rejected": -1.8077837228775024,
      "loss": 1.6811,
      "nll_loss": 1.6261268854141235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15444843471050262,
      "rewards/margins": 0.026329942047595978,
      "rewards/rejected": -0.180778369307518,
      "step": 338
    },
    {
      "epoch": 0.5342789598108747,
      "grad_norm": 0.2007884979248047,
      "learning_rate": 3.672986620364134e-06,
      "log_odds_chosen": 0.4424893260002136,
      "log_odds_ratio": -0.4993942081928253,
      "logits/chosen": 0.06502474844455719,
      "logits/rejected": -1.1232551336288452,
      "logps/chosen": -1.6207716464996338,
      "logps/rejected": -1.9894078969955444,
      "loss": 1.7286,
      "nll_loss": 1.6786723136901855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1620771735906601,
      "rewards/margins": 0.03686363995075226,
      "rewards/rejected": -0.19894081354141235,
      "step": 339
    },
    {
      "epoch": 0.5358550039401103,
      "grad_norm": 0.22352111339569092,
      "learning_rate": 3.669968019729481e-06,
      "log_odds_chosen": 0.5362953543663025,
      "log_odds_ratio": -0.4702385365962982,
      "logits/chosen": 0.00828593224287033,
      "logits/rejected": -1.415726661682129,
      "logps/chosen": -1.4964334964752197,
      "logps/rejected": -1.938150405883789,
      "loss": 1.6141,
      "nll_loss": 1.5671132802963257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14964337646961212,
      "rewards/margins": 0.044171661138534546,
      "rewards/rejected": -0.19381502270698547,
      "step": 340
    },
    {
      "epoch": 0.5374310480693459,
      "grad_norm": 0.21597124636173248,
      "learning_rate": 3.666936803284076e-06,
      "log_odds_chosen": 0.4687511920928955,
      "log_odds_ratio": -0.4885876774787903,
      "logits/chosen": 0.045443110167980194,
      "logits/rejected": -1.2455968856811523,
      "logps/chosen": -1.638263463973999,
      "logps/rejected": -2.0333070755004883,
      "loss": 1.7327,
      "nll_loss": 1.6838411092758179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1638263612985611,
      "rewards/margins": 0.039504360407590866,
      "rewards/rejected": -0.20333071053028107,
      "step": 341
    },
    {
      "epoch": 0.5390070921985816,
      "grad_norm": 0.2129712551832199,
      "learning_rate": 3.663892993927312e-06,
      "log_odds_chosen": 0.5479106903076172,
      "log_odds_ratio": -0.46075499057769775,
      "logits/chosen": -0.023554421961307526,
      "logits/rejected": -1.3818119764328003,
      "logps/chosen": -1.5896512269973755,
      "logps/rejected": -2.0474183559417725,
      "loss": 1.707,
      "nll_loss": 1.6609206199645996,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1589651256799698,
      "rewards/margins": 0.04577670246362686,
      "rewards/rejected": -0.20474180579185486,
      "step": 342
    },
    {
      "epoch": 0.5405831363278172,
      "grad_norm": 0.20988072454929352,
      "learning_rate": 3.6608366146537136e-06,
      "log_odds_chosen": 0.6004074215888977,
      "log_odds_ratio": -0.4560423493385315,
      "logits/chosen": -0.054608047008514404,
      "logits/rejected": -1.2286299467086792,
      "logps/chosen": -1.5147725343704224,
      "logps/rejected": -2.0184264183044434,
      "loss": 1.6343,
      "nll_loss": 1.5887430906295776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15147726237773895,
      "rewards/margins": 0.050365399569272995,
      "rewards/rejected": -0.20184265077114105,
      "step": 343
    },
    {
      "epoch": 0.5421591804570528,
      "grad_norm": 0.21432410180568695,
      "learning_rate": 3.6577676885527674e-06,
      "log_odds_chosen": 0.46950504183769226,
      "log_odds_ratio": -0.49097535014152527,
      "logits/chosen": -0.030298583209514618,
      "logits/rejected": -1.479987382888794,
      "logps/chosen": -1.5775291919708252,
      "logps/rejected": -1.9668179750442505,
      "loss": 1.6785,
      "nll_loss": 1.6294326782226562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15775293111801147,
      "rewards/margins": 0.03892885893583298,
      "rewards/rejected": -0.19668179750442505,
      "step": 344
    },
    {
      "epoch": 0.5437352245862884,
      "grad_norm": 0.20334196090698242,
      "learning_rate": 3.654686238808744e-06,
      "log_odds_chosen": 0.45373964309692383,
      "log_odds_ratio": -0.4997464418411255,
      "logits/chosen": -0.05928083136677742,
      "logits/rejected": -1.4065111875534058,
      "logps/chosen": -1.6670969724655151,
      "logps/rejected": -2.049161434173584,
      "loss": 1.7703,
      "nll_loss": 1.7203750610351562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16670972108840942,
      "rewards/margins": 0.038206446915864944,
      "rewards/rejected": -0.20491616427898407,
      "step": 345
    },
    {
      "epoch": 0.545311268715524,
      "grad_norm": 0.20314708352088928,
      "learning_rate": 3.6515922887005245e-06,
      "log_odds_chosen": 0.6570188403129578,
      "log_odds_ratio": -0.42130351066589355,
      "logits/chosen": -0.15550881624221802,
      "logits/rejected": -1.41984224319458,
      "logps/chosen": -1.5048458576202393,
      "logps/rejected": -2.044806480407715,
      "loss": 1.6173,
      "nll_loss": 1.575134515762329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1504845917224884,
      "rewards/margins": 0.05399605259299278,
      "rewards/rejected": -0.20448064804077148,
      "step": 346
    },
    {
      "epoch": 0.5468873128447597,
      "grad_norm": 0.1935390830039978,
      "learning_rate": 3.6484858616014236e-06,
      "log_odds_chosen": 0.28654223680496216,
      "log_odds_ratio": -0.5621439218521118,
      "logits/chosen": 0.017233777791261673,
      "logits/rejected": -1.4268871545791626,
      "logps/chosen": -1.6293790340423584,
      "logps/rejected": -1.8657327890396118,
      "loss": 1.7299,
      "nll_loss": 1.6737170219421387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16293790936470032,
      "rewards/margins": 0.023635366931557655,
      "rewards/rejected": -0.18657328188419342,
      "step": 347
    },
    {
      "epoch": 0.5484633569739953,
      "grad_norm": 0.2078549563884735,
      "learning_rate": 3.6453669809790154e-06,
      "log_odds_chosen": 0.3306597173213959,
      "log_odds_ratio": -0.5512881278991699,
      "logits/chosen": 0.036211688071489334,
      "logits/rejected": -1.0868035554885864,
      "logps/chosen": -1.5622117519378662,
      "logps/rejected": -1.834636926651001,
      "loss": 1.6717,
      "nll_loss": 1.616525411605835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1562211811542511,
      "rewards/margins": 0.02724250964820385,
      "rewards/rejected": -0.1834636926651001,
      "step": 348
    },
    {
      "epoch": 0.5500394011032309,
      "grad_norm": 0.20183558762073517,
      "learning_rate": 3.642235670394952e-06,
      "log_odds_chosen": 0.42099136114120483,
      "log_odds_ratio": -0.5093228220939636,
      "logits/chosen": 0.008305356837809086,
      "logits/rejected": -1.0708895921707153,
      "logps/chosen": -1.5708250999450684,
      "logps/rejected": -1.9134736061096191,
      "loss": 1.689,
      "nll_loss": 1.6381096839904785,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15708249807357788,
      "rewards/margins": 0.034264855086803436,
      "rewards/rejected": -0.19134736061096191,
      "step": 349
    },
    {
      "epoch": 0.5516154452324665,
      "grad_norm": 0.19396637380123138,
      "learning_rate": 3.63909195350479e-06,
      "log_odds_chosen": 0.40000632405281067,
      "log_odds_ratio": -0.5222204923629761,
      "logits/chosen": 0.0709347128868103,
      "logits/rejected": -1.2560487985610962,
      "logps/chosen": -1.6595628261566162,
      "logps/rejected": -1.9919466972351074,
      "loss": 1.7618,
      "nll_loss": 1.7095361948013306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1659562885761261,
      "rewards/margins": 0.03323838487267494,
      "rewards/rejected": -0.19919466972351074,
      "step": 350
    },
    {
      "epoch": 0.5531914893617021,
      "grad_norm": 0.20782439410686493,
      "learning_rate": 3.635935854057809e-06,
      "log_odds_chosen": 0.39174172282218933,
      "log_odds_ratio": -0.5250329375267029,
      "logits/chosen": -0.1102612316608429,
      "logits/rejected": -0.9730474948883057,
      "logps/chosen": -1.6109540462493896,
      "logps/rejected": -1.9376519918441772,
      "loss": 1.7198,
      "nll_loss": 1.6673376560211182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16109541058540344,
      "rewards/margins": 0.032669797539711,
      "rewards/rejected": -0.19376519322395325,
      "step": 351
    },
    {
      "epoch": 0.5547675334909378,
      "grad_norm": 0.19801416993141174,
      "learning_rate": 3.6327673958968327e-06,
      "log_odds_chosen": 0.5151069760322571,
      "log_odds_ratio": -0.47180619835853577,
      "logits/chosen": -0.06693591177463531,
      "logits/rejected": -1.3027480840682983,
      "logps/chosen": -1.4905201196670532,
      "logps/rejected": -1.9089621305465698,
      "loss": 1.6179,
      "nll_loss": 1.570717215538025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14905200898647308,
      "rewards/margins": 0.0418442040681839,
      "rewards/rejected": -0.19089621305465698,
      "step": 352
    },
    {
      "epoch": 0.5563435776201734,
      "grad_norm": 0.18420955538749695,
      "learning_rate": 3.6295866029580483e-06,
      "log_odds_chosen": 0.49202004075050354,
      "log_odds_ratio": -0.4803290367126465,
      "logits/chosen": -0.0715307965874672,
      "logits/rejected": -1.3764203786849976,
      "logps/chosen": -1.5352476835250854,
      "logps/rejected": -1.938019037246704,
      "loss": 1.6324,
      "nll_loss": 1.5843286514282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15352477133274078,
      "rewards/margins": 0.04027712717652321,
      "rewards/rejected": -0.1938018947839737,
      "step": 353
    },
    {
      "epoch": 0.557919621749409,
      "grad_norm": 0.19222892820835114,
      "learning_rate": 3.626393499270829e-06,
      "log_odds_chosen": 0.44654372334480286,
      "log_odds_ratio": -0.5027562975883484,
      "logits/chosen": -0.057136163115501404,
      "logits/rejected": -1.3216649293899536,
      "logps/chosen": -1.473515510559082,
      "logps/rejected": -1.8351870775222778,
      "loss": 1.6025,
      "nll_loss": 1.552269458770752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735153317451477,
      "rewards/margins": 0.03616715967655182,
      "rewards/rejected": -0.18351870775222778,
      "step": 354
    },
    {
      "epoch": 0.5594956658786446,
      "grad_norm": 0.198707714676857,
      "learning_rate": 3.6231881089575466e-06,
      "log_odds_chosen": 0.6093676686286926,
      "log_odds_ratio": -0.437423974275589,
      "logits/chosen": -0.1271413415670395,
      "logits/rejected": -1.4303815364837646,
      "logps/chosen": -1.5215667486190796,
      "logps/rejected": -2.027055263519287,
      "loss": 1.6059,
      "nll_loss": 1.5621567964553833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15215668082237244,
      "rewards/margins": 0.05054884031414986,
      "rewards/rejected": -0.202705517411232,
      "step": 355
    },
    {
      "epoch": 0.5610717100078803,
      "grad_norm": 0.26446157693862915,
      "learning_rate": 3.6199704562333945e-06,
      "log_odds_chosen": 0.4508443772792816,
      "log_odds_ratio": -0.5002342462539673,
      "logits/chosen": -0.09892217069864273,
      "logits/rejected": -1.3460979461669922,
      "logps/chosen": -1.494001030921936,
      "logps/rejected": -1.8570603132247925,
      "loss": 1.604,
      "nll_loss": 1.553969383239746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14940010011196136,
      "rewards/margins": 0.03630594164133072,
      "rewards/rejected": -0.18570604920387268,
      "step": 356
    },
    {
      "epoch": 0.5626477541371159,
      "grad_norm": 0.20473134517669678,
      "learning_rate": 3.6167405654062024e-06,
      "log_odds_chosen": 0.4710828363895416,
      "log_odds_ratio": -0.4989466369152069,
      "logits/chosen": -0.04282144084572792,
      "logits/rejected": -1.2687091827392578,
      "logps/chosen": -1.5530476570129395,
      "logps/rejected": -1.9394475221633911,
      "loss": 1.6523,
      "nll_loss": 1.602416753768921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15530475974082947,
      "rewards/margins": 0.03863998502492905,
      "rewards/rejected": -0.19394473731517792,
      "step": 357
    },
    {
      "epoch": 0.5642237982663515,
      "grad_norm": 0.18403887748718262,
      "learning_rate": 3.6134984608762515e-06,
      "log_odds_chosen": 0.5030588507652283,
      "log_odds_ratio": -0.476035475730896,
      "logits/chosen": -0.05343765392899513,
      "logits/rejected": -1.3506138324737549,
      "logps/chosen": -1.4735472202301025,
      "logps/rejected": -1.8805701732635498,
      "loss": 1.5937,
      "nll_loss": 1.546113133430481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14735473692417145,
      "rewards/margins": 0.04070229083299637,
      "rewards/rejected": -0.1880570352077484,
      "step": 358
    },
    {
      "epoch": 0.5657998423955871,
      "grad_norm": 0.21042759716510773,
      "learning_rate": 3.6102441671360945e-06,
      "log_odds_chosen": 0.48499730229377747,
      "log_odds_ratio": -0.49012240767478943,
      "logits/chosen": -0.0872420072555542,
      "logits/rejected": -1.223256230354309,
      "logps/chosen": -1.6216155290603638,
      "logps/rejected": -2.0279018878936768,
      "loss": 1.7136,
      "nll_loss": 1.6645991802215576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16216154396533966,
      "rewards/margins": 0.04062866419553757,
      "rewards/rejected": -0.20279023051261902,
      "step": 359
    },
    {
      "epoch": 0.5673758865248227,
      "grad_norm": 0.199711412191391,
      "learning_rate": 3.6069777087703654e-06,
      "log_odds_chosen": 0.5050286650657654,
      "log_odds_ratio": -0.47864657640457153,
      "logits/chosen": -0.06602545827627182,
      "logits/rejected": -1.2074447870254517,
      "logps/chosen": -1.5410226583480835,
      "logps/rejected": -1.956892728805542,
      "loss": 1.6585,
      "nll_loss": 1.610638976097107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15410228073596954,
      "rewards/margins": 0.04158700257539749,
      "rewards/rejected": -0.19568926095962524,
      "step": 360
    },
    {
      "epoch": 0.5689519306540584,
      "grad_norm": 0.20484104752540588,
      "learning_rate": 3.6036991104555973e-06,
      "log_odds_chosen": 0.4478030502796173,
      "log_odds_ratio": -0.5068904757499695,
      "logits/chosen": -0.04828740656375885,
      "logits/rejected": -1.331907033920288,
      "logps/chosen": -1.6810355186462402,
      "logps/rejected": -2.061274290084839,
      "loss": 1.771,
      "nll_loss": 1.720302939414978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16810354590415955,
      "rewards/margins": 0.038023896515369415,
      "rewards/rejected": -0.20612744987010956,
      "step": 361
    },
    {
      "epoch": 0.570527974783294,
      "grad_norm": 0.194278284907341,
      "learning_rate": 3.600408396960034e-06,
      "log_odds_chosen": 0.5042534470558167,
      "log_odds_ratio": -0.4804871380329132,
      "logits/chosen": -0.02915314957499504,
      "logits/rejected": -0.9427077174186707,
      "logps/chosen": -1.4871515035629272,
      "logps/rejected": -1.8912396430969238,
      "loss": 1.582,
      "nll_loss": 1.533990502357483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14871516823768616,
      "rewards/margins": 0.04040880501270294,
      "rewards/rejected": -0.1891239732503891,
      "step": 362
    },
    {
      "epoch": 0.5721040189125296,
      "grad_norm": 0.1945444643497467,
      "learning_rate": 3.5971055931434447e-06,
      "log_odds_chosen": 0.43649476766586304,
      "log_odds_ratio": -0.506388783454895,
      "logits/chosen": -0.02089090645313263,
      "logits/rejected": -1.2006688117980957,
      "logps/chosen": -1.536198377609253,
      "logps/rejected": -1.8970880508422852,
      "loss": 1.6438,
      "nll_loss": 1.5931452512741089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15361982583999634,
      "rewards/margins": 0.036088977009058,
      "rewards/rejected": -0.18970881402492523,
      "step": 363
    },
    {
      "epoch": 0.5736800630417652,
      "grad_norm": 0.18837140500545502,
      "learning_rate": 3.5937907239569343e-06,
      "log_odds_chosen": 0.4648410677909851,
      "log_odds_ratio": -0.4977383613586426,
      "logits/chosen": -0.09922034293413162,
      "logits/rejected": -1.3353502750396729,
      "logps/chosen": -1.5880086421966553,
      "logps/rejected": -1.9711687564849854,
      "loss": 1.6969,
      "nll_loss": 1.647141456604004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15880087018013,
      "rewards/margins": 0.03831600397825241,
      "rewards/rejected": -0.197116881608963,
      "step": 364
    },
    {
      "epoch": 0.5752561071710008,
      "grad_norm": 0.22331880033016205,
      "learning_rate": 3.5904638144427572e-06,
      "log_odds_chosen": 0.2761245369911194,
      "log_odds_ratio": -0.5722551941871643,
      "logits/chosen": -0.0540853813290596,
      "logits/rejected": -1.0778639316558838,
      "logps/chosen": -1.6464744806289673,
      "logps/rejected": -1.874767780303955,
      "loss": 1.7439,
      "nll_loss": 1.6866995096206665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1646474450826645,
      "rewards/margins": 0.022829342633485794,
      "rewards/rejected": -0.18747679889202118,
      "step": 365
    },
    {
      "epoch": 0.5768321513002365,
      "grad_norm": 0.1964210569858551,
      "learning_rate": 3.5871248897341246e-06,
      "log_odds_chosen": 0.5015654563903809,
      "log_odds_ratio": -0.4797275960445404,
      "logits/chosen": -0.0888606533408165,
      "logits/rejected": -1.0420361757278442,
      "logps/chosen": -1.4869009256362915,
      "logps/rejected": -1.9011142253875732,
      "loss": 1.6095,
      "nll_loss": 1.5615687370300293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486901044845581,
      "rewards/margins": 0.04142133146524429,
      "rewards/rejected": -0.1901114135980606,
      "step": 366
    },
    {
      "epoch": 0.578408195429472,
      "grad_norm": 0.18352432548999786,
      "learning_rate": 3.5837739750550182e-06,
      "log_odds_chosen": 0.48427703976631165,
      "log_odds_ratio": -0.48840558528900146,
      "logits/chosen": -0.11341613531112671,
      "logits/rejected": -1.6361585855484009,
      "logps/chosen": -1.564819574356079,
      "logps/rejected": -1.968957543373108,
      "loss": 1.6656,
      "nll_loss": 1.6167792081832886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15648198127746582,
      "rewards/margins": 0.040413789451122284,
      "rewards/rejected": -0.1968957632780075,
      "step": 367
    },
    {
      "epoch": 0.5799842395587076,
      "grad_norm": 0.18227070569992065,
      "learning_rate": 3.5804110957199977e-06,
      "log_odds_chosen": 0.5225378274917603,
      "log_odds_ratio": -0.47141802310943604,
      "logits/chosen": 0.0010305908508598804,
      "logits/rejected": -1.1611210107803345,
      "logps/chosen": -1.5438573360443115,
      "logps/rejected": -1.9776092767715454,
      "loss": 1.6428,
      "nll_loss": 1.5957022905349731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543857306241989,
      "rewards/margins": 0.043375201523303986,
      "rewards/rejected": -0.1977609246969223,
      "step": 368
    },
    {
      "epoch": 0.5815602836879432,
      "grad_norm": 0.19467630982398987,
      "learning_rate": 3.577036277134011e-06,
      "log_odds_chosen": 0.5960920453071594,
      "log_odds_ratio": -0.4427749514579773,
      "logits/chosen": -0.08947373181581497,
      "logits/rejected": -1.3899809122085571,
      "logps/chosen": -1.5279539823532104,
      "logps/rejected": -2.0212056636810303,
      "loss": 1.6406,
      "nll_loss": 1.5962986946105957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15279540419578552,
      "rewards/margins": 0.04932515323162079,
      "rewards/rejected": -0.2021205574274063,
      "step": 369
    },
    {
      "epoch": 0.5831363278171788,
      "grad_norm": 0.20579133927822113,
      "learning_rate": 3.5736495447922e-06,
      "log_odds_chosen": 0.3752758502960205,
      "log_odds_ratio": -0.5272895693778992,
      "logits/chosen": -0.08593582361936569,
      "logits/rejected": -1.4058607816696167,
      "logps/chosen": -1.6539585590362549,
      "logps/rejected": -1.966200351715088,
      "loss": 1.7574,
      "nll_loss": 1.7046749591827393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1653958410024643,
      "rewards/margins": 0.03122418001294136,
      "rewards/rejected": -0.19662001729011536,
      "step": 370
    },
    {
      "epoch": 0.5847123719464145,
      "grad_norm": 0.17624422907829285,
      "learning_rate": 3.5702509242797096e-06,
      "log_odds_chosen": 0.6972737312316895,
      "log_odds_ratio": -0.41951215267181396,
      "logits/chosen": -0.09982432425022125,
      "logits/rejected": -1.3790462017059326,
      "logps/chosen": -1.4812397956848145,
      "logps/rejected": -2.058272361755371,
      "loss": 1.595,
      "nll_loss": 1.5530776977539062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14812399446964264,
      "rewards/margins": 0.05770324915647507,
      "rewards/rejected": -0.2058272510766983,
      "step": 371
    },
    {
      "epoch": 0.5862884160756501,
      "grad_norm": 0.18756203353405,
      "learning_rate": 3.566840441271495e-06,
      "log_odds_chosen": 0.6480004787445068,
      "log_odds_ratio": -0.4360523819923401,
      "logits/chosen": -0.05456085875630379,
      "logits/rejected": -1.45460045337677,
      "logps/chosen": -1.49797523021698,
      "logps/rejected": -2.0297532081604004,
      "loss": 1.5874,
      "nll_loss": 1.5438189506530762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14979752898216248,
      "rewards/margins": 0.053177814930677414,
      "rewards/rejected": -0.2029753178358078,
      "step": 372
    },
    {
      "epoch": 0.5878644602048857,
      "grad_norm": 0.19936488568782806,
      "learning_rate": 3.5634181215321265e-06,
      "log_odds_chosen": 0.6168785095214844,
      "log_odds_ratio": -0.43505510687828064,
      "logits/chosen": -0.044426970183849335,
      "logits/rejected": -1.275886058807373,
      "logps/chosen": -1.553296685218811,
      "logps/rejected": -2.0681307315826416,
      "loss": 1.6652,
      "nll_loss": 1.6216744184494019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15532967448234558,
      "rewards/margins": 0.051483407616615295,
      "rewards/rejected": -0.20681308209896088,
      "step": 373
    },
    {
      "epoch": 0.5894405043341213,
      "grad_norm": 0.1882179081439972,
      "learning_rate": 3.5599839909155947e-06,
      "log_odds_chosen": 0.48124241828918457,
      "log_odds_ratio": -0.4906978905200958,
      "logits/chosen": -0.06898046284914017,
      "logits/rejected": -1.3314614295959473,
      "logps/chosen": -1.5185297727584839,
      "logps/rejected": -1.914254903793335,
      "loss": 1.6201,
      "nll_loss": 1.5710440874099731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15185298025608063,
      "rewards/margins": 0.039572522044181824,
      "rewards/rejected": -0.19142550230026245,
      "step": 374
    },
    {
      "epoch": 0.5910165484633569,
      "grad_norm": 0.18996809422969818,
      "learning_rate": 3.556538075365116e-06,
      "log_odds_chosen": 0.5556556582450867,
      "log_odds_ratio": -0.46352192759513855,
      "logits/chosen": -0.028684724122285843,
      "logits/rejected": -1.3568097352981567,
      "logps/chosen": -1.5122661590576172,
      "logps/rejected": -1.9674909114837646,
      "loss": 1.6132,
      "nll_loss": 1.566803216934204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15122662484645844,
      "rewards/margins": 0.04552246257662773,
      "rewards/rejected": -0.19674909114837646,
      "step": 375
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 0.1981869488954544,
      "learning_rate": 3.5530804009129367e-06,
      "log_odds_chosen": 0.4985346794128418,
      "log_odds_ratio": -0.4772438406944275,
      "logits/chosen": -0.07949502021074295,
      "logits/rejected": -1.1893216371536255,
      "logps/chosen": -1.548764944076538,
      "logps/rejected": -1.961025357246399,
      "loss": 1.6379,
      "nll_loss": 1.590188980102539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1548765003681183,
      "rewards/margins": 0.04122602939605713,
      "rewards/rejected": -0.19610252976417542,
      "step": 376
    },
    {
      "epoch": 0.5941686367218282,
      "grad_norm": 0.19308407604694366,
      "learning_rate": 3.5496109936801368e-06,
      "log_odds_chosen": 0.4907795190811157,
      "log_odds_ratio": -0.4855614900588989,
      "logits/chosen": -0.13920965790748596,
      "logits/rejected": -1.4071189165115356,
      "logps/chosen": -1.5704931020736694,
      "logps/rejected": -1.9831798076629639,
      "loss": 1.6681,
      "nll_loss": 1.6195642948150635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15704932808876038,
      "rewards/margins": 0.041268669068813324,
      "rewards/rejected": -0.1983179897069931,
      "step": 377
    },
    {
      "epoch": 0.5957446808510638,
      "grad_norm": 0.20303456485271454,
      "learning_rate": 3.546129879876429e-06,
      "log_odds_chosen": 0.35706377029418945,
      "log_odds_ratio": -0.5432414412498474,
      "logits/chosen": -0.023199474439024925,
      "logits/rejected": -1.086435317993164,
      "logps/chosen": -1.6312530040740967,
      "logps/rejected": -1.9287961721420288,
      "loss": 1.7262,
      "nll_loss": 1.6719212532043457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16312529146671295,
      "rewards/margins": 0.029754336923360825,
      "rewards/rejected": -0.19287963211536407,
      "step": 378
    },
    {
      "epoch": 0.5973207249802994,
      "grad_norm": 0.19225862622261047,
      "learning_rate": 3.5426370857999662e-06,
      "log_odds_chosen": 0.3647347092628479,
      "log_odds_ratio": -0.531550407409668,
      "logits/chosen": -0.07333146780729294,
      "logits/rejected": -1.2189780473709106,
      "logps/chosen": -1.6128994226455688,
      "logps/rejected": -1.9161568880081177,
      "loss": 1.7184,
      "nll_loss": 1.6652899980545044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16128993034362793,
      "rewards/margins": 0.030325744301080704,
      "rewards/rejected": -0.19161568582057953,
      "step": 379
    },
    {
      "epoch": 0.598896769109535,
      "grad_norm": 0.19579578936100006,
      "learning_rate": 3.53913263783714e-06,
      "log_odds_chosen": 0.41187989711761475,
      "log_odds_ratio": -0.5210331678390503,
      "logits/chosen": -0.1096126139163971,
      "logits/rejected": -1.3426750898361206,
      "logps/chosen": -1.5626639127731323,
      "logps/rejected": -1.9077504873275757,
      "loss": 1.6558,
      "nll_loss": 1.6037330627441406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15626640617847443,
      "rewards/margins": 0.03450864553451538,
      "rewards/rejected": -0.1907750368118286,
      "step": 380
    },
    {
      "epoch": 0.6004728132387707,
      "grad_norm": 0.18819411098957062,
      "learning_rate": 3.53561656246238e-06,
      "log_odds_chosen": 0.6348834037780762,
      "log_odds_ratio": -0.4286256432533264,
      "logits/chosen": -0.2015249878168106,
      "logits/rejected": -1.5831485986709595,
      "logps/chosen": -1.5856183767318726,
      "logps/rejected": -2.117565631866455,
      "loss": 1.6802,
      "nll_loss": 1.6373547315597534,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1585618406534195,
      "rewards/margins": 0.053194742649793625,
      "rewards/rejected": -0.21175658702850342,
      "step": 381
    },
    {
      "epoch": 0.6020488573680063,
      "grad_norm": 0.18237391114234924,
      "learning_rate": 3.532088886237956e-06,
      "log_odds_chosen": 0.5271506905555725,
      "log_odds_ratio": -0.4673537015914917,
      "logits/chosen": -0.12271787226200104,
      "logits/rejected": -1.5163284540176392,
      "logps/chosen": -1.6115859746932983,
      "logps/rejected": -2.0531668663024902,
      "loss": 1.6911,
      "nll_loss": 1.6443783044815063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16115859150886536,
      "rewards/margins": 0.04415808618068695,
      "rewards/rejected": -0.2053166925907135,
      "step": 382
    },
    {
      "epoch": 0.6036249014972419,
      "grad_norm": 0.172901913523674,
      "learning_rate": 3.528549635813778e-06,
      "log_odds_chosen": 0.5418984293937683,
      "log_odds_ratio": -0.4654310643672943,
      "logits/chosen": -0.15981490910053253,
      "logits/rejected": -1.3621118068695068,
      "logps/chosen": -1.4450870752334595,
      "logps/rejected": -1.8853989839553833,
      "loss": 1.5496,
      "nll_loss": 1.5030344724655151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1445087045431137,
      "rewards/margins": 0.04403119534254074,
      "rewards/rejected": -0.18853990733623505,
      "step": 383
    },
    {
      "epoch": 0.6052009456264775,
      "grad_norm": 0.19568444788455963,
      "learning_rate": 3.524998837927192e-06,
      "log_odds_chosen": 0.572688639163971,
      "log_odds_ratio": -0.4493398666381836,
      "logits/chosen": -0.11590529978275299,
      "logits/rejected": -1.2641996145248413,
      "logps/chosen": -1.5668047666549683,
      "logps/rejected": -2.04402756690979,
      "loss": 1.6464,
      "nll_loss": 1.6014786958694458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15668046474456787,
      "rewards/margins": 0.04772229492664337,
      "rewards/rejected": -0.20440277457237244,
      "step": 384
    },
    {
      "epoch": 0.6067769897557131,
      "grad_norm": 0.17970573902130127,
      "learning_rate": 3.5214365194027797e-06,
      "log_odds_chosen": 0.5927567481994629,
      "log_odds_ratio": -0.4437498152256012,
      "logits/chosen": -0.1304638385772705,
      "logits/rejected": -1.3869986534118652,
      "logps/chosen": -1.4864726066589355,
      "logps/rejected": -1.9743757247924805,
      "loss": 1.579,
      "nll_loss": 1.5346585512161255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1486472636461258,
      "rewards/margins": 0.04879031702876091,
      "rewards/rejected": -0.19743755459785461,
      "step": 385
    },
    {
      "epoch": 0.6083530338849488,
      "grad_norm": 0.209872767329216,
      "learning_rate": 3.517862707152157e-06,
      "log_odds_chosen": 0.43204668164253235,
      "log_odds_ratio": -0.5081393122673035,
      "logits/chosen": -0.04916413873434067,
      "logits/rejected": -1.1377692222595215,
      "logps/chosen": -1.6241986751556396,
      "logps/rejected": -1.984796166419983,
      "loss": 1.7062,
      "nll_loss": 1.6553831100463867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.162419855594635,
      "rewards/margins": 0.036059752106666565,
      "rewards/rejected": -0.19847962260246277,
      "step": 386
    },
    {
      "epoch": 0.6099290780141844,
      "grad_norm": 0.21656852960586548,
      "learning_rate": 3.5142774281737674e-06,
      "log_odds_chosen": 0.6231499314308167,
      "log_odds_ratio": -0.43155157566070557,
      "logits/chosen": -0.13221031427383423,
      "logits/rejected": -1.25809907913208,
      "logps/chosen": -1.6151927709579468,
      "logps/rejected": -2.1396982669830322,
      "loss": 1.6916,
      "nll_loss": 1.6484118700027466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16151930391788483,
      "rewards/margins": 0.05245053023099899,
      "rewards/rejected": -0.21396984159946442,
      "step": 387
    },
    {
      "epoch": 0.61150512214342,
      "grad_norm": 0.17429621517658234,
      "learning_rate": 3.5106807095526817e-06,
      "log_odds_chosen": 0.626970112323761,
      "log_odds_ratio": -0.43910783529281616,
      "logits/chosen": -0.14420942962169647,
      "logits/rejected": -1.505774974822998,
      "logps/chosen": -1.5748193264007568,
      "logps/rejected": -2.0963521003723145,
      "loss": 1.6681,
      "nll_loss": 1.62421715259552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15748193860054016,
      "rewards/margins": 0.052153266966342926,
      "rewards/rejected": -0.20963521301746368,
      "step": 388
    },
    {
      "epoch": 0.6130811662726556,
      "grad_norm": 0.18532711267471313,
      "learning_rate": 3.5070725784603905e-06,
      "log_odds_chosen": 0.5388447642326355,
      "log_odds_ratio": -0.4668474793434143,
      "logits/chosen": -0.21902410686016083,
      "logits/rejected": -1.2050061225891113,
      "logps/chosen": -1.4473650455474854,
      "logps/rejected": -1.8858047723770142,
      "loss": 1.5341,
      "nll_loss": 1.4874001741409302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14473651349544525,
      "rewards/margins": 0.04384396970272064,
      "rewards/rejected": -0.1885804831981659,
      "step": 389
    },
    {
      "epoch": 0.6146572104018913,
      "grad_norm": 0.20954233407974243,
      "learning_rate": 3.503453062154602e-06,
      "log_odds_chosen": 0.45336735248565674,
      "log_odds_ratio": -0.5033838748931885,
      "logits/chosen": -0.138980433344841,
      "logits/rejected": -1.1668821573257446,
      "logps/chosen": -1.611983060836792,
      "logps/rejected": -1.9972047805786133,
      "loss": 1.7107,
      "nll_loss": 1.6603914499282837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16119830310344696,
      "rewards/margins": 0.03852216154336929,
      "rewards/rejected": -0.19972047209739685,
      "step": 390
    },
    {
      "epoch": 0.6162332545311269,
      "grad_norm": 0.19238236546516418,
      "learning_rate": 3.499822187979032e-06,
      "log_odds_chosen": 0.45051056146621704,
      "log_odds_ratio": -0.502297043800354,
      "logits/chosen": -0.07424494624137878,
      "logits/rejected": -1.422592043876648,
      "logps/chosen": -1.5691637992858887,
      "logps/rejected": -1.9431030750274658,
      "loss": 1.6718,
      "nll_loss": 1.6215293407440186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15691637992858887,
      "rewards/margins": 0.03739393502473831,
      "rewards/rejected": -0.19431032240390778,
      "step": 391
    },
    {
      "epoch": 0.6178092986603625,
      "grad_norm": 0.1847350001335144,
      "learning_rate": 3.496179983363202e-06,
      "log_odds_chosen": 0.39740023016929626,
      "log_odds_ratio": -0.5181854963302612,
      "logits/chosen": -0.08911152184009552,
      "logits/rejected": -1.3096749782562256,
      "logps/chosen": -1.5776864290237427,
      "logps/rejected": -1.9072067737579346,
      "loss": 1.6621,
      "nll_loss": 1.610294222831726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1577686369419098,
      "rewards/margins": 0.03295203298330307,
      "rewards/rejected": -0.19072067737579346,
      "step": 392
    },
    {
      "epoch": 0.6193853427895981,
      "grad_norm": 0.20399923622608185,
      "learning_rate": 3.4925264758222268e-06,
      "log_odds_chosen": 0.6117810010910034,
      "log_odds_ratio": -0.43658775091171265,
      "logits/chosen": -0.14838597178459167,
      "logits/rejected": -1.166914939880371,
      "logps/chosen": -1.507293701171875,
      "logps/rejected": -2.0140793323516846,
      "loss": 1.6041,
      "nll_loss": 1.5604305267333984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15072937309741974,
      "rewards/margins": 0.050678566098213196,
      "rewards/rejected": -0.20140793919563293,
      "step": 393
    },
    {
      "epoch": 0.6209613869188337,
      "grad_norm": 0.38816723227500916,
      "learning_rate": 3.488861692956611e-06,
      "log_odds_chosen": 0.5418617129325867,
      "log_odds_ratio": -0.46506166458129883,
      "logits/chosen": -0.1680249571800232,
      "logits/rejected": -1.281784176826477,
      "logps/chosen": -1.5487433671951294,
      "logps/rejected": -2.0000991821289062,
      "loss": 1.6392,
      "nll_loss": 1.5926973819732666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15487435460090637,
      "rewards/margins": 0.04513557255268097,
      "rewards/rejected": -0.20000992715358734,
      "step": 394
    },
    {
      "epoch": 0.6225374310480694,
      "grad_norm": 0.1981612592935562,
      "learning_rate": 3.4851856624520394e-06,
      "log_odds_chosen": 0.5252317786216736,
      "log_odds_ratio": -0.4674740135669708,
      "logits/chosen": -0.1143113300204277,
      "logits/rejected": -1.3392878770828247,
      "logps/chosen": -1.5815508365631104,
      "logps/rejected": -2.017178773880005,
      "loss": 1.6669,
      "nll_loss": 1.620198130607605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15815508365631104,
      "rewards/margins": 0.04356277734041214,
      "rewards/rejected": -0.20171786844730377,
      "step": 395
    },
    {
      "epoch": 0.624113475177305,
      "grad_norm": 0.20509879291057587,
      "learning_rate": 3.4814984120791664e-06,
      "log_odds_chosen": 0.497008353471756,
      "log_odds_ratio": -0.47801053524017334,
      "logits/chosen": -0.11156664043664932,
      "logits/rejected": -1.1910896301269531,
      "logps/chosen": -1.553000807762146,
      "logps/rejected": -1.962215542793274,
      "loss": 1.6485,
      "nll_loss": 1.6007264852523804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1553000807762146,
      "rewards/margins": 0.04092148691415787,
      "rewards/rejected": -0.19622156023979187,
      "step": 396
    },
    {
      "epoch": 0.6256895193065406,
      "grad_norm": 0.18052570521831512,
      "learning_rate": 3.477799969693407e-06,
      "log_odds_chosen": 0.4731709361076355,
      "log_odds_ratio": -0.49316927790641785,
      "logits/chosen": -0.07338425517082214,
      "logits/rejected": -1.334223747253418,
      "logps/chosen": -1.5508506298065186,
      "logps/rejected": -1.936098575592041,
      "loss": 1.6401,
      "nll_loss": 1.5908257961273193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15508508682250977,
      "rewards/margins": 0.03852478042244911,
      "rewards/rejected": -0.19360986351966858,
      "step": 397
    },
    {
      "epoch": 0.6272655634357762,
      "grad_norm": 0.20021112263202667,
      "learning_rate": 3.474090363234728e-06,
      "log_odds_chosen": 0.7609047889709473,
      "log_odds_ratio": -0.3980966806411743,
      "logits/chosen": -0.24316024780273438,
      "logits/rejected": -1.3844587802886963,
      "logps/chosen": -1.4845788478851318,
      "logps/rejected": -2.117990016937256,
      "loss": 1.5681,
      "nll_loss": 1.528298020362854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14845788478851318,
      "rewards/margins": 0.06334113329648972,
      "rewards/rejected": -0.2117990255355835,
      "step": 398
    },
    {
      "epoch": 0.6288416075650118,
      "grad_norm": 0.19712619483470917,
      "learning_rate": 3.4703696207274325e-06,
      "log_odds_chosen": 0.5112850666046143,
      "log_odds_ratio": -0.4771292209625244,
      "logits/chosen": -0.12579302489757538,
      "logits/rejected": -1.206828236579895,
      "logps/chosen": -1.5673775672912598,
      "logps/rejected": -1.9893169403076172,
      "loss": 1.6413,
      "nll_loss": 1.593619704246521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15673774480819702,
      "rewards/margins": 0.0421939492225647,
      "rewards/rejected": -0.19893169403076172,
      "step": 399
    },
    {
      "epoch": 0.6304176516942475,
      "grad_norm": 0.18029263615608215,
      "learning_rate": 3.4666377702799545e-06,
      "log_odds_chosen": 0.5222339630126953,
      "log_odds_ratio": -0.47047317028045654,
      "logits/chosen": -0.23173952102661133,
      "logits/rejected": -1.5240064859390259,
      "logps/chosen": -1.4771634340286255,
      "logps/rejected": -1.9015573263168335,
      "loss": 1.5785,
      "nll_loss": 1.5314639806747437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14771634340286255,
      "rewards/margins": 0.04243940860033035,
      "rewards/rejected": -0.1901557445526123,
      "step": 400
    },
    {
      "epoch": 0.6319936958234831,
      "grad_norm": 0.19452960789203644,
      "learning_rate": 3.4628948400846417e-06,
      "log_odds_chosen": 0.6402549147605896,
      "log_odds_ratio": -0.43367528915405273,
      "logits/chosen": -0.15246199071407318,
      "logits/rejected": -1.4411405324935913,
      "logps/chosen": -1.5569361448287964,
      "logps/rejected": -2.0934228897094727,
      "loss": 1.6454,
      "nll_loss": 1.60203218460083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15569362044334412,
      "rewards/margins": 0.053648658096790314,
      "rewards/rejected": -0.20934227108955383,
      "step": 401
    },
    {
      "epoch": 0.6335697399527187,
      "grad_norm": 0.2060081660747528,
      "learning_rate": 3.4591408584175426e-06,
      "log_odds_chosen": 0.5116357207298279,
      "log_odds_ratio": -0.4790383577346802,
      "logits/chosen": -0.1661422997713089,
      "logits/rejected": -1.3209583759307861,
      "logps/chosen": -1.5975854396820068,
      "logps/rejected": -2.0274882316589355,
      "loss": 1.6893,
      "nll_loss": 1.6414045095443726,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1597585529088974,
      "rewards/margins": 0.042990271002054214,
      "rewards/rejected": -0.20274882018566132,
      "step": 402
    },
    {
      "epoch": 0.6351457840819543,
      "grad_norm": 0.1968107670545578,
      "learning_rate": 3.4553758536381974e-06,
      "log_odds_chosen": 0.5235389471054077,
      "log_odds_ratio": -0.47713008522987366,
      "logits/chosen": -0.15526294708251953,
      "logits/rejected": -1.3401628732681274,
      "logps/chosen": -1.554589033126831,
      "logps/rejected": -1.9898911714553833,
      "loss": 1.6418,
      "nll_loss": 1.5940942764282227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15545889735221863,
      "rewards/margins": 0.04353022575378418,
      "rewards/rejected": -0.198989138007164,
      "step": 403
    },
    {
      "epoch": 0.6367218282111899,
      "grad_norm": 0.2039954662322998,
      "learning_rate": 3.451599854189418e-06,
      "log_odds_chosen": 0.5396761894226074,
      "log_odds_ratio": -0.46344685554504395,
      "logits/chosen": -0.11397916078567505,
      "logits/rejected": -0.9760464429855347,
      "logps/chosen": -1.5517972707748413,
      "logps/rejected": -1.998788595199585,
      "loss": 1.6507,
      "nll_loss": 1.6043689250946045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15517973899841309,
      "rewards/margins": 0.04469913989305496,
      "rewards/rejected": -0.19987887144088745,
      "step": 404
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 0.19308118522167206,
      "learning_rate": 3.4478128885970765e-06,
      "log_odds_chosen": 0.6018125414848328,
      "log_odds_ratio": -0.4381783902645111,
      "logits/chosen": -0.16679120063781738,
      "logits/rejected": -1.3316242694854736,
      "logps/chosen": -1.6154249906539917,
      "logps/rejected": -2.122253894805908,
      "loss": 1.7001,
      "nll_loss": 1.6562443971633911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16154250502586365,
      "rewards/margins": 0.05068288370966911,
      "rewards/rejected": -0.21222539246082306,
      "step": 405
    },
    {
      "epoch": 0.6398739164696612,
      "grad_norm": 0.19146420061588287,
      "learning_rate": 3.44401498546989e-06,
      "log_odds_chosen": 0.4198459982872009,
      "log_odds_ratio": -0.5137929320335388,
      "logits/chosen": -0.10405167937278748,
      "logits/rejected": -1.477805733680725,
      "logps/chosen": -1.5691133737564087,
      "logps/rejected": -1.9161850214004517,
      "loss": 1.6395,
      "nll_loss": 1.5880873203277588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15691134333610535,
      "rewards/margins": 0.03470717370510101,
      "rewards/rejected": -0.19161850214004517,
      "step": 406
    },
    {
      "epoch": 0.6414499605988968,
      "grad_norm": 0.19537797570228577,
      "learning_rate": 3.4402061734992005e-06,
      "log_odds_chosen": 0.5202116370201111,
      "log_odds_ratio": -0.47119492292404175,
      "logits/chosen": -0.1433197259902954,
      "logits/rejected": -1.3762876987457275,
      "logps/chosen": -1.5083684921264648,
      "logps/rejected": -1.9353201389312744,
      "loss": 1.6033,
      "nll_loss": 1.5561641454696655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15083685517311096,
      "rewards/margins": 0.042695142328739166,
      "rewards/rejected": -0.19353200495243073,
      "step": 407
    },
    {
      "epoch": 0.6430260047281324,
      "grad_norm": 0.20898084342479706,
      "learning_rate": 3.4363864814587656e-06,
      "log_odds_chosen": 0.49365922808647156,
      "log_odds_ratio": -0.4801666736602783,
      "logits/chosen": -0.25102052092552185,
      "logits/rejected": -0.9237027764320374,
      "logps/chosen": -1.4852203130722046,
      "logps/rejected": -1.8896757364273071,
      "loss": 1.582,
      "nll_loss": 1.5339844226837158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1485220193862915,
      "rewards/margins": 0.04044555127620697,
      "rewards/rejected": -0.18896758556365967,
      "step": 408
    },
    {
      "epoch": 0.644602048857368,
      "grad_norm": 0.2019755095243454,
      "learning_rate": 3.4325559382045343e-06,
      "log_odds_chosen": 0.4014512598514557,
      "log_odds_ratio": -0.5167407989501953,
      "logits/chosen": -0.22251050174236298,
      "logits/rejected": -1.485706090927124,
      "logps/chosen": -1.4902509450912476,
      "logps/rejected": -1.812532663345337,
      "loss": 1.581,
      "nll_loss": 1.5293045043945312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.149025097489357,
      "rewards/margins": 0.032228171825408936,
      "rewards/rejected": -0.18125325441360474,
      "step": 409
    },
    {
      "epoch": 0.6461780929866037,
      "grad_norm": 0.20751270651817322,
      "learning_rate": 3.4287145726744295e-06,
      "log_odds_chosen": 0.5322403907775879,
      "log_odds_ratio": -0.46528854966163635,
      "logits/chosen": -0.24800266325473785,
      "logits/rejected": -1.3249245882034302,
      "logps/chosen": -1.478865623474121,
      "logps/rejected": -1.912164568901062,
      "loss": 1.5805,
      "nll_loss": 1.5339622497558594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14788655936717987,
      "rewards/margins": 0.0433298759162426,
      "rewards/rejected": -0.19121645390987396,
      "step": 410
    },
    {
      "epoch": 0.6477541371158393,
      "grad_norm": 0.20497266948223114,
      "learning_rate": 3.4248624138881335e-06,
      "log_odds_chosen": 0.4167112112045288,
      "log_odds_ratio": -0.5280964374542236,
      "logits/chosen": -0.146779865026474,
      "logits/rejected": -1.1234551668167114,
      "logps/chosen": -1.615991473197937,
      "logps/rejected": -1.9755668640136719,
      "loss": 1.7015,
      "nll_loss": 1.6486729383468628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16159915924072266,
      "rewards/margins": 0.035957541316747665,
      "rewards/rejected": -0.19755667448043823,
      "step": 411
    },
    {
      "epoch": 0.6493301812450749,
      "grad_norm": 0.2031044065952301,
      "learning_rate": 3.4209994909468672e-06,
      "log_odds_chosen": 0.6441755294799805,
      "log_odds_ratio": -0.4322493076324463,
      "logits/chosen": -0.27414995431900024,
      "logits/rejected": -1.0003337860107422,
      "logps/chosen": -1.5170173645019531,
      "logps/rejected": -2.0522003173828125,
      "loss": 1.6035,
      "nll_loss": 1.5602253675460815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15170171856880188,
      "rewards/margins": 0.05351831763982773,
      "rewards/rejected": -0.2052200585603714,
      "step": 412
    },
    {
      "epoch": 0.6509062253743105,
      "grad_norm": 0.20105931162834167,
      "learning_rate": 3.4171258330331667e-06,
      "log_odds_chosen": 0.43131789565086365,
      "log_odds_ratio": -0.5029494762420654,
      "logits/chosen": -0.11574351787567139,
      "logits/rejected": -1.0839128494262695,
      "logps/chosen": -1.603833556175232,
      "logps/rejected": -1.9600427150726318,
      "loss": 1.6866,
      "nll_loss": 1.6363379955291748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16038337349891663,
      "rewards/margins": 0.035620901733636856,
      "rewards/rejected": -0.19600427150726318,
      "step": 413
    },
    {
      "epoch": 0.6524822695035462,
      "grad_norm": 0.1987302601337433,
      "learning_rate": 3.4132414694106684e-06,
      "log_odds_chosen": 0.6570238471031189,
      "log_odds_ratio": -0.425781786441803,
      "logits/chosen": -0.15273043513298035,
      "logits/rejected": -1.4411128759384155,
      "logps/chosen": -1.510444164276123,
      "logps/rejected": -2.0561532974243164,
      "loss": 1.5935,
      "nll_loss": 1.550927758216858,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15104441344738007,
      "rewards/margins": 0.05457089841365814,
      "rewards/rejected": -0.2056153267621994,
      "step": 414
    },
    {
      "epoch": 0.6540583136327817,
      "grad_norm": 0.20442619919776917,
      "learning_rate": 3.409346429423884e-06,
      "log_odds_chosen": 0.45281243324279785,
      "log_odds_ratio": -0.4946718215942383,
      "logits/chosen": -0.008420580998063087,
      "logits/rejected": -1.278180480003357,
      "logps/chosen": -1.5783418416976929,
      "logps/rejected": -1.952226996421814,
      "loss": 1.6571,
      "nll_loss": 1.607609748840332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15783418715000153,
      "rewards/margins": 0.03738852217793465,
      "rewards/rejected": -0.19522270560264587,
      "step": 415
    },
    {
      "epoch": 0.6556343577620173,
      "grad_norm": 0.1937103271484375,
      "learning_rate": 3.40544074249798e-06,
      "log_odds_chosen": 0.6057698726654053,
      "log_odds_ratio": -0.43932780623435974,
      "logits/chosen": -0.18488506972789764,
      "logits/rejected": -1.447567343711853,
      "logps/chosen": -1.568244218826294,
      "logps/rejected": -2.074944496154785,
      "loss": 1.6516,
      "nll_loss": 1.607670545578003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15682442486286163,
      "rewards/margins": 0.05067002773284912,
      "rewards/rejected": -0.20749445259571075,
      "step": 416
    },
    {
      "epoch": 0.6572104018912529,
      "grad_norm": 0.21314595639705658,
      "learning_rate": 3.401524438138556e-06,
      "log_odds_chosen": 0.46506115794181824,
      "log_odds_ratio": -0.4945875406265259,
      "logits/chosen": -0.12872271239757538,
      "logits/rejected": -1.0993512868881226,
      "logps/chosen": -1.5732535123825073,
      "logps/rejected": -1.9601188898086548,
      "loss": 1.6377,
      "nll_loss": 1.5882155895233154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573253571987152,
      "rewards/margins": 0.03868655115365982,
      "rewards/rejected": -0.19601190090179443,
      "step": 417
    },
    {
      "epoch": 0.6587864460204885,
      "grad_norm": 0.20356151461601257,
      "learning_rate": 3.39759754593142e-06,
      "log_odds_chosen": 0.49928921461105347,
      "log_odds_ratio": -0.4814523756504059,
      "logits/chosen": -0.18617013096809387,
      "logits/rejected": -1.2594208717346191,
      "logps/chosen": -1.6336315870285034,
      "logps/rejected": -2.05531644821167,
      "loss": 1.7071,
      "nll_loss": 1.6589891910552979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16336317360401154,
      "rewards/margins": 0.04216848313808441,
      "rewards/rejected": -0.20553165674209595,
      "step": 418
    },
    {
      "epoch": 0.6603624901497241,
      "grad_norm": 0.1998877227306366,
      "learning_rate": 3.3936600955423683e-06,
      "log_odds_chosen": 0.703155517578125,
      "log_odds_ratio": -0.4173457622528076,
      "logits/chosen": -0.10339093208312988,
      "logits/rejected": -1.0794388055801392,
      "logps/chosen": -1.526084542274475,
      "logps/rejected": -2.111715078353882,
      "loss": 1.613,
      "nll_loss": 1.57127845287323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1526084691286087,
      "rewards/margins": 0.05856304615736008,
      "rewards/rejected": -0.21117150783538818,
      "step": 419
    },
    {
      "epoch": 0.6619385342789598,
      "grad_norm": 0.1923801749944687,
      "learning_rate": 3.3897121167169573e-06,
      "log_odds_chosen": 0.3884754776954651,
      "log_odds_ratio": -0.5280902981758118,
      "logits/chosen": -0.2249905914068222,
      "logits/rejected": -1.2855345010757446,
      "logps/chosen": -1.499492883682251,
      "logps/rejected": -1.8159973621368408,
      "loss": 1.5979,
      "nll_loss": 1.545102596282959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14994929730892181,
      "rewards/margins": 0.031650442630052567,
      "rewards/rejected": -0.18159973621368408,
      "step": 420
    },
    {
      "epoch": 0.6635145784081954,
      "grad_norm": 0.20332655310630798,
      "learning_rate": 3.38575363928028e-06,
      "log_odds_chosen": 0.5980296730995178,
      "log_odds_ratio": -0.444957435131073,
      "logits/chosen": -0.1456383615732193,
      "logits/rejected": -1.1062229871749878,
      "logps/chosen": -1.5091032981872559,
      "logps/rejected": -2.0027518272399902,
      "loss": 1.5809,
      "nll_loss": 1.5364110469818115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15091031789779663,
      "rewards/margins": 0.04936487227678299,
      "rewards/rejected": -0.20027519762516022,
      "step": 421
    },
    {
      "epoch": 0.665090622537431,
      "grad_norm": 0.19581378996372223,
      "learning_rate": 3.3817846931367452e-06,
      "log_odds_chosen": 0.40414246916770935,
      "log_odds_ratio": -0.5155224800109863,
      "logits/chosen": -0.22978416085243225,
      "logits/rejected": -1.1638431549072266,
      "logps/chosen": -1.4919337034225464,
      "logps/rejected": -1.8167966604232788,
      "loss": 1.5812,
      "nll_loss": 1.5296943187713623,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14919337630271912,
      "rewards/margins": 0.032486289739608765,
      "rewards/rejected": -0.18167968094348907,
      "step": 422
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 0.18812742829322815,
      "learning_rate": 3.377805308269844e-06,
      "log_odds_chosen": 0.6795855760574341,
      "log_odds_ratio": -0.42459002137184143,
      "logits/chosen": -0.11801916360855103,
      "logits/rejected": -1.4967668056488037,
      "logps/chosen": -1.54996919631958,
      "logps/rejected": -2.1208114624023438,
      "loss": 1.6253,
      "nll_loss": 1.5828850269317627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15499693155288696,
      "rewards/margins": 0.05708422511816025,
      "rewards/rejected": -0.21208113431930542,
      "step": 423
    },
    {
      "epoch": 0.6682427107959023,
      "grad_norm": 0.18950577080249786,
      "learning_rate": 3.3738155147419275e-06,
      "log_odds_chosen": 0.6117570996284485,
      "log_odds_ratio": -0.44386616349220276,
      "logits/chosen": -0.24225133657455444,
      "logits/rejected": -1.1781508922576904,
      "logps/chosen": -1.5445609092712402,
      "logps/rejected": -2.0556564331054688,
      "loss": 1.6058,
      "nll_loss": 1.561427354812622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15445609390735626,
      "rewards/margins": 0.05110953003168106,
      "rewards/rejected": -0.20556563138961792,
      "step": 424
    },
    {
      "epoch": 0.6698187549251379,
      "grad_norm": 0.2043224424123764,
      "learning_rate": 3.3698153426939824e-06,
      "log_odds_chosen": 0.6969811916351318,
      "log_odds_ratio": -0.4145274758338928,
      "logits/chosen": -0.2313898205757141,
      "logits/rejected": -1.0220508575439453,
      "logps/chosen": -1.473904013633728,
      "logps/rejected": -2.0466439723968506,
      "loss": 1.5474,
      "nll_loss": 1.5059936046600342,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14739039540290833,
      "rewards/margins": 0.05727401375770569,
      "rewards/rejected": -0.204664409160614,
      "step": 425
    },
    {
      "epoch": 0.6713947990543735,
      "grad_norm": 0.20883503556251526,
      "learning_rate": 3.3658048223453954e-06,
      "log_odds_chosen": 0.6612953543663025,
      "log_odds_ratio": -0.42457008361816406,
      "logits/chosen": -0.19671984016895294,
      "logits/rejected": -1.3579217195510864,
      "logps/chosen": -1.6156816482543945,
      "logps/rejected": -2.1756107807159424,
      "loss": 1.6623,
      "nll_loss": 1.6198010444641113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16156817972660065,
      "rewards/margins": 0.05599290132522583,
      "rewards/rejected": -0.21756106615066528,
      "step": 426
    },
    {
      "epoch": 0.6729708431836091,
      "grad_norm": 0.19400307536125183,
      "learning_rate": 3.3617839839937337e-06,
      "log_odds_chosen": 0.6474947333335876,
      "log_odds_ratio": -0.431326687335968,
      "logits/chosen": -0.11850985884666443,
      "logits/rejected": -1.453221321105957,
      "logps/chosen": -1.5760408639907837,
      "logps/rejected": -2.1231493949890137,
      "loss": 1.6424,
      "nll_loss": 1.59929358959198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15760409832000732,
      "rewards/margins": 0.054710857570171356,
      "rewards/rejected": -0.2123149335384369,
      "step": 427
    },
    {
      "epoch": 0.6745468873128447,
      "grad_norm": 0.20923368632793427,
      "learning_rate": 3.3577528580145107e-06,
      "log_odds_chosen": 0.33366745710372925,
      "log_odds_ratio": -0.5432456731796265,
      "logits/chosen": -0.14751064777374268,
      "logits/rejected": -1.2790288925170898,
      "logps/chosen": -1.5968459844589233,
      "logps/rejected": -1.8737503290176392,
      "loss": 1.678,
      "nll_loss": 1.6236892938613892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15968461334705353,
      "rewards/margins": 0.027690425515174866,
      "rewards/rejected": -0.1873750239610672,
      "step": 428
    },
    {
      "epoch": 0.6761229314420804,
      "grad_norm": 0.1988777220249176,
      "learning_rate": 3.353711474860956e-06,
      "log_odds_chosen": 0.5849066376686096,
      "log_odds_ratio": -0.45891109108924866,
      "logits/chosen": -0.18468442559242249,
      "logits/rejected": -1.206817865371704,
      "logps/chosen": -1.5448060035705566,
      "logps/rejected": -2.038527011871338,
      "loss": 1.6061,
      "nll_loss": 1.560197353363037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15448060631752014,
      "rewards/margins": 0.04937208443880081,
      "rewards/rejected": -0.20385268330574036,
      "step": 429
    },
    {
      "epoch": 0.677698975571316,
      "grad_norm": 0.1907055526971817,
      "learning_rate": 3.3496598650637916e-06,
      "log_odds_chosen": 0.5627853274345398,
      "log_odds_ratio": -0.45737147331237793,
      "logits/chosen": -0.213260680437088,
      "logits/rejected": -1.1075356006622314,
      "logps/chosen": -1.4453046321868896,
      "logps/rejected": -1.9003225564956665,
      "loss": 1.5279,
      "nll_loss": 1.4821749925613403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14453044533729553,
      "rewards/margins": 0.04550181329250336,
      "rewards/rejected": -0.19003227353096008,
      "step": 430
    },
    {
      "epoch": 0.6792750197005516,
      "grad_norm": 0.1916009783744812,
      "learning_rate": 3.3455980592309923e-06,
      "log_odds_chosen": 0.6095476150512695,
      "log_odds_ratio": -0.44210147857666016,
      "logits/chosen": -0.23830467462539673,
      "logits/rejected": -1.2122029066085815,
      "logps/chosen": -1.4246712923049927,
      "logps/rejected": -1.921633005142212,
      "loss": 1.515,
      "nll_loss": 1.4708294868469238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14246714115142822,
      "rewards/margins": 0.04969616234302521,
      "rewards/rejected": -0.19216330349445343,
      "step": 431
    },
    {
      "epoch": 0.6808510638297872,
      "grad_norm": 0.2036287635564804,
      "learning_rate": 3.341526088047562e-06,
      "log_odds_chosen": 0.5544840693473816,
      "log_odds_ratio": -0.46283283829689026,
      "logits/chosen": -0.18271717429161072,
      "logits/rejected": -1.2361128330230713,
      "logps/chosen": -1.612389326095581,
      "logps/rejected": -2.080324411392212,
      "loss": 1.6715,
      "nll_loss": 1.6252350807189941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16123893857002258,
      "rewards/margins": 0.046793513000011444,
      "rewards/rejected": -0.20803247392177582,
      "step": 432
    },
    {
      "epoch": 0.6824271079590228,
      "grad_norm": 0.21691077947616577,
      "learning_rate": 3.3374439822752972e-06,
      "log_odds_chosen": 0.4269826412200928,
      "log_odds_ratio": -0.5059152841567993,
      "logits/chosen": -0.1358519196510315,
      "logits/rejected": -1.0953370332717896,
      "logps/chosen": -1.5720406770706177,
      "logps/rejected": -1.927139401435852,
      "loss": 1.6598,
      "nll_loss": 1.609249472618103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15720407664775848,
      "rewards/margins": 0.03550989180803299,
      "rewards/rejected": -0.19271396100521088,
      "step": 433
    },
    {
      "epoch": 0.6840031520882585,
      "grad_norm": 0.25961580872535706,
      "learning_rate": 3.333351772752559e-06,
      "log_odds_chosen": 0.5796866416931152,
      "log_odds_ratio": -0.4593260884284973,
      "logits/chosen": -0.2504950165748596,
      "logits/rejected": -1.251611351966858,
      "logps/chosen": -1.6392018795013428,
      "logps/rejected": -2.125319004058838,
      "loss": 1.7019,
      "nll_loss": 1.655974268913269,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16392017900943756,
      "rewards/margins": 0.048611730337142944,
      "rewards/rejected": -0.2125319093465805,
      "step": 434
    },
    {
      "epoch": 0.6855791962174941,
      "grad_norm": 0.20323769748210907,
      "learning_rate": 3.3292494903940338e-06,
      "log_odds_chosen": 0.6984251141548157,
      "log_odds_ratio": -0.41105496883392334,
      "logits/chosen": -0.16568951308727264,
      "logits/rejected": -1.2594187259674072,
      "logps/chosen": -1.5446348190307617,
      "logps/rejected": -2.125537395477295,
      "loss": 1.6162,
      "nll_loss": 1.5750484466552734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1544634848833084,
      "rewards/margins": 0.058090247213840485,
      "rewards/rejected": -0.2125537395477295,
      "step": 435
    },
    {
      "epoch": 0.6871552403467297,
      "grad_norm": 0.18550781905651093,
      "learning_rate": 3.3251371661905063e-06,
      "log_odds_chosen": 0.5926481485366821,
      "log_odds_ratio": -0.45173323154449463,
      "logits/chosen": -0.19779422879219055,
      "logits/rejected": -1.1184755563735962,
      "logps/chosen": -1.3700029850006104,
      "logps/rejected": -1.8533730506896973,
      "loss": 1.4567,
      "nll_loss": 1.4115686416625977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13700029253959656,
      "rewards/margins": 0.04833700880408287,
      "rewards/rejected": -0.18533730506896973,
      "step": 436
    },
    {
      "epoch": 0.6887312844759653,
      "grad_norm": 0.19023758172988892,
      "learning_rate": 3.321014831208622e-06,
      "log_odds_chosen": 0.5900598764419556,
      "log_odds_ratio": -0.44593122601509094,
      "logits/chosen": -0.20720794796943665,
      "logits/rejected": -1.0833600759506226,
      "logps/chosen": -1.4328017234802246,
      "logps/rejected": -1.9078587293624878,
      "loss": 1.5262,
      "nll_loss": 1.4815677404403687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14328016340732574,
      "rewards/margins": 0.047505710273981094,
      "rewards/rejected": -0.19078589975833893,
      "step": 437
    },
    {
      "epoch": 0.6903073286052009,
      "grad_norm": 0.18916691839694977,
      "learning_rate": 3.316882516590652e-06,
      "log_odds_chosen": 0.6026834845542908,
      "log_odds_ratio": -0.43881577253341675,
      "logits/chosen": -0.19735684990882874,
      "logits/rejected": -1.3404430150985718,
      "logps/chosen": -1.4774234294891357,
      "logps/rejected": -1.9714411497116089,
      "loss": 1.5584,
      "nll_loss": 1.5145463943481445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1477423459291458,
      "rewards/margins": 0.04940177500247955,
      "rewards/rejected": -0.19714412093162537,
      "step": 438
    },
    {
      "epoch": 0.6918833727344366,
      "grad_norm": 0.2226303368806839,
      "learning_rate": 3.31274025355426e-06,
      "log_odds_chosen": 0.5777285099029541,
      "log_odds_ratio": -0.45020100474357605,
      "logits/chosen": -0.2297353744506836,
      "logits/rejected": -1.1321128606796265,
      "logps/chosen": -1.5695414543151855,
      "logps/rejected": -2.051584243774414,
      "loss": 1.6201,
      "nll_loss": 1.57511568069458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15695415437221527,
      "rewards/margins": 0.04820426553487778,
      "rewards/rejected": -0.20515841245651245,
      "step": 439
    },
    {
      "epoch": 0.6934594168636722,
      "grad_norm": 0.20178461074829102,
      "learning_rate": 3.308588073392265e-06,
      "log_odds_chosen": 0.5347023606300354,
      "log_odds_ratio": -0.4697195291519165,
      "logits/chosen": -0.22104787826538086,
      "logits/rejected": -1.1682895421981812,
      "logps/chosen": -1.5996606349945068,
      "logps/rejected": -2.0496530532836914,
      "loss": 1.6537,
      "nll_loss": 1.606750249862671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15996606647968292,
      "rewards/margins": 0.04499925673007965,
      "rewards/rejected": -0.20496532320976257,
      "step": 440
    },
    {
      "epoch": 0.6950354609929078,
      "grad_norm": 0.19244441390037537,
      "learning_rate": 3.3044260074724035e-06,
      "log_odds_chosen": 0.6290772557258606,
      "log_odds_ratio": -0.438933789730072,
      "logits/chosen": -0.21869786083698273,
      "logits/rejected": -1.3959290981292725,
      "logps/chosen": -1.5120939016342163,
      "logps/rejected": -2.030512809753418,
      "loss": 1.588,
      "nll_loss": 1.5441324710845947,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15120939910411835,
      "rewards/margins": 0.05184189975261688,
      "rewards/rejected": -0.20305128395557404,
      "step": 441
    },
    {
      "epoch": 0.6966115051221434,
      "grad_norm": 0.1978086680173874,
      "learning_rate": 3.300254087237097e-06,
      "log_odds_chosen": 0.5377320647239685,
      "log_odds_ratio": -0.46433717012405396,
      "logits/chosen": -0.18392041325569153,
      "logits/rejected": -1.2981102466583252,
      "logps/chosen": -1.4015955924987793,
      "logps/rejected": -1.8341153860092163,
      "loss": 1.5073,
      "nll_loss": 1.460841417312622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14015956223011017,
      "rewards/margins": 0.04325197637081146,
      "rewards/rejected": -0.18341155350208282,
      "step": 442
    },
    {
      "epoch": 0.698187549251379,
      "grad_norm": 0.19712939858436584,
      "learning_rate": 3.2960723442032105e-06,
      "log_odds_chosen": 0.7022231817245483,
      "log_odds_ratio": -0.4093382954597473,
      "logits/chosen": -0.24490830302238464,
      "logits/rejected": -1.5799317359924316,
      "logps/chosen": -1.6035810708999634,
      "logps/rejected": -2.200234889984131,
      "loss": 1.6564,
      "nll_loss": 1.6155025959014893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.16035811603069305,
      "rewards/margins": 0.05966537445783615,
      "rewards/rejected": -0.2200234830379486,
      "step": 443
    },
    {
      "epoch": 0.6997635933806147,
      "grad_norm": 0.1867726445198059,
      "learning_rate": 3.291880809961814e-06,
      "log_odds_chosen": 0.6228445768356323,
      "log_odds_ratio": -0.4409523904323578,
      "logits/chosen": -0.16951556503772736,
      "logits/rejected": -1.1806188821792603,
      "logps/chosen": -1.5619224309921265,
      "logps/rejected": -2.079741954803467,
      "loss": 1.6322,
      "nll_loss": 1.5880740880966187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15619224309921265,
      "rewards/margins": 0.05178195238113403,
      "rewards/rejected": -0.20797421038150787,
      "step": 444
    },
    {
      "epoch": 0.7013396375098503,
      "grad_norm": 0.19691401720046997,
      "learning_rate": 3.2876795161779473e-06,
      "log_odds_chosen": 0.7107241153717041,
      "log_odds_ratio": -0.4021419882774353,
      "logits/chosen": -0.18867270648479462,
      "logits/rejected": -1.5382509231567383,
      "logps/chosen": -1.5732825994491577,
      "logps/rejected": -2.173732280731201,
      "loss": 1.6228,
      "nll_loss": 1.5826274156570435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.157328262925148,
      "rewards/margins": 0.06004498153924942,
      "rewards/rejected": -0.21737323701381683,
      "step": 445
    },
    {
      "epoch": 0.7029156816390859,
      "grad_norm": 0.20158085227012634,
      "learning_rate": 3.2834684945903776e-06,
      "log_odds_chosen": 0.5534508228302002,
      "log_odds_ratio": -0.4586279094219208,
      "logits/chosen": -0.24863602221012115,
      "logits/rejected": -1.2043968439102173,
      "logps/chosen": -1.5018632411956787,
      "logps/rejected": -1.9528242349624634,
      "loss": 1.5808,
      "nll_loss": 1.5349347591400146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15018633008003235,
      "rewards/margins": 0.04509609937667847,
      "rewards/rejected": -0.19528241455554962,
      "step": 446
    },
    {
      "epoch": 0.7044917257683215,
      "grad_norm": 0.20299731194972992,
      "learning_rate": 3.2792477770113624e-06,
      "log_odds_chosen": 0.4938434362411499,
      "log_odds_ratio": -0.4814365804195404,
      "logits/chosen": -0.33849644660949707,
      "logits/rejected": -1.3542628288269043,
      "logps/chosen": -1.5917766094207764,
      "logps/rejected": -2.002875328063965,
      "loss": 1.6474,
      "nll_loss": 1.5992237329483032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15917766094207764,
      "rewards/margins": 0.041109874844551086,
      "rewards/rejected": -0.20028753578662872,
      "step": 447
    },
    {
      "epoch": 0.7060677698975572,
      "grad_norm": 0.18600669503211975,
      "learning_rate": 3.275017395326407e-06,
      "log_odds_chosen": 0.47598177194595337,
      "log_odds_ratio": -0.4912503659725189,
      "logits/chosen": -0.15067262947559357,
      "logits/rejected": -1.1057360172271729,
      "logps/chosen": -1.479712963104248,
      "logps/rejected": -1.8618954420089722,
      "loss": 1.5626,
      "nll_loss": 1.5134308338165283,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14797130227088928,
      "rewards/margins": 0.03821825981140137,
      "rewards/rejected": -0.18618956208229065,
      "step": 448
    },
    {
      "epoch": 0.7076438140267928,
      "grad_norm": 0.1918012499809265,
      "learning_rate": 3.2707773814940244e-06,
      "log_odds_chosen": 0.6454062461853027,
      "log_odds_ratio": -0.4280465841293335,
      "logits/chosen": -0.17873692512512207,
      "logits/rejected": -1.295732021331787,
      "logps/chosen": -1.507833480834961,
      "logps/rejected": -2.0341856479644775,
      "loss": 1.5887,
      "nll_loss": 1.5458601713180542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15078336000442505,
      "rewards/margins": 0.05263521522283554,
      "rewards/rejected": -0.20341856777668,
      "step": 449
    },
    {
      "epoch": 0.7092198581560284,
      "grad_norm": 0.1859048306941986,
      "learning_rate": 3.2665277675454935e-06,
      "log_odds_chosen": 0.6589299440383911,
      "log_odds_ratio": -0.434900164604187,
      "logits/chosen": -0.2396085262298584,
      "logits/rejected": -1.546783685684204,
      "logps/chosen": -1.5887700319290161,
      "logps/rejected": -2.151167392730713,
      "loss": 1.6406,
      "nll_loss": 1.5970611572265625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15887701511383057,
      "rewards/margins": 0.056239720433950424,
      "rewards/rejected": -0.2151167392730713,
      "step": 450
    },
    {
      "epoch": 0.710795902285264,
      "grad_norm": 0.18971043825149536,
      "learning_rate": 3.262268585584619e-06,
      "log_odds_chosen": 0.5915400981903076,
      "log_odds_ratio": -0.44323453307151794,
      "logits/chosen": -0.16203592717647552,
      "logits/rejected": -1.2521172761917114,
      "logps/chosen": -1.5381022691726685,
      "logps/rejected": -2.030604124069214,
      "loss": 1.6041,
      "nll_loss": 1.5598118305206299,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15381023287773132,
      "rewards/margins": 0.049250174313783646,
      "rewards/rejected": -0.20306040346622467,
      "step": 451
    },
    {
      "epoch": 0.7123719464144996,
      "grad_norm": 0.2022084891796112,
      "learning_rate": 3.2579998677874853e-06,
      "log_odds_chosen": 0.6777637600898743,
      "log_odds_ratio": -0.4203702211380005,
      "logits/chosen": -0.24360840022563934,
      "logits/rejected": -1.232491135597229,
      "logps/chosen": -1.5734740495681763,
      "logps/rejected": -2.1456642150878906,
      "loss": 1.6256,
      "nll_loss": 1.5835988521575928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1573474109172821,
      "rewards/margins": 0.0572190061211586,
      "rewards/rejected": -0.2145664095878601,
      "step": 452
    },
    {
      "epoch": 0.7139479905437353,
      "grad_norm": 0.1870575249195099,
      "learning_rate": 3.2537216464022155e-06,
      "log_odds_chosen": 0.6743752956390381,
      "log_odds_ratio": -0.4204288721084595,
      "logits/chosen": -0.22134186327457428,
      "logits/rejected": -1.2324929237365723,
      "logps/chosen": -1.5700197219848633,
      "logps/rejected": -2.1332030296325684,
      "loss": 1.6333,
      "nll_loss": 1.5912562608718872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15700197219848633,
      "rewards/margins": 0.05631833150982857,
      "rewards/rejected": -0.2133202999830246,
      "step": 453
    },
    {
      "epoch": 0.7155240346729709,
      "grad_norm": 0.1873411238193512,
      "learning_rate": 3.2494339537487314e-06,
      "log_odds_chosen": 0.5221689343452454,
      "log_odds_ratio": -0.4673764109611511,
      "logits/chosen": -0.24073247611522675,
      "logits/rejected": -1.4843155145645142,
      "logps/chosen": -1.574814796447754,
      "logps/rejected": -2.0072293281555176,
      "loss": 1.6482,
      "nll_loss": 1.6014328002929688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15748149156570435,
      "rewards/margins": 0.04324144124984741,
      "rewards/rejected": -0.20072293281555176,
      "step": 454
    },
    {
      "epoch": 0.7171000788022065,
      "grad_norm": 0.18600791692733765,
      "learning_rate": 3.2451368222185006e-06,
      "log_odds_chosen": 0.4506160616874695,
      "log_odds_ratio": -0.49926573038101196,
      "logits/chosen": -0.23936370015144348,
      "logits/rejected": -1.121845006942749,
      "logps/chosen": -1.5184142589569092,
      "logps/rejected": -1.8841001987457275,
      "loss": 1.5993,
      "nll_loss": 1.5493282079696655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1518414169549942,
      "rewards/margins": 0.03656858578324318,
      "rewards/rejected": -0.18841001391410828,
      "step": 455
    },
    {
      "epoch": 0.7186761229314421,
      "grad_norm": 0.17563648521900177,
      "learning_rate": 3.2408302842743007e-06,
      "log_odds_chosen": 0.6520451903343201,
      "log_odds_ratio": -0.4270837604999542,
      "logits/chosen": -0.19061179459095,
      "logits/rejected": -1.2755080461502075,
      "logps/chosen": -1.4686167240142822,
      "logps/rejected": -2.004254102706909,
      "loss": 1.5546,
      "nll_loss": 1.511888027191162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14686168730258942,
      "rewards/margins": 0.05356375128030777,
      "rewards/rejected": -0.2004254311323166,
      "step": 456
    },
    {
      "epoch": 0.7202521670606777,
      "grad_norm": 0.19001001119613647,
      "learning_rate": 3.2365143724499684e-06,
      "log_odds_chosen": 0.5796292424201965,
      "log_odds_ratio": -0.4465140104293823,
      "logits/chosen": -0.20313462615013123,
      "logits/rejected": -1.3108395338058472,
      "logps/chosen": -1.5164985656738281,
      "logps/rejected": -1.994590163230896,
      "loss": 1.6016,
      "nll_loss": 1.556924819946289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1516498476266861,
      "rewards/margins": 0.047809161245822906,
      "rewards/rejected": -0.1994590163230896,
      "step": 457
    },
    {
      "epoch": 0.7218282111899134,
      "grad_norm": 0.19313624501228333,
      "learning_rate": 3.2321891193501564e-06,
      "log_odds_chosen": 0.5773687958717346,
      "log_odds_ratio": -0.45744845271110535,
      "logits/chosen": -0.1832474172115326,
      "logits/rejected": -0.8535163998603821,
      "logps/chosen": -1.5230867862701416,
      "logps/rejected": -2.0000154972076416,
      "loss": 1.5894,
      "nll_loss": 1.5436917543411255,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15230868756771088,
      "rewards/margins": 0.04769286513328552,
      "rewards/rejected": -0.2000015527009964,
      "step": 458
    },
    {
      "epoch": 0.723404255319149,
      "grad_norm": 0.1829526573419571,
      "learning_rate": 3.2278545576500858e-06,
      "log_odds_chosen": 0.815370500087738,
      "log_odds_ratio": -0.3753301799297333,
      "logits/chosen": -0.10391156375408173,
      "logits/rejected": -1.0528960227966309,
      "logps/chosen": -1.3729358911514282,
      "logps/rejected": -2.040825843811035,
      "loss": 1.4585,
      "nll_loss": 1.4209495782852173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13729359209537506,
      "rewards/margins": 0.06678898632526398,
      "rewards/rejected": -0.20408257842063904,
      "step": 459
    },
    {
      "epoch": 0.7249802994483846,
      "grad_norm": 0.19639159739017487,
      "learning_rate": 3.223510720095299e-06,
      "log_odds_chosen": 0.7596641778945923,
      "log_odds_ratio": -0.3919370770454407,
      "logits/chosen": -0.2874103784561157,
      "logits/rejected": -1.3473809957504272,
      "logps/chosen": -1.551926851272583,
      "logps/rejected": -2.19168758392334,
      "loss": 1.6071,
      "nll_loss": 1.5678837299346924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15519270300865173,
      "rewards/margins": 0.06397604942321777,
      "rewards/rejected": -0.2191687524318695,
      "step": 460
    },
    {
      "epoch": 0.7265563435776202,
      "grad_norm": 0.18149657547473907,
      "learning_rate": 3.2191576395014158e-06,
      "log_odds_chosen": 0.7072669863700867,
      "log_odds_ratio": -0.40362295508384705,
      "logits/chosen": -0.23113706707954407,
      "logits/rejected": -1.2299100160598755,
      "logps/chosen": -1.4852123260498047,
      "logps/rejected": -2.0701723098754883,
      "loss": 1.5528,
      "nll_loss": 1.5123920440673828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14852122962474823,
      "rewards/margins": 0.05849601700901985,
      "rewards/rejected": -0.20701724290847778,
      "step": 461
    },
    {
      "epoch": 0.7281323877068558,
      "grad_norm": 0.18942932784557343,
      "learning_rate": 3.2147953487538794e-06,
      "log_odds_chosen": 0.6708348989486694,
      "log_odds_ratio": -0.41837048530578613,
      "logits/chosen": -0.1631353497505188,
      "logits/rejected": -1.3306108713150024,
      "logps/chosen": -1.455315113067627,
      "logps/rejected": -2.0053393840789795,
      "loss": 1.5252,
      "nll_loss": 1.4833753108978271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14553150534629822,
      "rewards/margins": 0.05500244349241257,
      "rewards/rejected": -0.2005339413881302,
      "step": 462
    },
    {
      "epoch": 0.7297084318360915,
      "grad_norm": 0.20128397643566132,
      "learning_rate": 3.2104238808077133e-06,
      "log_odds_chosen": 0.5619516968727112,
      "log_odds_ratio": -0.4598609507083893,
      "logits/chosen": -0.1880580186843872,
      "logits/rejected": -1.0508508682250977,
      "logps/chosen": -1.4888397455215454,
      "logps/rejected": -1.9507625102996826,
      "loss": 1.5634,
      "nll_loss": 1.5174212455749512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14888396859169006,
      "rewards/margins": 0.04619229957461357,
      "rewards/rejected": -0.19507628679275513,
      "step": 463
    },
    {
      "epoch": 0.731284475965327,
      "grad_norm": 0.18228676915168762,
      "learning_rate": 3.2060432686872704e-06,
      "log_odds_chosen": 0.8119957447052002,
      "log_odds_ratio": -0.37603914737701416,
      "logits/chosen": -0.2555653750896454,
      "logits/rejected": -1.1474933624267578,
      "logps/chosen": -1.398220181465149,
      "logps/rejected": -2.059756278991699,
      "loss": 1.4829,
      "nll_loss": 1.4452736377716064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13982200622558594,
      "rewards/margins": 0.0661536231637001,
      "rewards/rejected": -0.20597563683986664,
      "step": 464
    },
    {
      "epoch": 0.7328605200945626,
      "grad_norm": 0.18841765820980072,
      "learning_rate": 3.201653545485982e-06,
      "log_odds_chosen": 0.6405482888221741,
      "log_odds_ratio": -0.4303835332393646,
      "logits/chosen": -0.16426052153110504,
      "logits/rejected": -1.2122530937194824,
      "logps/chosen": -1.5435234308242798,
      "logps/rejected": -2.079439640045166,
      "loss": 1.599,
      "nll_loss": 1.5559314489364624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1543523371219635,
      "rewards/margins": 0.053591638803482056,
      "rewards/rejected": -0.20794397592544556,
      "step": 465
    },
    {
      "epoch": 0.7344365642237982,
      "grad_norm": 0.20733259618282318,
      "learning_rate": 3.197254744366111e-06,
      "log_odds_chosen": 0.6555430889129639,
      "log_odds_ratio": -0.42545032501220703,
      "logits/chosen": -0.14602793753147125,
      "logits/rejected": -1.174269676208496,
      "logps/chosen": -1.4550955295562744,
      "logps/rejected": -1.9908273220062256,
      "loss": 1.53,
      "nll_loss": 1.4874444007873535,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14550955593585968,
      "rewards/margins": 0.05357319116592407,
      "rewards/rejected": -0.19908274710178375,
      "step": 466
    },
    {
      "epoch": 0.7360126083530338,
      "grad_norm": 0.19345013797283173,
      "learning_rate": 3.192846898558498e-06,
      "log_odds_chosen": 0.5251720547676086,
      "log_odds_ratio": -0.46669673919677734,
      "logits/chosen": -0.18853773176670074,
      "logits/rejected": -1.2136728763580322,
      "logps/chosen": -1.5849101543426514,
      "logps/rejected": -2.023404359817505,
      "loss": 1.654,
      "nll_loss": 1.6073112487792969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15849103033542633,
      "rewards/margins": 0.04384942352771759,
      "rewards/rejected": -0.20234045386314392,
      "step": 467
    },
    {
      "epoch": 0.7375886524822695,
      "grad_norm": 0.19730189442634583,
      "learning_rate": 3.188430041362313e-06,
      "log_odds_chosen": 0.47901207208633423,
      "log_odds_ratio": -0.49027061462402344,
      "logits/chosen": -0.15382762253284454,
      "logits/rejected": -1.2334424257278442,
      "logps/chosen": -1.579338550567627,
      "logps/rejected": -1.979215383529663,
      "loss": 1.637,
      "nll_loss": 1.5880184173583984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15793386101722717,
      "rewards/margins": 0.03998768329620361,
      "rewards/rejected": -0.1979215443134308,
      "step": 468
    },
    {
      "epoch": 0.7391646966115051,
      "grad_norm": 0.19508811831474304,
      "learning_rate": 3.184004206144803e-06,
      "log_odds_chosen": 0.7172151803970337,
      "log_odds_ratio": -0.40794268250465393,
      "logits/chosen": -0.25483450293540955,
      "logits/rejected": -1.2210605144500732,
      "logps/chosen": -1.4742763042449951,
      "logps/rejected": -2.062727689743042,
      "loss": 1.5429,
      "nll_loss": 1.5020675659179688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14742764830589294,
      "rewards/margins": 0.05884512513875961,
      "rewards/rejected": -0.20627278089523315,
      "step": 469
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 0.1892954707145691,
      "learning_rate": 3.1795694263410386e-06,
      "log_odds_chosen": 0.7759031653404236,
      "log_odds_ratio": -0.39132991433143616,
      "logits/chosen": -0.21951723098754883,
      "logits/rejected": -1.2244356870651245,
      "logps/chosen": -1.465575933456421,
      "logps/rejected": -2.1147849559783936,
      "loss": 1.5275,
      "nll_loss": 1.488350749015808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14655759930610657,
      "rewards/margins": 0.06492087990045547,
      "rewards/rejected": -0.21147848665714264,
      "step": 470
    },
    {
      "epoch": 0.7423167848699763,
      "grad_norm": 0.18874920904636383,
      "learning_rate": 3.1751257354536634e-06,
      "log_odds_chosen": 0.5416457056999207,
      "log_odds_ratio": -0.4711780548095703,
      "logits/chosen": -0.17236806452274323,
      "logits/rejected": -1.3529729843139648,
      "logps/chosen": -1.4607828855514526,
      "logps/rejected": -1.9070994853973389,
      "loss": 1.5414,
      "nll_loss": 1.4942772388458252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14607828855514526,
      "rewards/margins": 0.04463164880871773,
      "rewards/rejected": -0.1907099336385727,
      "step": 471
    },
    {
      "epoch": 0.7438928289992119,
      "grad_norm": 0.19573341310024261,
      "learning_rate": 3.1706731670526394e-06,
      "log_odds_chosen": 0.604232132434845,
      "log_odds_ratio": -0.43818584084510803,
      "logits/chosen": -0.27332159876823425,
      "logits/rejected": -1.2359827756881714,
      "logps/chosen": -1.5119805335998535,
      "logps/rejected": -2.008622884750366,
      "loss": 1.5662,
      "nll_loss": 1.5224015712738037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15119807422161102,
      "rewards/margins": 0.04966423660516739,
      "rewards/rejected": -0.20086230337619781,
      "step": 472
    },
    {
      "epoch": 0.7454688731284476,
      "grad_norm": 0.19741228222846985,
      "learning_rate": 3.166211754774994e-06,
      "log_odds_chosen": 0.6473110914230347,
      "log_odds_ratio": -0.4283471703529358,
      "logits/chosen": -0.2195882350206375,
      "logits/rejected": -1.3575206995010376,
      "logps/chosen": -1.5655333995819092,
      "logps/rejected": -2.1075003147125244,
      "loss": 1.652,
      "nll_loss": 1.609170913696289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15655335783958435,
      "rewards/margins": 0.054196685552597046,
      "rewards/rejected": -0.2107500433921814,
      "step": 473
    },
    {
      "epoch": 0.7470449172576832,
      "grad_norm": 0.19509533047676086,
      "learning_rate": 3.1617415323245665e-06,
      "log_odds_chosen": 0.6397197246551514,
      "log_odds_ratio": -0.43335115909576416,
      "logits/chosen": -0.26395586133003235,
      "logits/rejected": -1.3436589241027832,
      "logps/chosen": -1.4741613864898682,
      "logps/rejected": -1.9915375709533691,
      "loss": 1.5348,
      "nll_loss": 1.491437554359436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1474161297082901,
      "rewards/margins": 0.05173762887716293,
      "rewards/rejected": -0.19915376603603363,
      "step": 474
    },
    {
      "epoch": 0.7486209613869188,
      "grad_norm": 0.18101076781749725,
      "learning_rate": 3.157262533471752e-06,
      "log_odds_chosen": 0.690365731716156,
      "log_odds_ratio": -0.41062238812446594,
      "logits/chosen": -0.1729176789522171,
      "logits/rejected": -1.3418751955032349,
      "logps/chosen": -1.5249015092849731,
      "logps/rejected": -2.1028430461883545,
      "loss": 1.5936,
      "nll_loss": 1.552560806274414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15249013900756836,
      "rewards/margins": 0.057794176042079926,
      "rewards/rejected": -0.2102843075990677,
      "step": 475
    },
    {
      "epoch": 0.7501970055161544,
      "grad_norm": 0.19821086525917053,
      "learning_rate": 3.1527747920532468e-06,
      "log_odds_chosen": 0.5523171424865723,
      "log_odds_ratio": -0.4564129710197449,
      "logits/chosen": -0.1943480521440506,
      "logits/rejected": -1.095949649810791,
      "logps/chosen": -1.4880762100219727,
      "logps/rejected": -1.9403318166732788,
      "loss": 1.5629,
      "nll_loss": 1.517241358757019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14880762994289398,
      "rewards/margins": 0.04522555693984032,
      "rewards/rejected": -0.1940331906080246,
      "step": 476
    },
    {
      "epoch": 0.75177304964539,
      "grad_norm": 0.19268104434013367,
      "learning_rate": 3.148278341971795e-06,
      "log_odds_chosen": 0.6278610229492188,
      "log_odds_ratio": -0.4335193932056427,
      "logits/chosen": -0.24915321171283722,
      "logits/rejected": -1.1845955848693848,
      "logps/chosen": -1.5173267126083374,
      "logps/rejected": -2.037794351577759,
      "loss": 1.5831,
      "nll_loss": 1.5396989583969116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1517326831817627,
      "rewards/margins": 0.052046775817871094,
      "rewards/rejected": -0.2037794440984726,
      "step": 477
    },
    {
      "epoch": 0.7533490937746257,
      "grad_norm": 0.1977827399969101,
      "learning_rate": 3.143773217195929e-06,
      "log_odds_chosen": 0.7068151831626892,
      "log_odds_ratio": -0.40859490633010864,
      "logits/chosen": -0.26628124713897705,
      "logits/rejected": -1.3386417627334595,
      "logps/chosen": -1.5787765979766846,
      "logps/rejected": -2.174910306930542,
      "loss": 1.6357,
      "nll_loss": 1.5947997570037842,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15787766873836517,
      "rewards/margins": 0.05961337685585022,
      "rewards/rejected": -0.2174910455942154,
      "step": 478
    },
    {
      "epoch": 0.7549251379038613,
      "grad_norm": 0.19414526224136353,
      "learning_rate": 3.139259451759714e-06,
      "log_odds_chosen": 0.555619478225708,
      "log_odds_ratio": -0.45742952823638916,
      "logits/chosen": -0.2637663781642914,
      "logits/rejected": -1.2724649906158447,
      "logps/chosen": -1.5820374488830566,
      "logps/rejected": -2.045403480529785,
      "loss": 1.6538,
      "nll_loss": 1.6080107688903809,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15820375084877014,
      "rewards/margins": 0.04633659869432449,
      "rewards/rejected": -0.20454034209251404,
      "step": 479
    },
    {
      "epoch": 0.7565011820330969,
      "grad_norm": 0.1868637353181839,
      "learning_rate": 3.134737079762493e-06,
      "log_odds_chosen": 0.5565078854560852,
      "log_odds_ratio": -0.45784488320350647,
      "logits/chosen": -0.21357539296150208,
      "logits/rejected": -1.2584372758865356,
      "logps/chosen": -1.5111368894577026,
      "logps/rejected": -1.9670014381408691,
      "loss": 1.5766,
      "nll_loss": 1.5307987928390503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15111368894577026,
      "rewards/margins": 0.045586466789245605,
      "rewards/rejected": -0.19670015573501587,
      "step": 480
    },
    {
      "epoch": 0.7580772261623325,
      "grad_norm": 0.2013820856809616,
      "learning_rate": 3.130206135368626e-06,
      "log_odds_chosen": 0.5862029194831848,
      "log_odds_ratio": -0.4538579285144806,
      "logits/chosen": -0.2686524987220764,
      "logits/rejected": -0.9783179759979248,
      "logps/chosen": -1.43379545211792,
      "logps/rejected": -1.9084105491638184,
      "loss": 1.524,
      "nll_loss": 1.478649616241455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1433795690536499,
      "rewards/margins": 0.04746149852871895,
      "rewards/rejected": -0.19084104895591736,
      "step": 481
    },
    {
      "epoch": 0.7596532702915682,
      "grad_norm": 0.19040453433990479,
      "learning_rate": 3.1256666528072327e-06,
      "log_odds_chosen": 0.755387544631958,
      "log_odds_ratio": -0.39063921570777893,
      "logits/chosen": -0.25499293208122253,
      "logits/rejected": -1.0305991172790527,
      "logps/chosen": -1.4815821647644043,
      "logps/rejected": -2.1069517135620117,
      "loss": 1.5446,
      "nll_loss": 1.5055186748504639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1481582075357437,
      "rewards/margins": 0.06253695487976074,
      "rewards/rejected": -0.21069514751434326,
      "step": 482
    },
    {
      "epoch": 0.7612293144208038,
      "grad_norm": 0.18193352222442627,
      "learning_rate": 3.121118666371937e-06,
      "log_odds_chosen": 0.5603374242782593,
      "log_odds_ratio": -0.4579220414161682,
      "logits/chosen": -0.182877317070961,
      "logits/rejected": -1.5083105564117432,
      "logps/chosen": -1.5461608171463013,
      "logps/rejected": -2.0124778747558594,
      "loss": 1.6023,
      "nll_loss": 1.5564837455749512,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1546160876750946,
      "rewards/margins": 0.046631716191768646,
      "rewards/rejected": -0.20124779641628265,
      "step": 483
    },
    {
      "epoch": 0.7628053585500394,
      "grad_norm": 0.20136219263076782,
      "learning_rate": 3.1165622104206034e-06,
      "log_odds_chosen": 0.7325907349586487,
      "log_odds_ratio": -0.39630693197250366,
      "logits/chosen": -0.3966868817806244,
      "logits/rejected": -1.1748623847961426,
      "logps/chosen": -1.4722321033477783,
      "logps/rejected": -2.079011917114258,
      "loss": 1.5403,
      "nll_loss": 1.500671148300171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14722320437431335,
      "rewards/margins": 0.06067798286676407,
      "rewards/rejected": -0.20790117979049683,
      "step": 484
    },
    {
      "epoch": 0.764381402679275,
      "grad_norm": 0.18245038390159607,
      "learning_rate": 3.1119973193750816e-06,
      "log_odds_chosen": 0.6451510190963745,
      "log_odds_ratio": -0.4264066815376282,
      "logits/chosen": -0.2894980013370514,
      "logits/rejected": -1.225103497505188,
      "logps/chosen": -1.4829157590866089,
      "logps/rejected": -2.0141286849975586,
      "loss": 1.5432,
      "nll_loss": 1.500580906867981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14829157292842865,
      "rewards/margins": 0.05312127619981766,
      "rewards/rejected": -0.2014128416776657,
      "step": 485
    },
    {
      "epoch": 0.7659574468085106,
      "grad_norm": 0.19604384899139404,
      "learning_rate": 3.1074240277209408e-06,
      "log_odds_chosen": 0.5782766938209534,
      "log_odds_ratio": -0.4525739848613739,
      "logits/chosen": -0.2679707705974579,
      "logits/rejected": -1.1799029111862183,
      "logps/chosen": -1.507366418838501,
      "logps/rejected": -1.9800548553466797,
      "loss": 1.5533,
      "nll_loss": 1.5080450773239136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15073664486408234,
      "rewards/margins": 0.04726884141564369,
      "rewards/rejected": -0.19800546765327454,
      "step": 486
    },
    {
      "epoch": 0.7675334909377463,
      "grad_norm": 0.21242745220661163,
      "learning_rate": 3.102842370007217e-06,
      "log_odds_chosen": 0.654798150062561,
      "log_odds_ratio": -0.42668747901916504,
      "logits/chosen": -0.18659040331840515,
      "logits/rejected": -1.0967789888381958,
      "logps/chosen": -1.5160582065582275,
      "logps/rejected": -2.0530829429626465,
      "loss": 1.5815,
      "nll_loss": 1.538811206817627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15160581469535828,
      "rewards/margins": 0.0537024661898613,
      "rewards/rejected": -0.20530828833580017,
      "step": 487
    },
    {
      "epoch": 0.7691095350669819,
      "grad_norm": 0.19127397239208221,
      "learning_rate": 3.0982523808461454e-06,
      "log_odds_chosen": 0.5358982086181641,
      "log_odds_ratio": -0.4667418897151947,
      "logits/chosen": -0.18278566002845764,
      "logits/rejected": -1.2197632789611816,
      "logps/chosen": -1.5576581954956055,
      "logps/rejected": -1.9974056482315063,
      "loss": 1.6145,
      "nll_loss": 1.567873239517212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557658314704895,
      "rewards/margins": 0.04397474229335785,
      "rewards/rejected": -0.19974057376384735,
      "step": 488
    },
    {
      "epoch": 0.7706855791962175,
      "grad_norm": 0.18905805051326752,
      "learning_rate": 3.0936540949129006e-06,
      "log_odds_chosen": 0.6154669523239136,
      "log_odds_ratio": -0.4424894452095032,
      "logits/chosen": -0.24286700785160065,
      "logits/rejected": -1.1417758464813232,
      "logps/chosen": -1.5341846942901611,
      "logps/rejected": -2.046876907348633,
      "loss": 1.5994,
      "nll_loss": 1.5551624298095703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15341848134994507,
      "rewards/margins": 0.05126923322677612,
      "rewards/rejected": -0.20468769967556,
      "step": 489
    },
    {
      "epoch": 0.7722616233254531,
      "grad_norm": 0.17579598724842072,
      "learning_rate": 3.0890475469453378e-06,
      "log_odds_chosen": 0.7146469354629517,
      "log_odds_ratio": -0.40994906425476074,
      "logits/chosen": -0.24190282821655273,
      "logits/rejected": -1.3688513040542603,
      "logps/chosen": -1.4117985963821411,
      "logps/rejected": -1.9928693771362305,
      "loss": 1.4803,
      "nll_loss": 1.439333438873291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1411798745393753,
      "rewards/margins": 0.05810706689953804,
      "rewards/rejected": -0.19928693771362305,
      "step": 490
    },
    {
      "epoch": 0.7738376674546887,
      "grad_norm": 0.1882636994123459,
      "learning_rate": 3.0844327717437263e-06,
      "log_odds_chosen": 0.6613863110542297,
      "log_odds_ratio": -0.4273605942726135,
      "logits/chosen": -0.23976236581802368,
      "logits/rejected": -1.243269443511963,
      "logps/chosen": -1.437576413154602,
      "logps/rejected": -1.9814512729644775,
      "loss": 1.5097,
      "nll_loss": 1.4669883251190186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1437576413154602,
      "rewards/margins": 0.05438747629523277,
      "rewards/rejected": -0.19814513623714447,
      "step": 491
    },
    {
      "epoch": 0.7754137115839244,
      "grad_norm": 0.18878327310085297,
      "learning_rate": 3.0798098041704892e-06,
      "log_odds_chosen": 0.478700190782547,
      "log_odds_ratio": -0.4873453974723816,
      "logits/chosen": -0.18805843591690063,
      "logits/rejected": -1.271687388420105,
      "logps/chosen": -1.3689371347427368,
      "logps/rejected": -1.7501403093338013,
      "loss": 1.4618,
      "nll_loss": 1.4130362272262573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13689371943473816,
      "rewards/margins": 0.03812030330300331,
      "rewards/rejected": -0.17501404881477356,
      "step": 492
    },
    {
      "epoch": 0.77698975571316,
      "grad_norm": 0.19235889613628387,
      "learning_rate": 3.0751786791499368e-06,
      "log_odds_chosen": 0.6095344424247742,
      "log_odds_ratio": -0.44119495153427124,
      "logits/chosen": -0.20893912017345428,
      "logits/rejected": -1.2534291744232178,
      "logps/chosen": -1.5530734062194824,
      "logps/rejected": -2.0629472732543945,
      "loss": 1.6223,
      "nll_loss": 1.5781906843185425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.155307337641716,
      "rewards/margins": 0.050987400114536285,
      "rewards/rejected": -0.2062947303056717,
      "step": 493
    },
    {
      "epoch": 0.7785657998423956,
      "grad_norm": 0.18117356300354004,
      "learning_rate": 3.070539431668008e-06,
      "log_odds_chosen": 0.6075527667999268,
      "log_odds_ratio": -0.4399612247943878,
      "logits/chosen": -0.2257227599620819,
      "logits/rejected": -1.1582225561141968,
      "logps/chosen": -1.5574135780334473,
      "logps/rejected": -2.067356824874878,
      "loss": 1.6169,
      "nll_loss": 1.5728775262832642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1557413637638092,
      "rewards/margins": 0.05099431425333023,
      "rewards/rejected": -0.20673568546772003,
      "step": 494
    },
    {
      "epoch": 0.7801418439716312,
      "grad_norm": 0.1898123025894165,
      "learning_rate": 3.0658920967720018e-06,
      "log_odds_chosen": 0.7518050074577332,
      "log_odds_ratio": -0.39490705728530884,
      "logits/chosen": -0.35575562715530396,
      "logits/rejected": -1.1922754049301147,
      "logps/chosen": -1.4951189756393433,
      "logps/rejected": -2.1237242221832275,
      "loss": 1.5649,
      "nll_loss": 1.525448203086853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1495118886232376,
      "rewards/margins": 0.06286052614450455,
      "rewards/rejected": -0.21237242221832275,
      "step": 495
    },
    {
      "epoch": 0.7817178881008668,
      "grad_norm": 0.1872013807296753,
      "learning_rate": 3.0612367095703116e-06,
      "log_odds_chosen": 0.6950463652610779,
      "log_odds_ratio": -0.4093022644519806,
      "logits/chosen": -0.21190622448921204,
      "logits/rejected": -1.4540730714797974,
      "logps/chosen": -1.5332351922988892,
      "logps/rejected": -2.1157336235046387,
      "loss": 1.5885,
      "nll_loss": 1.5475294589996338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15332353115081787,
      "rewards/margins": 0.05824984610080719,
      "rewards/rejected": -0.21157337725162506,
      "step": 496
    },
    {
      "epoch": 0.7832939322301025,
      "grad_norm": 0.18900524079799652,
      "learning_rate": 3.056573305232167e-06,
      "log_odds_chosen": 0.7527428865432739,
      "log_odds_ratio": -0.392995148897171,
      "logits/chosen": -0.2251337319612503,
      "logits/rejected": -1.3849214315414429,
      "logps/chosen": -1.4837510585784912,
      "logps/rejected": -2.1036033630371094,
      "loss": 1.5498,
      "nll_loss": 1.510545253753662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14837510883808136,
      "rewards/margins": 0.061985231935977936,
      "rewards/rejected": -0.2103603333234787,
      "step": 497
    },
    {
      "epoch": 0.7848699763593381,
      "grad_norm": 0.18676206469535828,
      "learning_rate": 3.051901918987359e-06,
      "log_odds_chosen": 0.7129929065704346,
      "log_odds_ratio": -0.4029238224029541,
      "logits/chosen": -0.3676823675632477,
      "logits/rejected": -1.319760799407959,
      "logps/chosen": -1.4286247491836548,
      "logps/rejected": -2.0117030143737793,
      "loss": 1.4979,
      "nll_loss": 1.4575592279434204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1428624838590622,
      "rewards/margins": 0.058307845145463943,
      "rewards/rejected": -0.20117032527923584,
      "step": 498
    },
    {
      "epoch": 0.7864460204885737,
      "grad_norm": 0.2265917956829071,
      "learning_rate": 3.047222586125979e-06,
      "log_odds_chosen": 0.725178599357605,
      "log_odds_ratio": -0.39900532364845276,
      "logits/chosen": -0.19229286909103394,
      "logits/rejected": -0.8906416296958923,
      "logps/chosen": -1.4400482177734375,
      "logps/rejected": -2.0356321334838867,
      "loss": 1.5177,
      "nll_loss": 1.4778459072113037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14400483667850494,
      "rewards/margins": 0.059558384120464325,
      "rewards/rejected": -0.20356319844722748,
      "step": 499
    },
    {
      "epoch": 0.7880220646178093,
      "grad_norm": 0.18137092888355255,
      "learning_rate": 3.042535341998152e-06,
      "log_odds_chosen": 0.5409306883811951,
      "log_odds_ratio": -0.46464020013809204,
      "logits/chosen": -0.10464775562286377,
      "logits/rejected": -1.2362757921218872,
      "logps/chosen": -1.5858546495437622,
      "logps/rejected": -2.039395332336426,
      "loss": 1.6316,
      "nll_loss": 1.5851597785949707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15858545899391174,
      "rewards/margins": 0.04535408318042755,
      "rewards/rejected": -0.2039395570755005,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 1270,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}