{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 3750,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.04,
      "grad_norm": 24.803234100341797,
      "learning_rate": 1.3066666666666667e-06,
      "logits/chosen": 2.3097052574157715,
      "logits/rejected": 2.3757848739624023,
      "logps/chosen": -67.16219329833984,
      "logps/rejected": -80.8543930053711,
      "loss": 1.0777,
      "nll_loss": 0.41877487301826477,
      "rewards/accuracies": 0.7237499952316284,
      "rewards/chosen": -6.716219425201416,
      "rewards/margins": 1.3692210912704468,
      "rewards/rejected": -8.085439682006836,
      "step": 50
    },
    {
      "epoch": 0.08,
      "grad_norm": 12.049943923950195,
      "learning_rate": 2.64e-06,
      "logits/chosen": 2.2844557762145996,
      "logits/rejected": 2.364481210708618,
      "logps/chosen": -61.06942367553711,
      "logps/rejected": -75.95172119140625,
      "loss": 1.0254,
      "nll_loss": 0.38137227296829224,
      "rewards/accuracies": 0.7174999713897705,
      "rewards/chosen": -6.106942176818848,
      "rewards/margins": 1.4882298707962036,
      "rewards/rejected": -7.595171928405762,
      "step": 100
    },
    {
      "epoch": 0.12,
      "grad_norm": 19.429025650024414,
      "learning_rate": 3.973333333333333e-06,
      "logits/chosen": 2.2412264347076416,
      "logits/rejected": 2.2957522869110107,
      "logps/chosen": -56.071502685546875,
      "logps/rejected": -67.70003509521484,
      "loss": 1.0618,
      "nll_loss": 0.3372814655303955,
      "rewards/accuracies": 0.6650000214576721,
      "rewards/chosen": -5.607150077819824,
      "rewards/margins": 1.1628532409667969,
      "rewards/rejected": -6.770003795623779,
      "step": 150
    },
    {
      "epoch": 0.16,
      "grad_norm": 14.493091583251953,
      "learning_rate": 5.306666666666667e-06,
      "logits/chosen": 2.1775848865509033,
      "logits/rejected": 2.25844144821167,
      "logps/chosen": -50.50484848022461,
      "logps/rejected": -64.37080383300781,
      "loss": 0.9298,
      "nll_loss": 0.3056701719760895,
      "rewards/accuracies": 0.7237499952316284,
      "rewards/chosen": -5.050484657287598,
      "rewards/margins": 1.3865959644317627,
      "rewards/rejected": -6.437079906463623,
      "step": 200
    },
    {
      "epoch": 0.2,
      "grad_norm": 17.336822509765625,
      "learning_rate": 6.640000000000001e-06,
      "logits/chosen": 2.203662872314453,
      "logits/rejected": 2.28377103805542,
      "logps/chosen": -51.80202102661133,
      "logps/rejected": -65.30438232421875,
      "loss": 0.9291,
      "nll_loss": 0.2998887896537781,
      "rewards/accuracies": 0.7149999737739563,
      "rewards/chosen": -5.180202007293701,
      "rewards/margins": 1.3502373695373535,
      "rewards/rejected": -6.530439376831055,
      "step": 250
    },
    {
      "epoch": 0.24,
      "grad_norm": 25.542375564575195,
      "learning_rate": 7.973333333333334e-06,
      "logits/chosen": 2.2555084228515625,
      "logits/rejected": 2.3327550888061523,
      "logps/chosen": -49.52461242675781,
      "logps/rejected": -64.92198181152344,
      "loss": 0.8288,
      "nll_loss": 0.29204535484313965,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": -4.9524617195129395,
      "rewards/margins": 1.5397369861602783,
      "rewards/rejected": -6.492198467254639,
      "step": 300
    },
    {
      "epoch": 0.28,
      "grad_norm": 10.780914306640625,
      "learning_rate": 9.306666666666667e-06,
      "logits/chosen": 2.2995264530181885,
      "logits/rejected": 2.3702523708343506,
      "logps/chosen": -49.0943489074707,
      "logps/rejected": -62.10791015625,
      "loss": 0.9434,
      "nll_loss": 0.29649874567985535,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -4.909435272216797,
      "rewards/margins": 1.3013556003570557,
      "rewards/rejected": -6.210791110992432,
      "step": 350
    },
    {
      "epoch": 0.32,
      "grad_norm": 11.368802070617676,
      "learning_rate": 9.998752338940612e-06,
      "logits/chosen": 2.4777705669403076,
      "logits/rejected": 2.5543289184570312,
      "logps/chosen": -46.3427848815918,
      "logps/rejected": -58.78908157348633,
      "loss": 0.8661,
      "nll_loss": 0.27519530057907104,
      "rewards/accuracies": 0.7137500047683716,
      "rewards/chosen": -4.634278297424316,
      "rewards/margins": 1.2446298599243164,
      "rewards/rejected": -5.878908634185791,
      "step": 400
    },
    {
      "epoch": 0.36,
      "grad_norm": 13.878002166748047,
      "learning_rate": 9.988142751731797e-06,
      "logits/chosen": 2.4681615829467773,
      "logits/rejected": 2.540001153945923,
      "logps/chosen": -46.92790985107422,
      "logps/rejected": -63.343467712402344,
      "loss": 0.8081,
      "nll_loss": 0.2872486710548401,
      "rewards/accuracies": 0.7412499785423279,
      "rewards/chosen": -4.692790985107422,
      "rewards/margins": 1.6415550708770752,
      "rewards/rejected": -6.334346294403076,
      "step": 450
    },
    {
      "epoch": 0.4,
      "grad_norm": 14.774677276611328,
      "learning_rate": 9.966729958067638e-06,
      "logits/chosen": 2.509334087371826,
      "logits/rejected": 2.5635790824890137,
      "logps/chosen": -48.37749481201172,
      "logps/rejected": -64.00433349609375,
      "loss": 0.873,
      "nll_loss": 0.2907007336616516,
      "rewards/accuracies": 0.7200000286102295,
      "rewards/chosen": -4.837749481201172,
      "rewards/margins": 1.5626837015151978,
      "rewards/rejected": -6.400432586669922,
      "step": 500
    },
    {
      "epoch": 0.44,
      "grad_norm": 18.610092163085938,
      "learning_rate": 9.934560333291077e-06,
      "logits/chosen": 2.5441677570343018,
      "logits/rejected": 2.613149404525757,
      "logps/chosen": -48.265201568603516,
      "logps/rejected": -63.49806213378906,
      "loss": 0.8581,
      "nll_loss": 0.2854183316230774,
      "rewards/accuracies": 0.7287499904632568,
      "rewards/chosen": -4.826520919799805,
      "rewards/margins": 1.5232864618301392,
      "rewards/rejected": -6.349807262420654,
      "step": 550
    },
    {
      "epoch": 0.48,
      "grad_norm": 13.354905128479004,
      "learning_rate": 9.89170354964594e-06,
      "logits/chosen": 2.646564245223999,
      "logits/rejected": 2.700756549835205,
      "logps/chosen": -45.61009216308594,
      "logps/rejected": -60.2086067199707,
      "loss": 0.8363,
      "nll_loss": 0.27716541290283203,
      "rewards/accuracies": 0.7300000190734863,
      "rewards/chosen": -4.561009883880615,
      "rewards/margins": 1.4598517417907715,
      "rewards/rejected": -6.0208611488342285,
      "step": 600
    },
    {
      "epoch": 0.52,
      "grad_norm": 5.81559419631958,
      "learning_rate": 9.83825242538238e-06,
      "logits/chosen": 2.519411087036133,
      "logits/rejected": 2.5865354537963867,
      "logps/chosen": -46.41543960571289,
      "logps/rejected": -63.667991638183594,
      "loss": 0.7762,
      "nll_loss": 0.27636775374412537,
      "rewards/accuracies": 0.7587500214576721,
      "rewards/chosen": -4.641544342041016,
      "rewards/margins": 1.7252554893493652,
      "rewards/rejected": -6.366799831390381,
      "step": 650
    },
    {
      "epoch": 0.56,
      "grad_norm": 9.440147399902344,
      "learning_rate": 9.774322723733216e-06,
      "logits/chosen": 2.4503791332244873,
      "logits/rejected": 2.5173845291137695,
      "logps/chosen": -44.43014144897461,
      "logps/rejected": -62.06157302856445,
      "loss": 0.7447,
      "nll_loss": 0.2715470492839813,
      "rewards/accuracies": 0.7724999785423279,
      "rewards/chosen": -4.443014621734619,
      "rewards/margins": 1.7631423473358154,
      "rewards/rejected": -6.206157207489014,
      "step": 700
    },
    {
      "epoch": 0.6,
      "grad_norm": 9.054909706115723,
      "learning_rate": 9.700052902196541e-06,
      "logits/chosen": 2.3488733768463135,
      "logits/rejected": 2.4224016666412354,
      "logps/chosen": -45.02783203125,
      "logps/rejected": -63.32554626464844,
      "loss": 0.7256,
      "nll_loss": 0.27446305751800537,
      "rewards/accuracies": 0.8075000047683716,
      "rewards/chosen": -4.502783298492432,
      "rewards/margins": 1.8297713994979858,
      "rewards/rejected": -6.332554817199707,
      "step": 750
    },
    {
      "epoch": 0.64,
      "grad_norm": 16.773876190185547,
      "learning_rate": 9.615603812667618e-06,
      "logits/chosen": 2.4701356887817383,
      "logits/rejected": 2.5356545448303223,
      "logps/chosen": -44.635948181152344,
      "logps/rejected": -62.55714416503906,
      "loss": 0.7836,
      "nll_loss": 0.2690128684043884,
      "rewards/accuracies": 0.7524999976158142,
      "rewards/chosen": -4.463594913482666,
      "rewards/margins": 1.7921198606491089,
      "rewards/rejected": -6.2557148933410645,
      "step": 800
    },
    {
      "epoch": 0.68,
      "grad_norm": 12.325571060180664,
      "learning_rate": 9.521158353069494e-06,
      "logits/chosen": 2.450385332107544,
      "logits/rejected": 2.5140254497528076,
      "logps/chosen": -47.131587982177734,
      "logps/rejected": -65.35118865966797,
      "loss": 0.7507,
      "nll_loss": 0.27970045804977417,
      "rewards/accuracies": 0.7762500047683716,
      "rewards/chosen": -4.713159561157227,
      "rewards/margins": 1.8219600915908813,
      "rewards/rejected": -6.535119533538818,
      "step": 850
    },
    {
      "epoch": 0.72,
      "grad_norm": 18.231292724609375,
      "learning_rate": 9.416921071236821e-06,
      "logits/chosen": 2.4923791885375977,
      "logits/rejected": 2.5595548152923584,
      "logps/chosen": -46.485965728759766,
      "logps/rejected": -63.454063415527344,
      "loss": 0.8048,
      "nll_loss": 0.27803030610084534,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -4.648596286773682,
      "rewards/margins": 1.6968092918395996,
      "rewards/rejected": -6.345406532287598,
      "step": 900
    },
    {
      "epoch": 0.76,
      "grad_norm": 18.762359619140625,
      "learning_rate": 9.303117721910801e-06,
      "logits/chosen": 2.571411371231079,
      "logits/rejected": 2.6245839595794678,
      "logps/chosen": -44.32810974121094,
      "logps/rejected": -61.370662689208984,
      "loss": 0.7592,
      "nll_loss": 0.2649703919887543,
      "rewards/accuracies": 0.7662500143051147,
      "rewards/chosen": -4.4328107833862305,
      "rewards/margins": 1.7042557001113892,
      "rewards/rejected": -6.137066841125488,
      "step": 950
    },
    {
      "epoch": 0.8,
      "grad_norm": 13.24950122833252,
      "learning_rate": 9.179994777804677e-06,
      "logits/chosen": 2.5039403438568115,
      "logits/rejected": 2.5774261951446533,
      "logps/chosen": -45.20160675048828,
      "logps/rejected": -62.881683349609375,
      "loss": 0.7717,
      "nll_loss": 0.2698582708835602,
      "rewards/accuracies": 0.7787500023841858,
      "rewards/chosen": -4.520160675048828,
      "rewards/margins": 1.7680076360702515,
      "rewards/rejected": -6.288168430328369,
      "step": 1000
    },
    {
      "epoch": 0.84,
      "grad_norm": 12.429143905639648,
      "learning_rate": 9.04781889579873e-06,
      "logits/chosen": 2.5070714950561523,
      "logits/rejected": 2.5675339698791504,
      "logps/chosen": -45.09945297241211,
      "logps/rejected": -64.37616729736328,
      "loss": 0.7095,
      "nll_loss": 0.2686724364757538,
      "rewards/accuracies": 0.7912499904632568,
      "rewards/chosen": -4.509944915771484,
      "rewards/margins": 1.9276715517044067,
      "rewards/rejected": -6.437616348266602,
      "step": 1050
    },
    {
      "epoch": 0.88,
      "grad_norm": 13.30483341217041,
      "learning_rate": 8.906876339420851e-06,
      "logits/chosen": 2.388920783996582,
      "logits/rejected": 2.4621076583862305,
      "logps/chosen": -42.8404541015625,
      "logps/rejected": -62.490779876708984,
      "loss": 0.6913,
      "nll_loss": 0.25781476497650146,
      "rewards/accuracies": 0.8112499713897705,
      "rewards/chosen": -4.284045219421387,
      "rewards/margins": 1.9650330543518066,
      "rewards/rejected": -6.249078750610352,
      "step": 1100
    },
    {
      "epoch": 0.92,
      "grad_norm": 13.460766792297363,
      "learning_rate": 8.757472358863481e-06,
      "logits/chosen": 2.4224061965942383,
      "logits/rejected": 2.5014472007751465,
      "logps/chosen": -45.438575744628906,
      "logps/rejected": -63.852115631103516,
      "loss": 0.7312,
      "nll_loss": 0.26547786593437195,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -4.543857574462891,
      "rewards/margins": 1.841354250907898,
      "rewards/rejected": -6.385211944580078,
      "step": 1150
    },
    {
      "epoch": 0.96,
      "grad_norm": 9.995412826538086,
      "learning_rate": 8.599930529879669e-06,
      "logits/chosen": 2.289586067199707,
      "logits/rejected": 2.3660268783569336,
      "logps/chosen": -43.745357513427734,
      "logps/rejected": -62.561038970947266,
      "loss": 0.6825,
      "nll_loss": 0.2639794647693634,
      "rewards/accuracies": 0.8100000023841858,
      "rewards/chosen": -4.37453556060791,
      "rewards/margins": 1.8815685510635376,
      "rewards/rejected": -6.256104946136475,
      "step": 1200
    },
    {
      "epoch": 1.0,
      "grad_norm": 9.7256498336792,
      "learning_rate": 8.434592052990044e-06,
      "logits/chosen": 2.1411895751953125,
      "logits/rejected": 2.216475248336792,
      "logps/chosen": -41.174503326416016,
      "logps/rejected": -59.75578308105469,
      "loss": 0.687,
      "nll_loss": 0.25248458981513977,
      "rewards/accuracies": 0.7912499904632568,
      "rewards/chosen": -4.11745023727417,
      "rewards/margins": 1.8581281900405884,
      "rewards/rejected": -5.9755778312683105,
      "step": 1250
    },
    {
      "epoch": 1.04,
      "grad_norm": 5.8602166175842285,
      "learning_rate": 8.261815014518465e-06,
      "logits/chosen": 2.094317674636841,
      "logits/rejected": 2.1707794666290283,
      "logps/chosen": -42.47966003417969,
      "logps/rejected": -67.64102172851562,
      "loss": 0.5271,
      "nll_loss": 0.25495222210884094,
      "rewards/accuracies": 0.8837500214576721,
      "rewards/chosen": -4.247966289520264,
      "rewards/margins": 2.516136884689331,
      "rewards/rejected": -6.764101982116699,
      "step": 1300
    },
    {
      "epoch": 1.08,
      "grad_norm": 7.209846019744873,
      "learning_rate": 8.081973611056784e-06,
      "logits/chosen": 1.899792194366455,
      "logits/rejected": 1.9656882286071777,
      "logps/chosen": -41.67923355102539,
      "logps/rejected": -67.45558166503906,
      "loss": 0.5335,
      "nll_loss": 0.24939106404781342,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": -4.167923450469971,
      "rewards/margins": 2.577634811401367,
      "rewards/rejected": -6.745559215545654,
      "step": 1350
    },
    {
      "epoch": 1.12,
      "grad_norm": 12.841861724853516,
      "learning_rate": 7.89545733903834e-06,
      "logits/chosen": 1.8675593137741089,
      "logits/rejected": 1.9421709775924683,
      "logps/chosen": -43.389644622802734,
      "logps/rejected": -68.71646881103516,
      "loss": 0.5398,
      "nll_loss": 0.2613371014595032,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": -4.338963985443115,
      "rewards/margins": 2.5326828956604004,
      "rewards/rejected": -6.871647834777832,
      "step": 1400
    },
    {
      "epoch": 1.16,
      "grad_norm": 13.226041793823242,
      "learning_rate": 7.702670151175435e-06,
      "logits/chosen": 1.8003557920455933,
      "logits/rejected": 1.877779245376587,
      "logps/chosen": -42.74818801879883,
      "logps/rejected": -67.07624053955078,
      "loss": 0.5569,
      "nll_loss": 0.25505152344703674,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -4.274819374084473,
      "rewards/margins": 2.4328057765960693,
      "rewards/rejected": -6.707624435424805,
      "step": 1450
    },
    {
      "epoch": 1.2,
      "grad_norm": 6.528555393218994,
      "learning_rate": 7.5040295815877e-06,
      "logits/chosen": 1.8589718341827393,
      "logits/rejected": 1.9315310716629028,
      "logps/chosen": -42.12541198730469,
      "logps/rejected": -68.34070587158203,
      "loss": 0.5316,
      "nll_loss": 0.25116249918937683,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": -4.212540626525879,
      "rewards/margins": 2.621530055999756,
      "rewards/rejected": -6.834071636199951,
      "step": 1500
    },
    {
      "epoch": 1.24,
      "grad_norm": 8.806782722473145,
      "learning_rate": 7.299965841516164e-06,
      "logits/chosen": 1.896395206451416,
      "logits/rejected": 1.9738116264343262,
      "logps/chosen": -42.76396560668945,
      "logps/rejected": -68.06053161621094,
      "loss": 0.5404,
      "nll_loss": 0.2575341463088989,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -4.27639627456665,
      "rewards/margins": 2.5296568870544434,
      "rewards/rejected": -6.8060526847839355,
      "step": 1550
    },
    {
      "epoch": 1.28,
      "grad_norm": 13.923650741577148,
      "learning_rate": 7.090920887581507e-06,
      "logits/chosen": 1.881974220275879,
      "logits/rejected": 1.9411964416503906,
      "logps/chosen": -41.719696044921875,
      "logps/rejected": -66.24463653564453,
      "loss": 0.5483,
      "nll_loss": 0.24970975518226624,
      "rewards/accuracies": 0.8762500286102295,
      "rewards/chosen": -4.171969413757324,
      "rewards/margins": 2.452493667602539,
      "rewards/rejected": -6.62446403503418,
      "step": 1600
    },
    {
      "epoch": 1.32,
      "grad_norm": 11.483983993530273,
      "learning_rate": 6.877347464604446e-06,
      "logits/chosen": 1.8503919839859009,
      "logits/rejected": 1.9176833629608154,
      "logps/chosen": -42.25181579589844,
      "logps/rejected": -67.97564697265625,
      "loss": 0.541,
      "nll_loss": 0.25359034538269043,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.225182056427002,
      "rewards/margins": 2.572382926940918,
      "rewards/rejected": -6.797564506530762,
      "step": 1650
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 9.980610847473145,
      "learning_rate": 6.659708125061242e-06,
      "logits/chosen": 1.7879056930541992,
      "logits/rejected": 1.8512917757034302,
      "logps/chosen": -41.724449157714844,
      "logps/rejected": -68.18392181396484,
      "loss": 0.5127,
      "nll_loss": 0.2496194839477539,
      "rewards/accuracies": 0.9075000286102295,
      "rewards/chosen": -4.172445297241211,
      "rewards/margins": 2.645946741104126,
      "rewards/rejected": -6.818392276763916,
      "step": 1700
    },
    {
      "epoch": 1.4,
      "grad_norm": 8.7877779006958,
      "learning_rate": 6.438474227298065e-06,
      "logits/chosen": 1.813609004020691,
      "logits/rejected": 1.8792080879211426,
      "logps/chosen": -42.19902420043945,
      "logps/rejected": -67.10396575927734,
      "loss": 0.5475,
      "nll_loss": 0.2563365399837494,
      "rewards/accuracies": 0.8887500166893005,
      "rewards/chosen": -4.219902038574219,
      "rewards/margins": 2.4904944896698,
      "rewards/rejected": -6.710396766662598,
      "step": 1750
    },
    {
      "epoch": 1.44,
      "grad_norm": 9.004395484924316,
      "learning_rate": 6.2141249146737545e-06,
      "logits/chosen": 1.8060498237609863,
      "logits/rejected": 1.8766037225723267,
      "logps/chosen": -40.13544845581055,
      "logps/rejected": -64.01081085205078,
      "loss": 0.5417,
      "nll_loss": 0.23975929617881775,
      "rewards/accuracies": 0.8712499737739563,
      "rewards/chosen": -4.013545036315918,
      "rewards/margins": 2.3875370025634766,
      "rewards/rejected": -6.401081562042236,
      "step": 1800
    },
    {
      "epoch": 1.48,
      "grad_norm": 14.659512519836426,
      "learning_rate": 5.987146077842015e-06,
      "logits/chosen": 1.7913141250610352,
      "logits/rejected": 1.8706274032592773,
      "logps/chosen": -41.8923454284668,
      "logps/rejected": -68.07263946533203,
      "loss": 0.5291,
      "nll_loss": 0.24965395033359528,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.189234733581543,
      "rewards/margins": 2.6180291175842285,
      "rewards/rejected": -6.8072638511657715,
      "step": 1850
    },
    {
      "epoch": 1.52,
      "grad_norm": 6.758883953094482,
      "learning_rate": 5.7580293024204455e-06,
      "logits/chosen": 1.7456320524215698,
      "logits/rejected": 1.8188166618347168,
      "logps/chosen": -40.59235763549805,
      "logps/rejected": -67.43293762207031,
      "loss": 0.4978,
      "nll_loss": 0.2416204810142517,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": -4.0592360496521,
      "rewards/margins": 2.6840572357177734,
      "rewards/rejected": -6.743292808532715,
      "step": 1900
    },
    {
      "epoch": 1.56,
      "grad_norm": 11.839475631713867,
      "learning_rate": 5.5272708043255605e-06,
      "logits/chosen": 1.771327257156372,
      "logits/rejected": 1.8334521055221558,
      "logps/chosen": -40.4869499206543,
      "logps/rejected": -64.32723999023438,
      "loss": 0.5586,
      "nll_loss": 0.24806198477745056,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": -4.048694610595703,
      "rewards/margins": 2.3840293884277344,
      "rewards/rejected": -6.4327239990234375,
      "step": 1950
    },
    {
      "epoch": 1.6,
      "grad_norm": 12.160818099975586,
      "learning_rate": 5.295370355079615e-06,
      "logits/chosen": 1.7191228866577148,
      "logits/rejected": 1.7949659824371338,
      "logps/chosen": -40.17674255371094,
      "logps/rejected": -67.11283874511719,
      "loss": 0.5123,
      "nll_loss": 0.24404479563236237,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": -4.017674446105957,
      "rewards/margins": 2.6936092376708984,
      "rewards/rejected": -6.711284160614014,
      "step": 2000
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 12.399029731750488,
      "learning_rate": 5.062830199416764e-06,
      "logits/chosen": 1.7645864486694336,
      "logits/rejected": 1.8290934562683105,
      "logps/chosen": -41.4688720703125,
      "logps/rejected": -67.27618408203125,
      "loss": 0.5257,
      "nll_loss": 0.24901245534420013,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -4.146886825561523,
      "rewards/margins": 2.580731153488159,
      "rewards/rejected": -6.727618217468262,
      "step": 2050
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 13.637982368469238,
      "learning_rate": 4.8301539675328205e-06,
      "logits/chosen": 1.6826503276824951,
      "logits/rejected": 1.7500274181365967,
      "logps/chosen": -40.96113586425781,
      "logps/rejected": -67.78974914550781,
      "loss": 0.513,
      "nll_loss": 0.2408064305782318,
      "rewards/accuracies": 0.8987500071525574,
      "rewards/chosen": -4.096114158630371,
      "rewards/margins": 2.6828606128692627,
      "rewards/rejected": -6.778974533081055,
      "step": 2100
    },
    {
      "epoch": 1.72,
      "grad_norm": 5.442174911499023,
      "learning_rate": 4.597845584334387e-06,
      "logits/chosen": 1.6728637218475342,
      "logits/rejected": 1.7252342700958252,
      "logps/chosen": -42.251564025878906,
      "logps/rejected": -69.24467468261719,
      "loss": 0.5304,
      "nll_loss": 0.25177592039108276,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -4.225156307220459,
      "rewards/margins": 2.699312210083008,
      "rewards/rejected": -6.92446756362915,
      "step": 2150
    },
    {
      "epoch": 1.76,
      "grad_norm": 10.488734245300293,
      "learning_rate": 4.366408178049728e-06,
      "logits/chosen": 1.6670646667480469,
      "logits/rejected": 1.7333457469940186,
      "logps/chosen": -39.98529052734375,
      "logps/rejected": -65.87380981445312,
      "loss": 0.5198,
      "nll_loss": 0.2439391314983368,
      "rewards/accuracies": 0.8762500286102295,
      "rewards/chosen": -3.9985289573669434,
      "rewards/margins": 2.588852643966675,
      "rewards/rejected": -6.587381362915039,
      "step": 2200
    },
    {
      "epoch": 1.8,
      "grad_norm": 11.035846710205078,
      "learning_rate": 4.136342990565055e-06,
      "logits/chosen": 1.6644293069839478,
      "logits/rejected": 1.7320383787155151,
      "logps/chosen": -40.47782897949219,
      "logps/rejected": -69.36231994628906,
      "loss": 0.5059,
      "nll_loss": 0.2478472888469696,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -4.047782897949219,
      "rewards/margins": 2.888448476791382,
      "rewards/rejected": -6.93623161315918,
      "step": 2250
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 14.5468168258667,
      "learning_rate": 3.908148291846225e-06,
      "logits/chosen": 1.6440070867538452,
      "logits/rejected": 1.6980115175247192,
      "logps/chosen": -37.658931732177734,
      "logps/rejected": -64.18938446044922,
      "loss": 0.4975,
      "nll_loss": 0.23423399031162262,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -3.7658934593200684,
      "rewards/margins": 2.6530449390411377,
      "rewards/rejected": -6.418938636779785,
      "step": 2300
    },
    {
      "epoch": 1.88,
      "grad_norm": 15.06531810760498,
      "learning_rate": 3.6823183007969375e-06,
      "logits/chosen": 1.633602261543274,
      "logits/rejected": 1.69327974319458,
      "logps/chosen": -38.722007751464844,
      "logps/rejected": -65.98956298828125,
      "loss": 0.512,
      "nll_loss": 0.23453904688358307,
      "rewards/accuracies": 0.8837500214576721,
      "rewards/chosen": -3.8722009658813477,
      "rewards/margins": 2.726755380630493,
      "rewards/rejected": -6.598956108093262,
      "step": 2350
    },
    {
      "epoch": 1.92,
      "grad_norm": 9.41374397277832,
      "learning_rate": 3.4593421148906523e-06,
      "logits/chosen": 1.6280794143676758,
      "logits/rejected": 1.694667935371399,
      "logps/chosen": -41.9329833984375,
      "logps/rejected": -68.24996948242188,
      "loss": 0.5314,
      "nll_loss": 0.24245284497737885,
      "rewards/accuracies": 0.8887500166893005,
      "rewards/chosen": -4.19329833984375,
      "rewards/margins": 2.6316983699798584,
      "rewards/rejected": -6.8249969482421875,
      "step": 2400
    },
    {
      "epoch": 1.96,
      "grad_norm": 10.686447143554688,
      "learning_rate": 3.239702650894364e-06,
      "logits/chosen": 1.648103952407837,
      "logits/rejected": 1.7120572328567505,
      "logps/chosen": -39.89656066894531,
      "logps/rejected": -67.84294128417969,
      "loss": 0.5132,
      "nll_loss": 0.23985882103443146,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -3.989656448364258,
      "rewards/margins": 2.794638156890869,
      "rewards/rejected": -6.784294605255127,
      "step": 2450
    },
    {
      "epoch": 2.0,
      "grad_norm": 6.338383197784424,
      "learning_rate": 3.023875598978419e-06,
      "logits/chosen": 1.6715092658996582,
      "logits/rejected": 1.7344988584518433,
      "logps/chosen": -39.51519012451172,
      "logps/rejected": -68.46276092529297,
      "loss": 0.4834,
      "nll_loss": 0.24019798636436462,
      "rewards/accuracies": 0.9112499952316284,
      "rewards/chosen": -3.951519250869751,
      "rewards/margins": 2.8947577476501465,
      "rewards/rejected": -6.846276760101318,
      "step": 2500
    },
    {
      "epoch": 2.04,
      "grad_norm": 5.144768238067627,
      "learning_rate": 2.812328392477536e-06,
      "logits/chosen": 1.5945950746536255,
      "logits/rejected": 1.6544932126998901,
      "logps/chosen": -37.963443756103516,
      "logps/rejected": -69.92556762695312,
      "loss": 0.3913,
      "nll_loss": 0.22887782752513885,
      "rewards/accuracies": 0.9712499976158142,
      "rewards/chosen": -3.796344041824341,
      "rewards/margins": 3.196213722229004,
      "rewards/rejected": -6.992558002471924,
      "step": 2550
    },
    {
      "epoch": 2.08,
      "grad_norm": 5.904083251953125,
      "learning_rate": 2.6055191955342886e-06,
      "logits/chosen": 1.5594576597213745,
      "logits/rejected": 1.6052813529968262,
      "logps/chosen": -39.600093841552734,
      "logps/rejected": -73.07125854492188,
      "loss": 0.398,
      "nll_loss": 0.23794788122177124,
      "rewards/accuracies": 0.9662500023841858,
      "rewards/chosen": -3.960009813308716,
      "rewards/margins": 3.3471157550811768,
      "rewards/rejected": -7.307126045227051,
      "step": 2600
    },
    {
      "epoch": 2.12,
      "grad_norm": 5.723136901855469,
      "learning_rate": 2.403895910817593e-06,
      "logits/chosen": 1.532933235168457,
      "logits/rejected": 1.5803064107894897,
      "logps/chosen": -38.91565704345703,
      "logps/rejected": -70.781982421875,
      "loss": 0.4101,
      "nll_loss": 0.23784016072750092,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -3.8915653228759766,
      "rewards/margins": 3.186633348464966,
      "rewards/rejected": -7.07819938659668,
      "step": 2650
    },
    {
      "epoch": 2.16,
      "grad_norm": 7.6983561515808105,
      "learning_rate": 2.2078952094652705e-06,
      "logits/chosen": 1.4740697145462036,
      "logits/rejected": 1.5456255674362183,
      "logps/chosen": -39.75840377807617,
      "logps/rejected": -75.84871673583984,
      "loss": 0.3908,
      "nll_loss": 0.23151804506778717,
      "rewards/accuracies": 0.9612500071525574,
      "rewards/chosen": -3.9758400917053223,
      "rewards/margins": 3.6090316772460938,
      "rewards/rejected": -7.584871292114258,
      "step": 2700
    },
    {
      "epoch": 2.2,
      "grad_norm": 7.529278755187988,
      "learning_rate": 2.017941585351591e-06,
      "logits/chosen": 1.495976209640503,
      "logits/rejected": 1.5389071702957153,
      "logps/chosen": -39.2768440246582,
      "logps/rejected": -71.7361831665039,
      "loss": 0.4138,
      "nll_loss": 0.24059143662452698,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -3.9276845455169678,
      "rewards/margins": 3.245933771133423,
      "rewards/rejected": -7.173618793487549,
      "step": 2750
    },
    {
      "epoch": 2.24,
      "grad_norm": 6.076731204986572,
      "learning_rate": 1.8344464357280722e-06,
      "logits/chosen": 1.4751191139221191,
      "logits/rejected": 1.5268608331680298,
      "logps/chosen": -38.4973258972168,
      "logps/rejected": -71.8733901977539,
      "loss": 0.3943,
      "nll_loss": 0.22699041664600372,
      "rewards/accuracies": 0.9700000286102295,
      "rewards/chosen": -3.8497328758239746,
      "rewards/margins": 3.3376071453094482,
      "rewards/rejected": -7.187338829040527,
      "step": 2800
    },
    {
      "epoch": 2.2800000000000002,
      "grad_norm": 5.921169757843018,
      "learning_rate": 1.6578071702286396e-06,
      "logits/chosen": 1.4693129062652588,
      "logits/rejected": 1.5106064081192017,
      "logps/chosen": -38.8640022277832,
      "logps/rejected": -72.8025131225586,
      "loss": 0.4103,
      "nll_loss": 0.2377665936946869,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.8864006996154785,
      "rewards/margins": 3.393850088119507,
      "rewards/rejected": -7.280250072479248,
      "step": 2850
    },
    {
      "epoch": 2.32,
      "grad_norm": 5.996314525604248,
      "learning_rate": 1.4884063501688539e-06,
      "logits/chosen": 1.4711755514144897,
      "logits/rejected": 1.537732720375061,
      "logps/chosen": -38.81364440917969,
      "logps/rejected": -73.80380249023438,
      "loss": 0.3908,
      "nll_loss": 0.22893303632736206,
      "rewards/accuracies": 0.9737499952316284,
      "rewards/chosen": -3.881364345550537,
      "rewards/margins": 3.499016523361206,
      "rewards/rejected": -7.380380153656006,
      "step": 2900
    },
    {
      "epoch": 2.36,
      "grad_norm": 10.235599517822266,
      "learning_rate": 1.3266108600032928e-06,
      "logits/chosen": 1.469245195388794,
      "logits/rejected": 1.5233672857284546,
      "logps/chosen": -39.08128356933594,
      "logps/rejected": -70.8739242553711,
      "loss": 0.4124,
      "nll_loss": 0.2312445044517517,
      "rewards/accuracies": 0.9512500166893005,
      "rewards/chosen": -3.9081289768218994,
      "rewards/margins": 3.1792635917663574,
      "rewards/rejected": -7.087392807006836,
      "step": 2950
    },
    {
      "epoch": 2.4,
      "grad_norm": 8.700451850891113,
      "learning_rate": 1.1727711127355118e-06,
      "logits/chosen": 1.445876955986023,
      "logits/rejected": 1.494391679763794,
      "logps/chosen": -38.116580963134766,
      "logps/rejected": -70.0914535522461,
      "loss": 0.3982,
      "nll_loss": 0.22633756697177887,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -3.8116586208343506,
      "rewards/margins": 3.1974875926971436,
      "rewards/rejected": -7.009146213531494,
      "step": 3000
    },
    {
      "epoch": 2.44,
      "grad_norm": 9.24150562286377,
      "learning_rate": 1.0272202910015083e-06,
      "logits/chosen": 1.4509037733078003,
      "logits/rejected": 1.5117911100387573,
      "logps/chosen": -39.70808410644531,
      "logps/rejected": -74.1922836303711,
      "loss": 0.4038,
      "nll_loss": 0.23606754839420319,
      "rewards/accuracies": 0.9662500023841858,
      "rewards/chosen": -3.970808744430542,
      "rewards/margins": 3.4484190940856934,
      "rewards/rejected": -7.419227123260498,
      "step": 3050
    },
    {
      "epoch": 2.48,
      "grad_norm": 10.763381958007812,
      "learning_rate": 8.902736254703347e-07,
      "logits/chosen": 1.4675711393356323,
      "logits/rejected": 1.5065741539001465,
      "logps/chosen": -36.786865234375,
      "logps/rejected": -67.48006439208984,
      "loss": 0.405,
      "nll_loss": 0.22343981266021729,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.6786866188049316,
      "rewards/margins": 3.0693204402923584,
      "rewards/rejected": -6.748007297515869,
      "step": 3100
    },
    {
      "epoch": 2.52,
      "grad_norm": 5.758388996124268,
      "learning_rate": 7.622277121246513e-07,
      "logits/chosen": 1.483445405960083,
      "logits/rejected": 1.5389055013656616,
      "logps/chosen": -38.62077331542969,
      "logps/rejected": -71.79267883300781,
      "loss": 0.4121,
      "nll_loss": 0.23168590664863586,
      "rewards/accuracies": 0.9587500095367432,
      "rewards/chosen": -3.8620777130126953,
      "rewards/margins": 3.3171918392181396,
      "rewards/rejected": -7.179268836975098,
      "step": 3150
    },
    {
      "epoch": 2.56,
      "grad_norm": 7.307446002960205,
      "learning_rate": 6.433598698998766e-07,
      "logits/chosen": 1.4688174724578857,
      "logits/rejected": 1.5306994915008545,
      "logps/chosen": -39.013912200927734,
      "logps/rejected": -71.06527709960938,
      "loss": 0.416,
      "nll_loss": 0.23500658571720123,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -3.9013912677764893,
      "rewards/margins": 3.2051360607147217,
      "rewards/rejected": -7.106527328491211,
      "step": 3200
    },
    {
      "epoch": 2.6,
      "grad_norm": 6.810940742492676,
      "learning_rate": 5.339275400731331e-07,
      "logits/chosen": 1.4731531143188477,
      "logits/rejected": 1.5175625085830688,
      "logps/chosen": -39.05414581298828,
      "logps/rejected": -69.38569641113281,
      "loss": 0.4207,
      "nll_loss": 0.2375660389661789,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.9054150581359863,
      "rewards/margins": 3.0331552028656006,
      "rewards/rejected": -6.93856954574585,
      "step": 3250
    },
    {
      "epoch": 2.64,
      "grad_norm": 5.935739517211914,
      "learning_rate": 4.3416772870275295e-07,
      "logits/chosen": 1.447837233543396,
      "logits/rejected": 1.5085737705230713,
      "logps/chosen": -38.072574615478516,
      "logps/rejected": -71.93220520019531,
      "loss": 0.3981,
      "nll_loss": 0.22540073096752167,
      "rewards/accuracies": 0.9537500143051147,
      "rewards/chosen": -3.807257652282715,
      "rewards/margins": 3.385963201522827,
      "rewards/rejected": -7.193220138549805,
      "step": 3300
    },
    {
      "epoch": 2.68,
      "grad_norm": 6.746799468994141,
      "learning_rate": 3.442964933259474e-07,
      "logits/chosen": 1.4588629007339478,
      "logits/rejected": 1.517850637435913,
      "logps/chosen": -38.37602615356445,
      "logps/rejected": -72.57811737060547,
      "loss": 0.3998,
      "nll_loss": 0.23044270277023315,
      "rewards/accuracies": 0.9587500095367432,
      "rewards/chosen": -3.8376026153564453,
      "rewards/margins": 3.4202094078063965,
      "rewards/rejected": -7.257812023162842,
      "step": 3350
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 7.958431243896484,
      "learning_rate": 2.6450847502627883e-07,
      "logits/chosen": 1.4416861534118652,
      "logits/rejected": 1.5023283958435059,
      "logps/chosen": -39.2740364074707,
      "logps/rejected": -73.73572540283203,
      "loss": 0.4036,
      "nll_loss": 0.22917690873146057,
      "rewards/accuracies": 0.9662500023841858,
      "rewards/chosen": -3.927403450012207,
      "rewards/margins": 3.446169376373291,
      "rewards/rejected": -7.373574256896973,
      "step": 3400
    },
    {
      "epoch": 2.76,
      "grad_norm": 8.157621383666992,
      "learning_rate": 1.9497647688442478e-07,
      "logits/chosen": 1.4702584743499756,
      "logits/rejected": 1.5254310369491577,
      "logps/chosen": -38.0818977355957,
      "logps/rejected": -70.6051254272461,
      "loss": 0.4055,
      "nll_loss": 0.23146916925907135,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -3.808190107345581,
      "rewards/margins": 3.252322196960449,
      "rewards/rejected": -7.060512065887451,
      "step": 3450
    },
    {
      "epoch": 2.8,
      "grad_norm": 6.671606540679932,
      "learning_rate": 1.358510897251808e-07,
      "logits/chosen": 1.4628223180770874,
      "logits/rejected": 1.525000810623169,
      "logps/chosen": -37.097007751464844,
      "logps/rejected": -72.08160400390625,
      "loss": 0.3906,
      "nll_loss": 0.22892680764198303,
      "rewards/accuracies": 0.9700000286102295,
      "rewards/chosen": -3.7097012996673584,
      "rewards/margins": 3.498459577560425,
      "rewards/rejected": -7.208160400390625,
      "step": 3500
    },
    {
      "epoch": 2.84,
      "grad_norm": 8.076554298400879,
      "learning_rate": 8.726036597126619e-08,
      "logits/chosen": 1.470629096031189,
      "logits/rejected": 1.535922646522522,
      "logps/chosen": -36.950157165527344,
      "logps/rejected": -68.89533996582031,
      "loss": 0.395,
      "nll_loss": 0.22603529691696167,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -3.6950161457061768,
      "rewards/margins": 3.1945183277130127,
      "rewards/rejected": -6.8895344734191895,
      "step": 3550
    },
    {
      "epoch": 2.88,
      "grad_norm": 6.550802707672119,
      "learning_rate": 4.93095423102935e-08,
      "logits/chosen": 1.4719560146331787,
      "logits/rejected": 1.5248959064483643,
      "logps/chosen": -40.710941314697266,
      "logps/rejected": -74.42230987548828,
      "loss": 0.4132,
      "nll_loss": 0.24109821021556854,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -4.071094512939453,
      "rewards/margins": 3.371136426925659,
      "rewards/rejected": -7.442230701446533,
      "step": 3600
    },
    {
      "epoch": 2.92,
      "grad_norm": 6.743546485900879,
      "learning_rate": 2.2080811775535006e-08,
      "logits/chosen": 1.4744410514831543,
      "logits/rejected": 1.5339540243148804,
      "logps/chosen": -39.543033599853516,
      "logps/rejected": -74.6586685180664,
      "loss": 0.3964,
      "nll_loss": 0.23712339997291565,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": -3.954303026199341,
      "rewards/margins": 3.511564254760742,
      "rewards/rejected": -7.46586799621582,
      "step": 3650
    },
    {
      "epoch": 2.96,
      "grad_norm": 11.845396041870117,
      "learning_rate": 5.633145734114665e-09,
      "logits/chosen": 1.4899775981903076,
      "logits/rejected": 1.533370018005371,
      "logps/chosen": -38.53838348388672,
      "logps/rejected": -71.89134979248047,
      "loss": 0.4058,
      "nll_loss": 0.23659667372703552,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": -3.853839159011841,
      "rewards/margins": 3.3352959156036377,
      "rewards/rejected": -7.18913459777832,
      "step": 3700
    },
    {
      "epoch": 3.0,
      "grad_norm": 9.659139633178711,
      "learning_rate": 2.1661681620654963e-12,
      "logits/chosen": 1.4696215391159058,
      "logits/rejected": 1.526814579963684,
      "logps/chosen": -38.22508239746094,
      "logps/rejected": -72.54460144042969,
      "loss": 0.3938,
      "nll_loss": 0.22828102111816406,
      "rewards/accuracies": 0.9637500047683716,
      "rewards/chosen": -3.8225083351135254,
      "rewards/margins": 3.4319519996643066,
      "rewards/rejected": -7.254461288452148,
      "step": 3750
    }
  ],
  "logging_steps": 50,
  "max_steps": 3750,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}