{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.99860529986053,
  "eval_steps": 500,
  "global_step": 358,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01394700139470014,
      "grad_norm": 8.443673133850098,
      "learning_rate": 3.1067090740782183e-06,
      "logits/chosen": 13.965856552124023,
      "logits/rejected": 15.354721069335938,
      "logps/chosen": -263.2093200683594,
      "logps/rejected": -286.745361328125,
      "loss": 0.6931,
      "rewards/accuracies": 0.3083333373069763,
      "rewards/chosen": 0.001035079127177596,
      "rewards/margins": 0.004346395842730999,
      "rewards/rejected": -0.003311316715553403,
      "step": 5
    },
    {
      "epoch": 0.02789400278940028,
      "grad_norm": 9.05903434753418,
      "learning_rate": 6.99009541667599e-06,
      "logits/chosen": 13.079957962036133,
      "logits/rejected": 15.2965726852417,
      "logps/chosen": -246.61495971679688,
      "logps/rejected": -292.19561767578125,
      "loss": 0.6512,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.004167428705841303,
      "rewards/margins": 0.10341107845306396,
      "rewards/rejected": -0.10757851600646973,
      "step": 10
    },
    {
      "epoch": 0.04184100418410042,
      "grad_norm": 7.4054036140441895,
      "learning_rate": 1.0873481759273766e-05,
      "logits/chosen": 13.588569641113281,
      "logits/rejected": 14.664776802062988,
      "logps/chosen": -250.9985809326172,
      "logps/rejected": -271.63397216796875,
      "loss": 0.6306,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -0.07446910440921783,
      "rewards/margins": 0.2632213234901428,
      "rewards/rejected": -0.33769041299819946,
      "step": 15
    },
    {
      "epoch": 0.05578800557880056,
      "grad_norm": 7.973468780517578,
      "learning_rate": 1.4756868101871536e-05,
      "logits/chosen": 10.776374816894531,
      "logits/rejected": 12.654977798461914,
      "logps/chosen": -219.0859375,
      "logps/rejected": -279.8912048339844,
      "loss": 0.5803,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -0.15387776494026184,
      "rewards/margins": 0.5580573678016663,
      "rewards/rejected": -0.7119351625442505,
      "step": 20
    },
    {
      "epoch": 0.0697350069735007,
      "grad_norm": 8.493500709533691,
      "learning_rate": 1.864025444446931e-05,
      "logits/chosen": 11.028432846069336,
      "logits/rejected": 12.664867401123047,
      "logps/chosen": -269.8415832519531,
      "logps/rejected": -324.97613525390625,
      "loss": 0.6957,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -0.31670600175857544,
      "rewards/margins": 0.7376400232315063,
      "rewards/rejected": -1.0543458461761475,
      "step": 25
    },
    {
      "epoch": 0.08368200836820083,
      "grad_norm": 6.57462739944458,
      "learning_rate": 2.2523640787067085e-05,
      "logits/chosen": 12.154653549194336,
      "logits/rejected": 13.542457580566406,
      "logps/chosen": -250.53341674804688,
      "logps/rejected": -290.4168395996094,
      "loss": 0.6559,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -0.11686629056930542,
      "rewards/margins": 0.5811147093772888,
      "rewards/rejected": -0.6979809999465942,
      "step": 30
    },
    {
      "epoch": 0.09762900976290098,
      "grad_norm": 7.757661819458008,
      "learning_rate": 2.6407027129664858e-05,
      "logits/chosen": 11.735052108764648,
      "logits/rejected": 12.871235847473145,
      "logps/chosen": -196.1747589111328,
      "logps/rejected": -223.5957794189453,
      "loss": 0.6182,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.00016996636986732483,
      "rewards/margins": 0.3299906551837921,
      "rewards/rejected": -0.33016061782836914,
      "step": 35
    },
    {
      "epoch": 0.11157601115760112,
      "grad_norm": 9.74497127532959,
      "learning_rate": 2.7182958819900885e-05,
      "logits/chosen": 10.667373657226562,
      "logits/rejected": 12.847865104675293,
      "logps/chosen": -208.2180633544922,
      "logps/rejected": -302.39971923828125,
      "loss": 0.5985,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03828881308436394,
      "rewards/margins": 0.43971744179725647,
      "rewards/rejected": -0.4780062735080719,
      "step": 40
    },
    {
      "epoch": 0.12552301255230125,
      "grad_norm": 9.502607345581055,
      "learning_rate": 2.7179930095042222e-05,
      "logits/chosen": 11.666415214538574,
      "logits/rejected": 13.399192810058594,
      "logps/chosen": -252.95516967773438,
      "logps/rejected": -297.4845275878906,
      "loss": 0.6683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2012624740600586,
      "rewards/margins": 0.4417993426322937,
      "rewards/rejected": -0.6430618166923523,
      "step": 45
    },
    {
      "epoch": 0.1394700139470014,
      "grad_norm": 6.734437942504883,
      "learning_rate": 2.717457231667877e-05,
      "logits/chosen": 11.400947570800781,
      "logits/rejected": 12.905207633972168,
      "logps/chosen": -199.80596923828125,
      "logps/rejected": -254.4221954345703,
      "loss": 0.5718,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -0.022540345788002014,
      "rewards/margins": 0.5749102830886841,
      "rewards/rejected": -0.5974506139755249,
      "step": 50
    },
    {
      "epoch": 0.15341701534170155,
      "grad_norm": 6.349542617797852,
      "learning_rate": 2.7166886709384802e-05,
      "logits/chosen": 9.220715522766113,
      "logits/rejected": 10.65298080444336,
      "logps/chosen": -187.50601196289062,
      "logps/rejected": -232.32241821289062,
      "loss": 0.6379,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -0.3121258318424225,
      "rewards/margins": 0.4689728617668152,
      "rewards/rejected": -0.7810987234115601,
      "step": 55
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 6.804388046264648,
      "learning_rate": 2.715687502978336e-05,
      "logits/chosen": 10.728836059570312,
      "logits/rejected": 11.548182487487793,
      "logps/chosen": -220.2592010498047,
      "logps/rejected": -248.6510009765625,
      "loss": 0.5624,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -0.24719354510307312,
      "rewards/margins": 0.5707910656929016,
      "rewards/rejected": -0.8179847002029419,
      "step": 60
    },
    {
      "epoch": 0.18131101813110181,
      "grad_norm": 8.308326721191406,
      "learning_rate": 2.714453956614478e-05,
      "logits/chosen": 9.632620811462402,
      "logits/rejected": 10.766716003417969,
      "logps/chosen": -234.59494018554688,
      "logps/rejected": -246.2835235595703,
      "loss": 0.6495,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.4380224347114563,
      "rewards/margins": 0.505369246006012,
      "rewards/rejected": -0.9433916211128235,
      "step": 65
    },
    {
      "epoch": 0.19525801952580196,
      "grad_norm": 5.469903945922852,
      "learning_rate": 2.7129883137863668e-05,
      "logits/chosen": 10.187846183776855,
      "logits/rejected": 11.750052452087402,
      "logps/chosen": -201.72952270507812,
      "logps/rejected": -254.0623779296875,
      "loss": 0.6407,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.28986990451812744,
      "rewards/margins": 0.4815855026245117,
      "rewards/rejected": -0.7714553475379944,
      "step": 70
    },
    {
      "epoch": 0.20920502092050208,
      "grad_norm": 6.822120189666748,
      "learning_rate": 2.7112909094814497e-05,
      "logits/chosen": 11.570058822631836,
      "logits/rejected": 12.092260360717773,
      "logps/chosen": -201.8399658203125,
      "logps/rejected": -255.96774291992188,
      "loss": 0.5926,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.17586487531661987,
      "rewards/margins": 0.4662502706050873,
      "rewards/rejected": -0.6421152353286743,
      "step": 75
    },
    {
      "epoch": 0.22315202231520223,
      "grad_norm": 6.789912223815918,
      "learning_rate": 2.7093621316585976e-05,
      "logits/chosen": 9.428289413452148,
      "logits/rejected": 12.114514350891113,
      "logps/chosen": -199.76634216308594,
      "logps/rejected": -259.3163146972656,
      "loss": 0.6011,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -0.12371706962585449,
      "rewards/margins": 0.6005428433418274,
      "rewards/rejected": -0.7242598533630371,
      "step": 80
    },
    {
      "epoch": 0.23709902370990238,
      "grad_norm": 8.106010437011719,
      "learning_rate": 2.7072024211594312e-05,
      "logits/chosen": 10.688722610473633,
      "logits/rejected": 11.625075340270996,
      "logps/chosen": -221.6556854248047,
      "logps/rejected": -274.34808349609375,
      "loss": 0.6676,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.237966850399971,
      "rewards/margins": 0.4688500761985779,
      "rewards/rejected": -0.7068168520927429,
      "step": 85
    },
    {
      "epoch": 0.2510460251046025,
      "grad_norm": 7.263269901275635,
      "learning_rate": 2.7048122716075636e-05,
      "logits/chosen": 12.357452392578125,
      "logits/rejected": 14.314582824707031,
      "logps/chosen": -224.59451293945312,
      "logps/rejected": -291.54840087890625,
      "loss": 0.6033,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": -0.07308916002511978,
      "rewards/margins": 0.504932165145874,
      "rewards/rejected": -0.5780213475227356,
      "step": 90
    },
    {
      "epoch": 0.2649930264993027,
      "grad_norm": 8.925468444824219,
      "learning_rate": 2.7021922292957776e-05,
      "logits/chosen": 12.812413215637207,
      "logits/rejected": 13.796060562133789,
      "logps/chosen": -230.6768798828125,
      "logps/rejected": -276.98828125,
      "loss": 0.6054,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -0.03974637761712074,
      "rewards/margins": 0.5660965442657471,
      "rewards/rejected": -0.6058429479598999,
      "step": 95
    },
    {
      "epoch": 0.2789400278940028,
      "grad_norm": 5.422610282897949,
      "learning_rate": 2.6993428930611634e-05,
      "logits/chosen": 12.078475952148438,
      "logits/rejected": 14.003347396850586,
      "logps/chosen": -229.6229705810547,
      "logps/rejected": -276.1091613769531,
      "loss": 0.5103,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": 0.10478214919567108,
      "rewards/margins": 0.8226801156997681,
      "rewards/rejected": -0.7178980112075806,
      "step": 100
    },
    {
      "epoch": 0.2928870292887029,
      "grad_norm": 7.437990188598633,
      "learning_rate": 2.69626491414825e-05,
      "logits/chosen": 11.215384483337402,
      "logits/rejected": 14.319795608520508,
      "logps/chosen": -253.2958221435547,
      "logps/rejected": -353.27728271484375,
      "loss": 0.4647,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.20068514347076416,
      "rewards/margins": 1.1105215549468994,
      "rewards/rejected": -1.3112064599990845,
      "step": 105
    },
    {
      "epoch": 0.3068340306834031,
      "grad_norm": 4.438156604766846,
      "learning_rate": 2.6929589960601567e-05,
      "logits/chosen": 10.04852294921875,
      "logits/rejected": 11.834890365600586,
      "logps/chosen": -177.2984619140625,
      "logps/rejected": -235.3513946533203,
      "loss": 0.5941,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14255599677562714,
      "rewards/margins": 1.0380289554595947,
      "rewards/rejected": -1.1805849075317383,
      "step": 110
    },
    {
      "epoch": 0.3207810320781032,
      "grad_norm": 5.81158971786499,
      "learning_rate": 2.689425894397799e-05,
      "logits/chosen": 11.626302719116211,
      "logits/rejected": 13.641179084777832,
      "logps/chosen": -258.5274353027344,
      "logps/rejected": -290.1232604980469,
      "loss": 0.7585,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -0.3435746729373932,
      "rewards/margins": 0.7051381468772888,
      "rewards/rejected": -1.0487128496170044,
      "step": 115
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 6.80866003036499,
      "learning_rate": 2.685666416687189e-05,
      "logits/chosen": 12.343037605285645,
      "logits/rejected": 14.431178092956543,
      "logps/chosen": -240.32119750976562,
      "logps/rejected": -306.0838623046875,
      "loss": 0.595,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": 0.007438424974679947,
      "rewards/margins": 0.7371615171432495,
      "rewards/rejected": -0.7297230362892151,
      "step": 120
    },
    {
      "epoch": 0.3486750348675035,
      "grad_norm": 5.887704372406006,
      "learning_rate": 2.6816814221948682e-05,
      "logits/chosen": 12.360528945922852,
      "logits/rejected": 13.354936599731445,
      "logps/chosen": -229.65194702148438,
      "logps/rejected": -286.4953918457031,
      "loss": 0.5958,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04780165106058121,
      "rewards/margins": 0.5758650898933411,
      "rewards/rejected": -0.5280634164810181,
      "step": 125
    },
    {
      "epoch": 0.36262203626220363,
      "grad_norm": 6.528600215911865,
      "learning_rate": 2.6774718217315124e-05,
      "logits/chosen": 12.22896671295166,
      "logits/rejected": 14.30839729309082,
      "logps/chosen": -258.0196838378906,
      "logps/rejected": -312.25018310546875,
      "loss": 0.5731,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -0.1742248833179474,
      "rewards/margins": 0.7956671118736267,
      "rewards/rejected": -0.9698920249938965,
      "step": 130
    },
    {
      "epoch": 0.37656903765690375,
      "grad_norm": 4.8263702392578125,
      "learning_rate": 2.6730385774437575e-05,
      "logits/chosen": 11.001718521118164,
      "logits/rejected": 12.369864463806152,
      "logps/chosen": -186.54306030273438,
      "logps/rejected": -229.1658935546875,
      "loss": 0.5747,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.10876262187957764,
      "rewards/margins": 0.783820390701294,
      "rewards/rejected": -0.8925830721855164,
      "step": 135
    },
    {
      "epoch": 0.3905160390516039,
      "grad_norm": 6.88525915145874,
      "learning_rate": 2.668382702594289e-05,
      "logits/chosen": 12.428079605102539,
      "logits/rejected": 13.376548767089844,
      "logps/chosen": -244.35336303710938,
      "logps/rejected": -314.2310485839844,
      "loss": 0.5727,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19110675156116486,
      "rewards/margins": 0.8208184242248535,
      "rewards/rejected": -1.011925220489502,
      "step": 140
    },
    {
      "epoch": 0.40446304044630405,
      "grad_norm": 5.504042148590088,
      "learning_rate": 2.663505261330254e-05,
      "logits/chosen": 11.734379768371582,
      "logits/rejected": 13.473363876342773,
      "logps/chosen": -204.04251098632812,
      "logps/rejected": -258.86737060546875,
      "loss": 0.6128,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.17432288825511932,
      "rewards/margins": 0.4802930951118469,
      "rewards/rejected": -0.6546159982681274,
      "step": 145
    },
    {
      "epoch": 0.41841004184100417,
      "grad_norm": 6.70412540435791,
      "learning_rate": 2.6584073684400373e-05,
      "logits/chosen": 13.045954704284668,
      "logits/rejected": 13.6720552444458,
      "logps/chosen": -220.8832550048828,
      "logps/rejected": -269.233642578125,
      "loss": 0.6032,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04318929463624954,
      "rewards/margins": 0.618157684803009,
      "rewards/rejected": -0.6613470315933228,
      "step": 150
    },
    {
      "epoch": 0.43235704323570434,
      "grad_norm": 12.566320419311523,
      "learning_rate": 2.653090189098466e-05,
      "logits/chosen": 11.808868408203125,
      "logits/rejected": 13.49694538116455,
      "logps/chosen": -248.18212890625,
      "logps/rejected": -286.3559265136719,
      "loss": 0.6096,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15974543988704681,
      "rewards/margins": 0.6279758214950562,
      "rewards/rejected": -0.7877213954925537,
      "step": 155
    },
    {
      "epoch": 0.44630404463040446,
      "grad_norm": 6.7717509269714355,
      "learning_rate": 2.647554938600497e-05,
      "logits/chosen": 11.906153678894043,
      "logits/rejected": 13.252202033996582,
      "logps/chosen": -230.92593383789062,
      "logps/rejected": -248.78488159179688,
      "loss": 0.6474,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.20924147963523865,
      "rewards/margins": 0.49998435378074646,
      "rewards/rejected": -0.7092257738113403,
      "step": 160
    },
    {
      "epoch": 0.4602510460251046,
      "grad_norm": 7.113363742828369,
      "learning_rate": 2.6418028820834483e-05,
      "logits/chosen": 11.409849166870117,
      "logits/rejected": 13.500089645385742,
      "logps/chosen": -242.9384765625,
      "logps/rejected": -325.8098449707031,
      "loss": 0.5703,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -0.4405299127101898,
      "rewards/margins": 0.8166629076004028,
      "rewards/rejected": -1.257192850112915,
      "step": 165
    },
    {
      "epoch": 0.47419804741980476,
      "grad_norm": 4.8473429679870605,
      "learning_rate": 2.6358353342378405e-05,
      "logits/chosen": 8.604534149169922,
      "logits/rejected": 11.609006881713867,
      "logps/chosen": -185.0061492919922,
      "logps/rejected": -256.83465576171875,
      "loss": 0.4518,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -0.5300418734550476,
      "rewards/margins": 1.1527934074401855,
      "rewards/rejected": -1.682835340499878,
      "step": 170
    },
    {
      "epoch": 0.4881450488145049,
      "grad_norm": 6.59335994720459,
      "learning_rate": 2.6296536590069104e-05,
      "logits/chosen": 9.987462997436523,
      "logits/rejected": 11.572959899902344,
      "logps/chosen": -253.36050415039062,
      "logps/rejected": -306.0450439453125,
      "loss": 0.7439,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.0800180435180664,
      "rewards/margins": 0.8873661160469055,
      "rewards/rejected": -1.9673839807510376,
      "step": 175
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 7.2572808265686035,
      "learning_rate": 2.6232592692748676e-05,
      "logits/chosen": 9.321784019470215,
      "logits/rejected": 10.399984359741211,
      "logps/chosen": -234.3979949951172,
      "logps/rejected": -294.28961181640625,
      "loss": 0.489,
      "rewards/accuracies": 0.7916667461395264,
      "rewards/chosen": -0.8451215624809265,
      "rewards/margins": 1.1555068492889404,
      "rewards/rejected": -2.0006284713745117,
      "step": 180
    },
    {
      "epoch": 0.5160390516039052,
      "grad_norm": 6.049355506896973,
      "learning_rate": 2.6166536265439664e-05,
      "logits/chosen": 11.809592247009277,
      "logits/rejected": 13.855669021606445,
      "logps/chosen": -235.9927215576172,
      "logps/rejected": -315.3490295410156,
      "loss": 0.5868,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -0.6016494631767273,
      "rewards/margins": 0.7112706899642944,
      "rewards/rejected": -1.3129202127456665,
      "step": 185
    },
    {
      "epoch": 0.5299860529986054,
      "grad_norm": 11.038729667663574,
      "learning_rate": 2.609838240600464e-05,
      "logits/chosen": 11.31627082824707,
      "logits/rejected": 12.770919799804688,
      "logps/chosen": -221.79501342773438,
      "logps/rejected": -262.8851013183594,
      "loss": 0.5937,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -0.5558231472969055,
      "rewards/margins": 0.6753066778182983,
      "rewards/rejected": -1.2311298847198486,
      "step": 190
    },
    {
      "epoch": 0.5439330543933054,
      "grad_norm": 4.267746925354004,
      "learning_rate": 2.602814669169543e-05,
      "logits/chosen": 11.312234878540039,
      "logits/rejected": 13.296246528625488,
      "logps/chosen": -221.83663940429688,
      "logps/rejected": -302.32623291015625,
      "loss": 0.5809,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -0.5700381994247437,
      "rewards/margins": 0.8077206611633301,
      "rewards/rejected": -1.3777590990066528,
      "step": 195
    },
    {
      "epoch": 0.5578800557880056,
      "grad_norm": 5.395613193511963,
      "learning_rate": 2.5955845175592813e-05,
      "logits/chosen": 11.410247802734375,
      "logits/rejected": 13.515039443969727,
      "logps/chosen": -232.266845703125,
      "logps/rejected": -293.22467041015625,
      "loss": 0.5553,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5373277068138123,
      "rewards/margins": 0.8851755857467651,
      "rewards/rejected": -1.4225032329559326,
      "step": 200
    },
    {
      "epoch": 0.5718270571827058,
      "grad_norm": 5.884355545043945,
      "learning_rate": 2.5881494382937354e-05,
      "logits/chosen": 11.658515930175781,
      "logits/rejected": 13.375651359558105,
      "logps/chosen": -212.52932739257812,
      "logps/rejected": -286.71783447265625,
      "loss": 0.5126,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6171377897262573,
      "rewards/margins": 1.1004387140274048,
      "rewards/rejected": -1.7175763845443726,
      "step": 205
    },
    {
      "epoch": 0.5857740585774058,
      "grad_norm": 8.411831855773926,
      "learning_rate": 2.5805111307352483e-05,
      "logits/chosen": 11.734652519226074,
      "logits/rejected": 12.831028938293457,
      "logps/chosen": -247.45736694335938,
      "logps/rejected": -297.03485107421875,
      "loss": 0.5907,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.6340610384941101,
      "rewards/margins": 1.0966707468032837,
      "rewards/rejected": -1.7307319641113281,
      "step": 210
    },
    {
      "epoch": 0.599721059972106,
      "grad_norm": 6.429418087005615,
      "learning_rate": 2.5726713406960365e-05,
      "logits/chosen": 9.269506454467773,
      "logits/rejected": 12.285856246948242,
      "logps/chosen": -187.17591857910156,
      "logps/rejected": -269.7003479003906,
      "loss": 0.5349,
      "rewards/accuracies": 0.7666667699813843,
      "rewards/chosen": -0.5631422400474548,
      "rewards/margins": 1.0805346965789795,
      "rewards/rejected": -1.6436771154403687,
      "step": 215
    },
    {
      "epoch": 0.6136680613668062,
      "grad_norm": 8.299524307250977,
      "learning_rate": 2.5646318600391693e-05,
      "logits/chosen": 10.429685592651367,
      "logits/rejected": 11.87198543548584,
      "logps/chosen": -234.75802612304688,
      "logps/rejected": -284.33642578125,
      "loss": 0.5775,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -0.5350970029830933,
      "rewards/margins": 1.0643174648284912,
      "rewards/rejected": -1.599414348602295,
      "step": 220
    },
    {
      "epoch": 0.6276150627615062,
      "grad_norm": 3.9937915802001953,
      "learning_rate": 2.556394526269021e-05,
      "logits/chosen": 11.182074546813965,
      "logits/rejected": 12.9561767578125,
      "logps/chosen": -209.6283721923828,
      "logps/rejected": -261.5816345214844,
      "loss": 0.7124,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.651893675327301,
      "rewards/margins": 0.631694495677948,
      "rewards/rejected": -1.283588171005249,
      "step": 225
    },
    {
      "epoch": 0.6415620641562064,
      "grad_norm": 8.43625259399414,
      "learning_rate": 2.5479612221112888e-05,
      "logits/chosen": 11.575922966003418,
      "logits/rejected": 12.958845138549805,
      "logps/chosen": -241.3083953857422,
      "logps/rejected": -291.8955993652344,
      "loss": 0.6766,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.6825530529022217,
      "rewards/margins": 0.6574732661247253,
      "rewards/rejected": -1.3400263786315918,
      "step": 230
    },
    {
      "epoch": 0.6555090655509066,
      "grad_norm": 4.0093817710876465,
      "learning_rate": 2.5393338750826796e-05,
      "logits/chosen": 12.776809692382812,
      "logits/rejected": 14.406654357910156,
      "logps/chosen": -229.2129669189453,
      "logps/rejected": -300.2899475097656,
      "loss": 0.5466,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -0.5617952346801758,
      "rewards/margins": 0.752461850643158,
      "rewards/rejected": -1.314257264137268,
      "step": 235
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 6.296603679656982,
      "learning_rate": 2.5305144570503554e-05,
      "logits/chosen": 12.071104049682617,
      "logits/rejected": 14.155171394348145,
      "logps/chosen": -226.9972381591797,
      "logps/rejected": -299.86993408203125,
      "loss": 0.5546,
      "rewards/accuracies": 0.7166667580604553,
      "rewards/chosen": -0.5309011936187744,
      "rewards/margins": 0.8765324354171753,
      "rewards/rejected": -1.4074336290359497,
      "step": 240
    },
    {
      "epoch": 0.6834030683403068,
      "grad_norm": 6.713784694671631,
      "learning_rate": 2.5215049837812413e-05,
      "logits/chosen": 11.046672821044922,
      "logits/rejected": 13.329177856445312,
      "logps/chosen": -222.3082275390625,
      "logps/rejected": -302.4698181152344,
      "loss": 0.4944,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -0.6500851511955261,
      "rewards/margins": 1.216412901878357,
      "rewards/rejected": -1.8664979934692383,
      "step": 245
    },
    {
      "epoch": 0.697350069735007,
      "grad_norm": 3.64483904838562,
      "learning_rate": 2.5123075144813044e-05,
      "logits/chosen": 9.95425796508789,
      "logits/rejected": 12.83178997039795,
      "logps/chosen": -245.83987426757812,
      "logps/rejected": -347.21807861328125,
      "loss": 0.4497,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -0.7453610897064209,
      "rewards/margins": 1.5202277898788452,
      "rewards/rejected": -2.2655892372131348,
      "step": 250
    },
    {
      "epoch": 0.7112970711297071,
      "grad_norm": 6.323943138122559,
      "learning_rate": 2.5029241513248992e-05,
      "logits/chosen": 10.718851089477539,
      "logits/rejected": 11.787945747375488,
      "logps/chosen": -243.6724853515625,
      "logps/rejected": -275.3341064453125,
      "loss": 0.7227,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8060539364814758,
      "rewards/margins": 0.8095922470092773,
      "rewards/rejected": -1.6156460046768188,
      "step": 255
    },
    {
      "epoch": 0.7252440725244073,
      "grad_norm": 7.24354887008667,
      "learning_rate": 2.4933570389742975e-05,
      "logits/chosen": 10.517350196838379,
      "logits/rejected": 12.290593147277832,
      "logps/chosen": -217.3165740966797,
      "logps/rejected": -277.9285888671875,
      "loss": 0.5759,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.4403415620326996,
      "rewards/margins": 0.91583251953125,
      "rewards/rejected": -1.3561739921569824,
      "step": 260
    },
    {
      "epoch": 0.7391910739191074,
      "grad_norm": 5.851767063140869,
      "learning_rate": 2.4836083640895016e-05,
      "logits/chosen": 11.360766410827637,
      "logits/rejected": 13.0468111038208,
      "logps/chosen": -253.7987518310547,
      "logps/rejected": -321.97967529296875,
      "loss": 0.552,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -0.37671542167663574,
      "rewards/margins": 1.0203909873962402,
      "rewards/rejected": -1.3971065282821655,
      "step": 265
    },
    {
      "epoch": 0.7531380753138075,
      "grad_norm": 5.513597011566162,
      "learning_rate": 2.473680354828461e-05,
      "logits/chosen": 10.999284744262695,
      "logits/rejected": 12.660871505737305,
      "logps/chosen": -218.94735717773438,
      "logps/rejected": -259.43133544921875,
      "loss": 0.5662,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -0.4751812517642975,
      "rewards/margins": 0.7894707322120667,
      "rewards/rejected": -1.264651894569397,
      "step": 270
    },
    {
      "epoch": 0.7670850767085077,
      "grad_norm": 5.750617504119873,
      "learning_rate": 2.4635752803378063e-05,
      "logits/chosen": 11.961132049560547,
      "logits/rejected": 14.314129829406738,
      "logps/chosen": -221.6836700439453,
      "logps/rejected": -307.97467041015625,
      "loss": 0.5598,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -0.42271748185157776,
      "rewards/margins": 0.7973464727401733,
      "rewards/rejected": -1.2200638055801392,
      "step": 275
    },
    {
      "epoch": 0.7810320781032078,
      "grad_norm": 6.243062496185303,
      "learning_rate": 2.453295450234211e-05,
      "logits/chosen": 11.414163589477539,
      "logits/rejected": 13.276272773742676,
      "logps/chosen": -223.52529907226562,
      "logps/rejected": -324.9430236816406,
      "loss": 0.516,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -0.5223814249038696,
      "rewards/margins": 1.0250855684280396,
      "rewards/rejected": -1.5474669933319092,
      "step": 280
    },
    {
      "epoch": 0.7949790794979079,
      "grad_norm": 7.279259204864502,
      "learning_rate": 2.442843214076507e-05,
      "logits/chosen": 11.585288047790527,
      "logits/rejected": 12.664071083068848,
      "logps/chosen": -244.99154663085938,
      "logps/rejected": -254.84158325195312,
      "loss": 0.5448,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -0.7385099530220032,
      "rewards/margins": 0.9241247177124023,
      "rewards/rejected": -1.6626346111297607,
      "step": 285
    },
    {
      "epoch": 0.8089260808926081,
      "grad_norm": 4.353116512298584,
      "learning_rate": 2.4322209608286686e-05,
      "logits/chosen": 8.986726760864258,
      "logits/rejected": 12.175418853759766,
      "logps/chosen": -206.93594360351562,
      "logps/rejected": -292.4324645996094,
      "loss": 0.5075,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -0.7221769094467163,
      "rewards/margins": 1.2281521558761597,
      "rewards/rejected": -1.9503291845321655,
      "step": 290
    },
    {
      "epoch": 0.8228730822873083,
      "grad_norm": 8.33502197265625,
      "learning_rate": 2.421431118313789e-05,
      "logits/chosen": 10.475484848022461,
      "logits/rejected": 11.844476699829102,
      "logps/chosen": -240.02206420898438,
      "logps/rejected": -279.66064453125,
      "loss": 0.7693,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0663460493087769,
      "rewards/margins": 0.940434455871582,
      "rewards/rejected": -2.0067803859710693,
      "step": 295
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 7.870750904083252,
      "learning_rate": 2.41047615265918e-05,
      "logits/chosen": 11.179079055786133,
      "logits/rejected": 13.722526550292969,
      "logps/chosen": -243.4619140625,
      "logps/rejected": -338.6166687011719,
      "loss": 0.5564,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.5845211744308472,
      "rewards/margins": 1.252018690109253,
      "rewards/rejected": -1.8365398645401,
      "step": 300
    },
    {
      "epoch": 0.8507670850767085,
      "grad_norm": 5.815814018249512,
      "learning_rate": 2.3993585677327107e-05,
      "logits/chosen": 13.206197738647461,
      "logits/rejected": 13.943652153015137,
      "logps/chosen": -276.7415466308594,
      "logps/rejected": -329.16949462890625,
      "loss": 0.5341,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4414283335208893,
      "rewards/margins": 0.9674088358879089,
      "rewards/rejected": -1.408837080001831,
      "step": 305
    },
    {
      "epoch": 0.8647140864714087,
      "grad_norm": 8.953941345214844,
      "learning_rate": 2.3880809045705262e-05,
      "logits/chosen": 12.374226570129395,
      "logits/rejected": 13.989924430847168,
      "logps/chosen": -244.1103515625,
      "logps/rejected": -294.4126281738281,
      "loss": 0.5197,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -0.3111744523048401,
      "rewards/margins": 1.076432228088379,
      "rewards/rejected": -1.3876066207885742,
      "step": 310
    },
    {
      "epoch": 0.8786610878661087,
      "grad_norm": 5.921610355377197,
      "learning_rate": 2.3766457407962654e-05,
      "logits/chosen": 11.312451362609863,
      "logits/rejected": 13.527206420898438,
      "logps/chosen": -234.7072296142578,
      "logps/rejected": -290.8281555175781,
      "loss": 0.6344,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -0.5945696234703064,
      "rewards/margins": 0.7386828660964966,
      "rewards/rejected": -1.3332524299621582,
      "step": 315
    },
    {
      "epoch": 0.8926080892608089,
      "grad_norm": 7.662965774536133,
      "learning_rate": 2.3650556900319204e-05,
      "logits/chosen": 11.144214630126953,
      "logits/rejected": 13.931811332702637,
      "logps/chosen": -198.17422485351562,
      "logps/rejected": -286.37152099609375,
      "loss": 0.5295,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -0.3581700325012207,
      "rewards/margins": 1.0556398630142212,
      "rewards/rejected": -1.4138100147247314,
      "step": 320
    },
    {
      "epoch": 0.9065550906555091,
      "grad_norm": 6.2693071365356445,
      "learning_rate": 2.3533134013004666e-05,
      "logits/chosen": 11.203069686889648,
      "logits/rejected": 11.721672058105469,
      "logps/chosen": -192.05380249023438,
      "logps/rejected": -221.40042114257812,
      "loss": 0.6338,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -0.56825852394104,
      "rewards/margins": 0.6851181983947754,
      "rewards/rejected": -1.253376841545105,
      "step": 325
    },
    {
      "epoch": 0.9205020920502092,
      "grad_norm": 4.502702236175537,
      "learning_rate": 2.341421558420403e-05,
      "logits/chosen": 10.94641399383545,
      "logits/rejected": 13.1921968460083,
      "logps/chosen": -214.18911743164062,
      "logps/rejected": -298.7375183105469,
      "loss": 0.4812,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -0.6049180030822754,
      "rewards/margins": 1.2558748722076416,
      "rewards/rejected": -1.860793113708496,
      "step": 330
    },
    {
      "epoch": 0.9344490934449093,
      "grad_norm": 6.939857482910156,
      "learning_rate": 2.3293828793923365e-05,
      "logits/chosen": 11.944158554077148,
      "logits/rejected": 13.312121391296387,
      "logps/chosen": -258.62774658203125,
      "logps/rejected": -314.3907165527344,
      "loss": 0.619,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -0.5666269063949585,
      "rewards/margins": 1.0656505823135376,
      "rewards/rejected": -1.632277488708496,
      "step": 335
    },
    {
      "epoch": 0.9483960948396095,
      "grad_norm": 4.783708572387695,
      "learning_rate": 2.3172001157777566e-05,
      "logits/chosen": 11.259064674377441,
      "logits/rejected": 12.700227737426758,
      "logps/chosen": -222.77395629882812,
      "logps/rejected": -291.24884033203125,
      "loss": 0.6497,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -0.5113255977630615,
      "rewards/margins": 0.866672158241272,
      "rewards/rejected": -1.377997636795044,
      "step": 340
    },
    {
      "epoch": 0.9623430962343096,
      "grad_norm": 4.230240821838379,
      "learning_rate": 2.3048760520701374e-05,
      "logits/chosen": 11.741998672485352,
      "logits/rejected": 13.120841979980469,
      "logps/chosen": -246.1442108154297,
      "logps/rejected": -311.65972900390625,
      "loss": 0.485,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -0.4931299090385437,
      "rewards/margins": 1.0040347576141357,
      "rewards/rejected": -1.4971646070480347,
      "step": 345
    },
    {
      "epoch": 0.9762900976290098,
      "grad_norm": 4.901912212371826,
      "learning_rate": 2.2924135050585152e-05,
      "logits/chosen": 11.310202598571777,
      "logits/rejected": 13.040916442871094,
      "logps/chosen": -223.5089111328125,
      "logps/rejected": -253.26953125,
      "loss": 0.5638,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -0.6479132771492004,
      "rewards/margins": 0.7726107835769653,
      "rewards/rejected": -1.4205242395401,
      "step": 350
    },
    {
      "epoch": 0.9902370990237099,
      "grad_norm": 4.653822898864746,
      "learning_rate": 2.2798153231836813e-05,
      "logits/chosen": 12.201348304748535,
      "logits/rejected": 13.601194381713867,
      "logps/chosen": -268.86328125,
      "logps/rejected": -299.11578369140625,
      "loss": 0.5367,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -0.5867849588394165,
      "rewards/margins": 1.1348934173583984,
      "rewards/rejected": -1.721678376197815,
      "step": 355
    },
    {
      "epoch": 0.99860529986053,
      "eval_logits/chosen": 11.91541862487793,
      "eval_logits/rejected": 12.808130264282227,
      "eval_logps/chosen": -229.4397430419922,
      "eval_logps/rejected": -286.59234619140625,
      "eval_loss": 0.6298339366912842,
      "eval_rewards/accuracies": 0.7099999785423279,
      "eval_rewards/chosen": -0.7923385500907898,
      "eval_rewards/margins": 0.9493054151535034,
      "eval_rewards/rejected": -1.7416436672210693,
      "eval_runtime": 24.5973,
      "eval_samples_per_second": 8.131,
      "eval_steps_per_second": 8.131,
      "step": 358
    }
  ],
  "logging_steps": 5,
  "max_steps": 1074,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 12,
  "trial_name": null,
  "trial_params": null
}