{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.08977063602495623,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0008977063602495623,
      "grad_norm": 49.75,
      "learning_rate": 1.8797356064157479e-06,
      "logits/chosen": -1.5944416522979736,
      "logits/rejected": -1.6120755672454834,
      "logps/chosen": -218.08145141601562,
      "logps/rejected": -238.650634765625,
      "loss": 0.69,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003223979379981756,
      "rewards/margins": 0.006512450985610485,
      "rewards/rejected": -0.003288471605628729,
      "step": 5
    },
    {
      "epoch": 0.0017954127204991247,
      "grad_norm": 52.75,
      "learning_rate": 4.229405114435433e-06,
      "logits/chosen": -1.638082504272461,
      "logits/rejected": -1.644774079322815,
      "logps/chosen": -218.1611785888672,
      "logps/rejected": -238.66098022460938,
      "loss": 0.6066,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07871033251285553,
      "rewards/margins": 0.1871051788330078,
      "rewards/rejected": -0.10839483886957169,
      "step": 10
    },
    {
      "epoch": 0.002693119080748687,
      "grad_norm": 28.625,
      "learning_rate": 6.579074622455118e-06,
      "logits/chosen": -1.7269313335418701,
      "logits/rejected": -1.7303335666656494,
      "logps/chosen": -226.7982635498047,
      "logps/rejected": -258.6452331542969,
      "loss": 0.3006,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -0.6204186081886292,
      "rewards/margins": 1.243154764175415,
      "rewards/rejected": -1.8635733127593994,
      "step": 15
    },
    {
      "epoch": 0.0035908254409982494,
      "grad_norm": 7.84375,
      "learning_rate": 8.928744130474802e-06,
      "logits/chosen": -1.7967208623886108,
      "logits/rejected": -1.814859390258789,
      "logps/chosen": -229.9993133544922,
      "logps/rejected": -288.5595397949219,
      "loss": 0.0873,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8708345890045166,
      "rewards/margins": 3.8205904960632324,
      "rewards/rejected": -5.691425323486328,
      "step": 20
    },
    {
      "epoch": 0.004488531801247812,
      "grad_norm": 14.875,
      "learning_rate": 1.1278413638494489e-05,
      "logits/chosen": -1.7308677434921265,
      "logits/rejected": -1.7561269998550415,
      "logps/chosen": -243.46029663085938,
      "logps/rejected": -340.27764892578125,
      "loss": 0.018,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -3.0787580013275146,
      "rewards/margins": 7.586331367492676,
      "rewards/rejected": -10.665090560913086,
      "step": 25
    },
    {
      "epoch": 0.005386238161497374,
      "grad_norm": 8.25,
      "learning_rate": 1.3628083146514173e-05,
      "logits/chosen": -1.6984357833862305,
      "logits/rejected": -1.7242708206176758,
      "logps/chosen": -264.53125,
      "logps/rejected": -376.6803894042969,
      "loss": 0.0285,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.747864246368408,
      "rewards/margins": 9.422819137573242,
      "rewards/rejected": -14.170684814453125,
      "step": 30
    },
    {
      "epoch": 0.006283944521746937,
      "grad_norm": 4.1875,
      "learning_rate": 1.5977752654533858e-05,
      "logits/chosen": -1.6455342769622803,
      "logits/rejected": -1.6728187799453735,
      "logps/chosen": -281.365966796875,
      "logps/rejected": -407.96337890625,
      "loss": 0.0216,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -6.47125768661499,
      "rewards/margins": 10.662993431091309,
      "rewards/rejected": -17.13425064086914,
      "step": 35
    },
    {
      "epoch": 0.007181650881996499,
      "grad_norm": 6.3125,
      "learning_rate": 1.6447684804072058e-05,
      "logits/chosen": -1.5919939279556274,
      "logits/rejected": -1.617920160293579,
      "logps/chosen": -278.3464660644531,
      "logps/rejected": -404.8271484375,
      "loss": 0.0342,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -6.385420799255371,
      "rewards/margins": 10.654914855957031,
      "rewards/rejected": -17.040334701538086,
      "step": 40
    },
    {
      "epoch": 0.00807935724224606,
      "grad_norm": 6.625,
      "learning_rate": 1.6447677686306693e-05,
      "logits/chosen": -1.6035076379776,
      "logits/rejected": -1.6163572072982788,
      "logps/chosen": -279.767822265625,
      "logps/rejected": -383.13458251953125,
      "loss": 0.0432,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -5.704493999481201,
      "rewards/margins": 8.454301834106445,
      "rewards/rejected": -14.158796310424805,
      "step": 45
    },
    {
      "epoch": 0.008977063602495623,
      "grad_norm": 4.8125,
      "learning_rate": 1.6447665093343918e-05,
      "logits/chosen": -1.6678664684295654,
      "logits/rejected": -1.6700479984283447,
      "logps/chosen": -264.4732971191406,
      "logps/rejected": -363.1940612792969,
      "loss": 0.0464,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -5.7117018699646,
      "rewards/margins": 7.980559349060059,
      "rewards/rejected": -13.692262649536133,
      "step": 50
    },
    {
      "epoch": 0.009874769962745186,
      "grad_norm": 4.65625,
      "learning_rate": 1.6447647025194904e-05,
      "logits/chosen": -1.5799241065979004,
      "logits/rejected": -1.5821675062179565,
      "logps/chosen": -268.1691589355469,
      "logps/rejected": -374.00518798828125,
      "loss": 0.0205,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.694095134735107,
      "rewards/margins": 8.601041793823242,
      "rewards/rejected": -14.295137405395508,
      "step": 55
    },
    {
      "epoch": 0.010772476322994749,
      "grad_norm": 4.625,
      "learning_rate": 1.6447623481875693e-05,
      "logits/chosen": -1.615523338317871,
      "logits/rejected": -1.6053167581558228,
      "logps/chosen": -269.4774475097656,
      "logps/rejected": -389.2843933105469,
      "loss": 0.0186,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.536202907562256,
      "rewards/margins": 9.901906967163086,
      "rewards/rejected": -15.4381103515625,
      "step": 60
    },
    {
      "epoch": 0.011670182683244311,
      "grad_norm": 5.0625,
      "learning_rate": 1.644759446340718e-05,
      "logits/chosen": -1.62222158908844,
      "logits/rejected": -1.6126108169555664,
      "logps/chosen": -273.30316162109375,
      "logps/rejected": -401.03851318359375,
      "loss": 0.0237,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.82913875579834,
      "rewards/margins": 10.689005851745605,
      "rewards/rejected": -16.518144607543945,
      "step": 65
    },
    {
      "epoch": 0.012567889043493874,
      "grad_norm": 5.0625,
      "learning_rate": 1.644755996981513e-05,
      "logits/chosen": -1.6640081405639648,
      "logits/rejected": -1.6507833003997803,
      "logps/chosen": -274.74053955078125,
      "logps/rejected": -404.7392883300781,
      "loss": 0.0319,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -5.577990531921387,
      "rewards/margins": 10.969307899475098,
      "rewards/rejected": -16.547298431396484,
      "step": 70
    },
    {
      "epoch": 0.013465595403743435,
      "grad_norm": 30.25,
      "learning_rate": 1.6447520001130158e-05,
      "logits/chosen": -1.5772068500518799,
      "logits/rejected": -1.5707197189331055,
      "logps/chosen": -278.74658203125,
      "logps/rejected": -406.4679260253906,
      "loss": 0.0555,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -6.121307849884033,
      "rewards/margins": 10.843521118164062,
      "rewards/rejected": -16.96483039855957,
      "step": 75
    },
    {
      "epoch": 0.014363301763992998,
      "grad_norm": 3.109375,
      "learning_rate": 1.6447474557387748e-05,
      "logits/chosen": -1.53738534450531,
      "logits/rejected": -1.534790277481079,
      "logps/chosen": -314.1097717285156,
      "logps/rejected": -420.47308349609375,
      "loss": 0.0185,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -9.349452018737793,
      "rewards/margins": 8.684282302856445,
      "rewards/rejected": -18.033733367919922,
      "step": 80
    },
    {
      "epoch": 0.01526100812424256,
      "grad_norm": 4.03125,
      "learning_rate": 1.6447423638628237e-05,
      "logits/chosen": -1.5148117542266846,
      "logits/rejected": -1.5227998495101929,
      "logps/chosen": -316.7289123535156,
      "logps/rejected": -430.1024475097656,
      "loss": 0.0115,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -10.431829452514648,
      "rewards/margins": 9.403815269470215,
      "rewards/rejected": -19.835643768310547,
      "step": 85
    },
    {
      "epoch": 0.01615871448449212,
      "grad_norm": 3.84375,
      "learning_rate": 1.6447367244896826e-05,
      "logits/chosen": -1.606527328491211,
      "logits/rejected": -1.6100928783416748,
      "logps/chosen": -319.0159606933594,
      "logps/rejected": -440.75738525390625,
      "loss": 0.0257,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -10.429452896118164,
      "rewards/margins": 10.196511268615723,
      "rewards/rejected": -20.625965118408203,
      "step": 90
    },
    {
      "epoch": 0.017056420844741686,
      "grad_norm": 24.125,
      "learning_rate": 1.644730537624358e-05,
      "logits/chosen": -1.6584317684173584,
      "logits/rejected": -1.6650241613388062,
      "logps/chosen": -315.8913269042969,
      "logps/rejected": -439.966552734375,
      "loss": 0.0371,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -10.410109519958496,
      "rewards/margins": 10.561999320983887,
      "rewards/rejected": -20.972110748291016,
      "step": 95
    },
    {
      "epoch": 0.017954127204991246,
      "grad_norm": 13.375,
      "learning_rate": 1.644723803272341e-05,
      "logits/chosen": -1.6700522899627686,
      "logits/rejected": -1.6734033823013306,
      "logps/chosen": -319.60516357421875,
      "logps/rejected": -441.1056213378906,
      "loss": 0.0222,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.69153881072998,
      "rewards/margins": 10.231417655944824,
      "rewards/rejected": -20.922958374023438,
      "step": 100
    },
    {
      "epoch": 0.01885183356524081,
      "grad_norm": 6.3125,
      "learning_rate": 1.644716521439611e-05,
      "logits/chosen": -1.7060085535049438,
      "logits/rejected": -1.7089792490005493,
      "logps/chosen": -347.458984375,
      "logps/rejected": -468.8707580566406,
      "loss": 0.0224,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -12.833531379699707,
      "rewards/margins": 10.041297912597656,
      "rewards/rejected": -22.874828338623047,
      "step": 105
    },
    {
      "epoch": 0.019749539925490372,
      "grad_norm": 10.8125,
      "learning_rate": 1.644708692132631e-05,
      "logits/chosen": -1.7060142755508423,
      "logits/rejected": -1.7084630727767944,
      "logps/chosen": -356.79656982421875,
      "logps/rejected": -479.97235107421875,
      "loss": 0.0264,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -13.447436332702637,
      "rewards/margins": 10.534585952758789,
      "rewards/rejected": -23.982025146484375,
      "step": 110
    },
    {
      "epoch": 0.020647246285739933,
      "grad_norm": 6.15625,
      "learning_rate": 1.6447003153583514e-05,
      "logits/chosen": -1.642289161682129,
      "logits/rejected": -1.6480754613876343,
      "logps/chosen": -356.51251220703125,
      "logps/rejected": -481.41827392578125,
      "loss": 0.0122,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -13.347066879272461,
      "rewards/margins": 10.536771774291992,
      "rewards/rejected": -23.883838653564453,
      "step": 115
    },
    {
      "epoch": 0.021544952645989497,
      "grad_norm": 8.3125,
      "learning_rate": 1.644691391124208e-05,
      "logits/chosen": -1.6251140832901,
      "logits/rejected": -1.6293065547943115,
      "logps/chosen": -362.5952453613281,
      "logps/rejected": -487.474853515625,
      "loss": 0.0638,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -14.098132133483887,
      "rewards/margins": 10.43175983428955,
      "rewards/rejected": -24.529891967773438,
      "step": 120
    },
    {
      "epoch": 0.022442659006239058,
      "grad_norm": 0.9375,
      "learning_rate": 1.6446819194381232e-05,
      "logits/chosen": -1.6321861743927002,
      "logits/rejected": -1.6401519775390625,
      "logps/chosen": -365.3531188964844,
      "logps/rejected": -472.98590087890625,
      "loss": 0.0341,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -14.91484546661377,
      "rewards/margins": 8.75818920135498,
      "rewards/rejected": -23.673038482666016,
      "step": 125
    },
    {
      "epoch": 0.023340365366488623,
      "grad_norm": 16.125,
      "learning_rate": 1.6446719003085048e-05,
      "logits/chosen": -1.682080864906311,
      "logits/rejected": -1.6901333332061768,
      "logps/chosen": -375.67047119140625,
      "logps/rejected": -475.9076232910156,
      "loss": 0.0263,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -14.836702346801758,
      "rewards/margins": 8.173124313354492,
      "rewards/rejected": -23.00982666015625,
      "step": 130
    },
    {
      "epoch": 0.024238071726738183,
      "grad_norm": 5.09375,
      "learning_rate": 1.6446613337442464e-05,
      "logits/chosen": -1.7631546258926392,
      "logits/rejected": -1.75924813747406,
      "logps/chosen": -334.9482421875,
      "logps/rejected": -438.7393493652344,
      "loss": 0.0274,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.050386428833008,
      "rewards/margins": 8.398730278015137,
      "rewards/rejected": -20.44911766052246,
      "step": 135
    },
    {
      "epoch": 0.025135778086987748,
      "grad_norm": 16.125,
      "learning_rate": 1.6446502197547285e-05,
      "logits/chosen": -1.6700756549835205,
      "logits/rejected": -1.6583993434906006,
      "logps/chosen": -327.59222412109375,
      "logps/rejected": -443.2696228027344,
      "loss": 0.0349,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.80932903289795,
      "rewards/margins": 9.692026138305664,
      "rewards/rejected": -20.501354217529297,
      "step": 140
    },
    {
      "epoch": 0.02603348444723731,
      "grad_norm": 6.5625,
      "learning_rate": 1.6446385583498166e-05,
      "logits/chosen": -1.603623628616333,
      "logits/rejected": -1.5888742208480835,
      "logps/chosen": -325.58660888671875,
      "logps/rejected": -446.7493591308594,
      "loss": 0.0473,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -11.347869873046875,
      "rewards/margins": 10.037898063659668,
      "rewards/rejected": -21.38576889038086,
      "step": 145
    },
    {
      "epoch": 0.02693119080748687,
      "grad_norm": 5.21875,
      "learning_rate": 1.6446263495398625e-05,
      "logits/chosen": -1.6120811700820923,
      "logits/rejected": -1.5870082378387451,
      "logps/chosen": -317.60162353515625,
      "logps/rejected": -438.42633056640625,
      "loss": 0.0193,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -9.432976722717285,
      "rewards/margins": 10.237409591674805,
      "rewards/rejected": -19.670385360717773,
      "step": 150
    },
    {
      "epoch": 0.027828897167736434,
      "grad_norm": 3.4375,
      "learning_rate": 1.644613593335704e-05,
      "logits/chosen": -1.5875444412231445,
      "logits/rejected": -1.5749518871307373,
      "logps/chosen": -303.3853454589844,
      "logps/rejected": -432.47808837890625,
      "loss": 0.0154,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -8.602149963378906,
      "rewards/margins": 10.840978622436523,
      "rewards/rejected": -19.443126678466797,
      "step": 155
    },
    {
      "epoch": 0.028726603527985995,
      "grad_norm": 3.9375,
      "learning_rate": 1.6446002897486648e-05,
      "logits/chosen": -1.618011236190796,
      "logits/rejected": -1.6145331859588623,
      "logps/chosen": -312.6946105957031,
      "logps/rejected": -446.9215393066406,
      "loss": 0.0265,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.892807960510254,
      "rewards/margins": 11.515321731567383,
      "rewards/rejected": -20.408130645751953,
      "step": 160
    },
    {
      "epoch": 0.02962430988823556,
      "grad_norm": 5.96875,
      "learning_rate": 1.644586438790554e-05,
      "logits/chosen": -1.5836814641952515,
      "logits/rejected": -1.587181806564331,
      "logps/chosen": -306.9125061035156,
      "logps/rejected": -445.1659240722656,
      "loss": 0.0256,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.431074142456055,
      "rewards/margins": 11.796531677246094,
      "rewards/rejected": -21.22760581970215,
      "step": 165
    },
    {
      "epoch": 0.03052201624848512,
      "grad_norm": 11.125,
      "learning_rate": 1.6445720404736678e-05,
      "logits/chosen": -1.6508190631866455,
      "logits/rejected": -1.65244460105896,
      "logps/chosen": -310.2176208496094,
      "logps/rejected": -440.75714111328125,
      "loss": 0.0288,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.615917205810547,
      "rewards/margins": 11.127912521362305,
      "rewards/rejected": -20.743831634521484,
      "step": 170
    },
    {
      "epoch": 0.031419722608734685,
      "grad_norm": 7.0,
      "learning_rate": 1.644557094810787e-05,
      "logits/chosen": -1.7216987609863281,
      "logits/rejected": -1.7154676914215088,
      "logps/chosen": -340.46466064453125,
      "logps/rejected": -450.27294921875,
      "loss": 0.0397,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -11.357550621032715,
      "rewards/margins": 8.966830253601074,
      "rewards/rejected": -20.32438087463379,
      "step": 175
    },
    {
      "epoch": 0.03231742896898424,
      "grad_norm": 24.0,
      "learning_rate": 1.6445416018151788e-05,
      "logits/chosen": -1.7959930896759033,
      "logits/rejected": -1.800244927406311,
      "logps/chosen": -331.75506591796875,
      "logps/rejected": -427.2474670410156,
      "loss": 0.0207,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -12.001955032348633,
      "rewards/margins": 7.540495872497559,
      "rewards/rejected": -19.542451858520508,
      "step": 180
    },
    {
      "epoch": 0.03321513532923381,
      "grad_norm": 43.75,
      "learning_rate": 1.644525561500596e-05,
      "logits/chosen": -1.9910930395126343,
      "logits/rejected": -1.9894500970840454,
      "logps/chosen": -350.335693359375,
      "logps/rejected": -449.16796875,
      "loss": 0.0458,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.615007400512695,
      "rewards/margins": 8.14229679107666,
      "rewards/rejected": -21.757305145263672,
      "step": 185
    },
    {
      "epoch": 0.03411284168948337,
      "grad_norm": 13.3125,
      "learning_rate": 1.6445089738812785e-05,
      "logits/chosen": -1.9771511554718018,
      "logits/rejected": -1.9749290943145752,
      "logps/chosen": -348.55828857421875,
      "logps/rejected": -453.4300842285156,
      "loss": 0.0432,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -13.186309814453125,
      "rewards/margins": 8.536267280578613,
      "rewards/rejected": -21.722576141357422,
      "step": 190
    },
    {
      "epoch": 0.035010548049732935,
      "grad_norm": 0.146484375,
      "learning_rate": 1.6444918389719505e-05,
      "logits/chosen": -1.9536895751953125,
      "logits/rejected": -1.9450359344482422,
      "logps/chosen": -335.36553955078125,
      "logps/rejected": -448.2103576660156,
      "loss": 0.0296,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -11.7564697265625,
      "rewards/margins": 9.423359870910645,
      "rewards/rejected": -21.179828643798828,
      "step": 195
    },
    {
      "epoch": 0.03590825440998249,
      "grad_norm": 20.25,
      "learning_rate": 1.644474156787822e-05,
      "logits/chosen": -1.878861665725708,
      "logits/rejected": -1.8584403991699219,
      "logps/chosen": -319.38067626953125,
      "logps/rejected": -443.59033203125,
      "loss": 0.0307,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.937708854675293,
      "rewards/margins": 10.651717185974121,
      "rewards/rejected": -21.589426040649414,
      "step": 200
    },
    {
      "epoch": 0.03680596077023206,
      "grad_norm": 17.875,
      "learning_rate": 1.6444559273445908e-05,
      "logits/chosen": -1.6908838748931885,
      "logits/rejected": -1.6792293787002563,
      "logps/chosen": -321.28009033203125,
      "logps/rejected": -460.1475524902344,
      "loss": 0.022,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -10.206222534179688,
      "rewards/margins": 11.932024955749512,
      "rewards/rejected": -22.138248443603516,
      "step": 205
    },
    {
      "epoch": 0.03770366713048162,
      "grad_norm": 7.0,
      "learning_rate": 1.6444371506584377e-05,
      "logits/chosen": -1.6957308053970337,
      "logits/rejected": -1.690157175064087,
      "logps/chosen": -290.0798034667969,
      "logps/rejected": -412.7769470214844,
      "loss": 0.018,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -7.47702693939209,
      "rewards/margins": 10.390680313110352,
      "rewards/rejected": -17.86771011352539,
      "step": 210
    },
    {
      "epoch": 0.03860137349073118,
      "grad_norm": 8.5625,
      "learning_rate": 1.644417826746031e-05,
      "logits/chosen": -1.650665521621704,
      "logits/rejected": -1.6541109085083008,
      "logps/chosen": -290.309326171875,
      "logps/rejected": -417.1390686035156,
      "loss": 0.0184,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.279687404632568,
      "rewards/margins": 10.711103439331055,
      "rewards/rejected": -17.990793228149414,
      "step": 215
    },
    {
      "epoch": 0.039499079850980744,
      "grad_norm": 8.4375,
      "learning_rate": 1.6443979556245252e-05,
      "logits/chosen": -1.6047160625457764,
      "logits/rejected": -1.6234970092773438,
      "logps/chosen": -322.4940185546875,
      "logps/rejected": -458.4981384277344,
      "loss": 0.0263,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -9.725212097167969,
      "rewards/margins": 11.667040824890137,
      "rewards/rejected": -21.39225196838379,
      "step": 220
    },
    {
      "epoch": 0.04039678621123031,
      "grad_norm": 23.5,
      "learning_rate": 1.6443775373115592e-05,
      "logits/chosen": -1.5689036846160889,
      "logits/rejected": -1.5908584594726562,
      "logps/chosen": -343.6766052246094,
      "logps/rejected": -472.40948486328125,
      "loss": 0.0679,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -11.692893028259277,
      "rewards/margins": 10.971635818481445,
      "rewards/rejected": -22.66452980041504,
      "step": 225
    },
    {
      "epoch": 0.041294492571479866,
      "grad_norm": 8.6875,
      "learning_rate": 1.6443565718252586e-05,
      "logits/chosen": -1.5273631811141968,
      "logits/rejected": -1.5362848043441772,
      "logps/chosen": -333.0375061035156,
      "logps/rejected": -450.404541015625,
      "loss": 0.0259,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -11.445978164672852,
      "rewards/margins": 9.806886672973633,
      "rewards/rejected": -21.25286293029785,
      "step": 230
    },
    {
      "epoch": 0.04219219893172943,
      "grad_norm": 9.25,
      "learning_rate": 1.644335059184234e-05,
      "logits/chosen": -1.4887597560882568,
      "logits/rejected": -1.5063436031341553,
      "logps/chosen": -334.2455139160156,
      "logps/rejected": -466.03326416015625,
      "loss": 0.0239,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -12.205533981323242,
      "rewards/margins": 10.970166206359863,
      "rewards/rejected": -23.175701141357422,
      "step": 235
    },
    {
      "epoch": 0.043089905291978994,
      "grad_norm": 5.125,
      "learning_rate": 1.644312999407582e-05,
      "logits/chosen": -1.4916335344314575,
      "logits/rejected": -1.5103265047073364,
      "logps/chosen": -339.73175048828125,
      "logps/rejected": -466.05645751953125,
      "loss": 0.0523,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.136640548706055,
      "rewards/margins": 10.656683921813965,
      "rewards/rejected": -22.793325424194336,
      "step": 240
    },
    {
      "epoch": 0.04398761165222856,
      "grad_norm": 6.1875,
      "learning_rate": 1.644290392514886e-05,
      "logits/chosen": -1.4491441249847412,
      "logits/rejected": -1.4810426235198975,
      "logps/chosen": -330.1545104980469,
      "logps/rejected": -447.7718811035156,
      "loss": 0.0283,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -11.471087455749512,
      "rewards/margins": 9.670295715332031,
      "rewards/rejected": -21.14138412475586,
      "step": 245
    },
    {
      "epoch": 0.044885318012478116,
      "grad_norm": 9.75,
      "learning_rate": 1.6442672385262126e-05,
      "logits/chosen": -1.3768192529678345,
      "logits/rejected": -1.4130717515945435,
      "logps/chosen": -315.2447204589844,
      "logps/rejected": -440.4739685058594,
      "loss": 0.0267,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.256295204162598,
      "rewards/margins": 10.560578346252441,
      "rewards/rejected": -19.81687355041504,
      "step": 250
    },
    {
      "epoch": 0.04578302437272768,
      "grad_norm": 1.8125,
      "learning_rate": 1.6442435374621164e-05,
      "logits/chosen": -1.3219325542449951,
      "logits/rejected": -1.3581187725067139,
      "logps/chosen": -295.7016906738281,
      "logps/rejected": -422.5862731933594,
      "loss": 0.0325,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -8.51601791381836,
      "rewards/margins": 10.66891098022461,
      "rewards/rejected": -19.18492889404297,
      "step": 255
    },
    {
      "epoch": 0.046680730732977245,
      "grad_norm": 1.265625,
      "learning_rate": 1.6442192893436368e-05,
      "logits/chosen": -1.2778997421264648,
      "logits/rejected": -1.312280535697937,
      "logps/chosen": -303.11151123046875,
      "logps/rejected": -420.34912109375,
      "loss": 0.024,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -8.392851829528809,
      "rewards/margins": 9.67725658416748,
      "rewards/rejected": -18.070110321044922,
      "step": 260
    },
    {
      "epoch": 0.0475784370932268,
      "grad_norm": 2.765625,
      "learning_rate": 1.644194494192298e-05,
      "logits/chosen": -1.2928860187530518,
      "logits/rejected": -1.3260154724121094,
      "logps/chosen": -286.2437438964844,
      "logps/rejected": -393.56231689453125,
      "loss": 0.0281,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.124878883361816,
      "rewards/margins": 8.837203979492188,
      "rewards/rejected": -15.962081909179688,
      "step": 265
    },
    {
      "epoch": 0.04847614345347637,
      "grad_norm": 16.125,
      "learning_rate": 1.6441691520301115e-05,
      "logits/chosen": -1.278626799583435,
      "logits/rejected": -1.3031818866729736,
      "logps/chosen": -310.00946044921875,
      "logps/rejected": -427.1468200683594,
      "loss": 0.0328,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.098076820373535,
      "rewards/margins": 9.82015609741211,
      "rewards/rejected": -18.918231964111328,
      "step": 270
    },
    {
      "epoch": 0.04937384981372593,
      "grad_norm": 2.625,
      "learning_rate": 1.644143262879573e-05,
      "logits/chosen": -1.355022668838501,
      "logits/rejected": -1.377715826034546,
      "logps/chosen": -317.8216247558594,
      "logps/rejected": -431.4881896972656,
      "loss": 0.0385,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -9.668425559997559,
      "rewards/margins": 9.433794975280762,
      "rewards/rejected": -19.102222442626953,
      "step": 275
    },
    {
      "epoch": 0.050271556173975496,
      "grad_norm": 9.75,
      "learning_rate": 1.644116826763664e-05,
      "logits/chosen": -1.3636281490325928,
      "logits/rejected": -1.384377360343933,
      "logps/chosen": -302.4226379394531,
      "logps/rejected": -413.4556579589844,
      "loss": 0.0475,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -9.204570770263672,
      "rewards/margins": 9.083320617675781,
      "rewards/rejected": -18.287891387939453,
      "step": 280
    },
    {
      "epoch": 0.05116926253422505,
      "grad_norm": 8.125,
      "learning_rate": 1.6440898437058523e-05,
      "logits/chosen": -1.340986728668213,
      "logits/rejected": -1.3553143739700317,
      "logps/chosen": -313.5350341796875,
      "logps/rejected": -417.8633728027344,
      "loss": 0.0624,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.931371688842773,
      "rewards/margins": 8.754600524902344,
      "rewards/rejected": -17.68597412109375,
      "step": 285
    },
    {
      "epoch": 0.05206696889447462,
      "grad_norm": 1.4609375,
      "learning_rate": 1.64406231373009e-05,
      "logits/chosen": -1.3220717906951904,
      "logits/rejected": -1.3361364603042603,
      "logps/chosen": -301.2831726074219,
      "logps/rejected": -406.24481201171875,
      "loss": 0.033,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.154109001159668,
      "rewards/margins": 8.63221263885498,
      "rewards/rejected": -17.78632164001465,
      "step": 290
    },
    {
      "epoch": 0.05296467525472418,
      "grad_norm": 5.15625,
      "learning_rate": 1.6440342368608156e-05,
      "logits/chosen": -1.2657798528671265,
      "logits/rejected": -1.2791422605514526,
      "logps/chosen": -326.9562072753906,
      "logps/rejected": -439.09356689453125,
      "loss": 0.061,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -11.56185531616211,
      "rewards/margins": 9.128196716308594,
      "rewards/rejected": -20.690053939819336,
      "step": 295
    },
    {
      "epoch": 0.05386238161497374,
      "grad_norm": 0.0198974609375,
      "learning_rate": 1.6440056131229532e-05,
      "logits/chosen": -1.2754865884780884,
      "logits/rejected": -1.2849574089050293,
      "logps/chosen": -345.1870422363281,
      "logps/rejected": -475.9234924316406,
      "loss": 0.0144,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -11.819540977478027,
      "rewards/margins": 11.19267463684082,
      "rewards/rejected": -23.012216567993164,
      "step": 300
    },
    {
      "epoch": 0.054760087975223304,
      "grad_norm": 1.828125,
      "learning_rate": 1.6439764425419112e-05,
      "logits/chosen": -1.274107813835144,
      "logits/rejected": -1.2885282039642334,
      "logps/chosen": -330.4737854003906,
      "logps/rejected": -473.398681640625,
      "loss": 0.0267,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -11.204086303710938,
      "rewards/margins": 12.114290237426758,
      "rewards/rejected": -23.318378448486328,
      "step": 305
    },
    {
      "epoch": 0.05565779433547287,
      "grad_norm": 13.9375,
      "learning_rate": 1.6439467251435852e-05,
      "logits/chosen": -1.2366708517074585,
      "logits/rejected": -1.2527769804000854,
      "logps/chosen": -323.2499694824219,
      "logps/rejected": -467.12890625,
      "loss": 0.023,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -11.250632286071777,
      "rewards/margins": 12.270976066589355,
      "rewards/rejected": -23.521610260009766,
      "step": 310
    },
    {
      "epoch": 0.05655550069572243,
      "grad_norm": 2.21875,
      "learning_rate": 1.6439164609543545e-05,
      "logits/chosen": -1.287007212638855,
      "logits/rejected": -1.315598726272583,
      "logps/chosen": -315.24200439453125,
      "logps/rejected": -456.1424255371094,
      "loss": 0.0308,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -10.730131149291992,
      "rewards/margins": 11.966339111328125,
      "rewards/rejected": -22.696468353271484,
      "step": 315
    },
    {
      "epoch": 0.05745320705597199,
      "grad_norm": 4.03125,
      "learning_rate": 1.6438856500010842e-05,
      "logits/chosen": -1.412188172340393,
      "logits/rejected": -1.4369876384735107,
      "logps/chosen": -304.62298583984375,
      "logps/rejected": -442.72808837890625,
      "loss": 0.0438,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -9.49343490600586,
      "rewards/margins": 11.810284614562988,
      "rewards/rejected": -21.303720474243164,
      "step": 320
    },
    {
      "epoch": 0.058350913416221555,
      "grad_norm": 3.34375,
      "learning_rate": 1.643854292311126e-05,
      "logits/chosen": -1.4016748666763306,
      "logits/rejected": -1.4312589168548584,
      "logps/chosen": -321.81982421875,
      "logps/rejected": -464.9969787597656,
      "loss": 0.016,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -9.75760269165039,
      "rewards/margins": 12.245366096496582,
      "rewards/rejected": -22.00296974182129,
      "step": 325
    },
    {
      "epoch": 0.05924861977647112,
      "grad_norm": 8.125,
      "learning_rate": 1.6438223879123157e-05,
      "logits/chosen": -1.420204520225525,
      "logits/rejected": -1.4521286487579346,
      "logps/chosen": -334.3448486328125,
      "logps/rejected": -486.096435546875,
      "loss": 0.0395,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -11.95421028137207,
      "rewards/margins": 13.074414253234863,
      "rewards/rejected": -25.02862548828125,
      "step": 330
    },
    {
      "epoch": 0.060146326136720676,
      "grad_norm": 7.4375,
      "learning_rate": 1.6437899368329744e-05,
      "logits/chosen": -1.4968700408935547,
      "logits/rejected": -1.516984224319458,
      "logps/chosen": -362.8880310058594,
      "logps/rejected": -503.55047607421875,
      "loss": 0.0443,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.416742324829102,
      "rewards/margins": 12.286725997924805,
      "rewards/rejected": -25.703466415405273,
      "step": 335
    },
    {
      "epoch": 0.06104403249697024,
      "grad_norm": 3.65625,
      "learning_rate": 1.643756939101909e-05,
      "logits/chosen": -1.4715522527694702,
      "logits/rejected": -1.500880479812622,
      "logps/chosen": -347.5955505371094,
      "logps/rejected": -490.23345947265625,
      "loss": 0.0188,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -13.808749198913574,
      "rewards/margins": 12.222585678100586,
      "rewards/rejected": -26.031335830688477,
      "step": 340
    },
    {
      "epoch": 0.061941738857219805,
      "grad_norm": 0.0283203125,
      "learning_rate": 1.6437233947484115e-05,
      "logits/chosen": -1.4634774923324585,
      "logits/rejected": -1.4903171062469482,
      "logps/chosen": -346.1238708496094,
      "logps/rejected": -480.40740966796875,
      "loss": 0.026,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -13.692280769348145,
      "rewards/margins": 11.461533546447754,
      "rewards/rejected": -25.153812408447266,
      "step": 345
    },
    {
      "epoch": 0.06283944521746937,
      "grad_norm": 2.734375,
      "learning_rate": 1.6436893038022587e-05,
      "logits/chosen": -1.4172029495239258,
      "logits/rejected": -1.442546010017395,
      "logps/chosen": -339.7004089355469,
      "logps/rejected": -476.07861328125,
      "loss": 0.0222,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -13.242881774902344,
      "rewards/margins": 11.535491943359375,
      "rewards/rejected": -24.77837562561035,
      "step": 350
    },
    {
      "epoch": 0.06373715157771893,
      "grad_norm": 2.515625,
      "learning_rate": 1.6436546662937136e-05,
      "logits/chosen": -1.4132306575775146,
      "logits/rejected": -1.438727855682373,
      "logps/chosen": -340.2590637207031,
      "logps/rejected": -478.34759521484375,
      "loss": 0.03,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.395490646362305,
      "rewards/margins": 11.930428504943848,
      "rewards/rejected": -24.325918197631836,
      "step": 355
    },
    {
      "epoch": 0.06463485793796848,
      "grad_norm": 1.9609375,
      "learning_rate": 1.6436194822535237e-05,
      "logits/chosen": -1.3696801662445068,
      "logits/rejected": -1.4052057266235352,
      "logps/chosen": -332.30072021484375,
      "logps/rejected": -476.0782165527344,
      "loss": 0.0272,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.618162155151367,
      "rewards/margins": 12.268811225891113,
      "rewards/rejected": -24.886974334716797,
      "step": 360
    },
    {
      "epoch": 0.06553256429821805,
      "grad_norm": 1.421875,
      "learning_rate": 1.643583751712921e-05,
      "logits/chosen": -1.3992929458618164,
      "logits/rejected": -1.4282127618789673,
      "logps/chosen": -334.0565490722656,
      "logps/rejected": -480.23785400390625,
      "loss": 0.0185,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -11.715566635131836,
      "rewards/margins": 12.651227951049805,
      "rewards/rejected": -24.36679458618164,
      "step": 365
    },
    {
      "epoch": 0.06643027065846761,
      "grad_norm": 5.1875,
      "learning_rate": 1.6435474747036243e-05,
      "logits/chosen": -1.453920602798462,
      "logits/rejected": -1.4755427837371826,
      "logps/chosen": -322.3940124511719,
      "logps/rejected": -467.23126220703125,
      "loss": 0.0371,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -11.112676620483398,
      "rewards/margins": 12.558187484741211,
      "rewards/rejected": -23.67086410522461,
      "step": 370
    },
    {
      "epoch": 0.06732797701871718,
      "grad_norm": 3.84375,
      "learning_rate": 1.643510651257836e-05,
      "logits/chosen": -1.4459034204483032,
      "logits/rejected": -1.4675936698913574,
      "logps/chosen": -320.50347900390625,
      "logps/rejected": -463.59014892578125,
      "loss": 0.0144,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.755430221557617,
      "rewards/margins": 12.185154914855957,
      "rewards/rejected": -22.94058609008789,
      "step": 375
    },
    {
      "epoch": 0.06822568337896674,
      "grad_norm": 0.0186767578125,
      "learning_rate": 1.6434732814082442e-05,
      "logits/chosen": -1.4478992223739624,
      "logits/rejected": -1.4632583856582642,
      "logps/chosen": -331.53289794921875,
      "logps/rejected": -466.4090881347656,
      "loss": 0.0205,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -10.466972351074219,
      "rewards/margins": 11.554471969604492,
      "rewards/rejected": -22.021446228027344,
      "step": 380
    },
    {
      "epoch": 0.0691233897392163,
      "grad_norm": 5.21875,
      "learning_rate": 1.6434353651880223e-05,
      "logits/chosen": -1.4576263427734375,
      "logits/rejected": -1.470090627670288,
      "logps/chosen": -322.73773193359375,
      "logps/rejected": -458.134521484375,
      "loss": 0.0305,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -10.470837593078613,
      "rewards/margins": 11.618741035461426,
      "rewards/rejected": -22.089576721191406,
      "step": 385
    },
    {
      "epoch": 0.07002109609946587,
      "grad_norm": 2.875,
      "learning_rate": 1.643396902630828e-05,
      "logits/chosen": -1.4584577083587646,
      "logits/rejected": -1.4607679843902588,
      "logps/chosen": -317.8088073730469,
      "logps/rejected": -451.45440673828125,
      "loss": 0.0322,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -10.205190658569336,
      "rewards/margins": 11.55008316040039,
      "rewards/rejected": -21.75527572631836,
      "step": 390
    },
    {
      "epoch": 0.07091880245971542,
      "grad_norm": 6.96875,
      "learning_rate": 1.6433578937708046e-05,
      "logits/chosen": -1.4126781225204468,
      "logits/rejected": -1.4239190816879272,
      "logps/chosen": -322.39801025390625,
      "logps/rejected": -458.4278259277344,
      "loss": 0.0256,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.876721382141113,
      "rewards/margins": 11.59015941619873,
      "rewards/rejected": -21.466880798339844,
      "step": 395
    },
    {
      "epoch": 0.07181650881996499,
      "grad_norm": 0.02001953125,
      "learning_rate": 1.64331833864258e-05,
      "logits/chosen": -1.4148705005645752,
      "logits/rejected": -1.4251186847686768,
      "logps/chosen": -316.4896545410156,
      "logps/rejected": -450.53240966796875,
      "loss": 0.0164,
      "rewards/accuracies": 0.9906250238418579,
      "rewards/chosen": -9.625123023986816,
      "rewards/margins": 11.543768882751465,
      "rewards/rejected": -21.168895721435547,
      "step": 400
    },
    {
      "epoch": 0.07271421518021455,
      "grad_norm": 0.890625,
      "learning_rate": 1.643278237281267e-05,
      "logits/chosen": -1.421555757522583,
      "logits/rejected": -1.4265415668487549,
      "logps/chosen": -308.0295104980469,
      "logps/rejected": -445.5311584472656,
      "loss": 0.0249,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.17651653289795,
      "rewards/margins": 11.756936073303223,
      "rewards/rejected": -20.933452606201172,
      "step": 405
    },
    {
      "epoch": 0.07361192154046411,
      "grad_norm": 2.140625,
      "learning_rate": 1.6432375897224637e-05,
      "logits/chosen": -1.3315099477767944,
      "logits/rejected": -1.3350989818572998,
      "logps/chosen": -315.9861755371094,
      "logps/rejected": -451.1835021972656,
      "loss": 0.044,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.11551284790039,
      "rewards/margins": 11.56936264038086,
      "rewards/rejected": -20.68487548828125,
      "step": 410
    },
    {
      "epoch": 0.07450962790071368,
      "grad_norm": 0.94140625,
      "learning_rate": 1.6431963960022524e-05,
      "logits/chosen": -1.2902719974517822,
      "logits/rejected": -1.2910665273666382,
      "logps/chosen": -309.9100341796875,
      "logps/rejected": -442.4892578125,
      "loss": 0.0278,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -9.109766006469727,
      "rewards/margins": 11.337265968322754,
      "rewards/rejected": -20.447031021118164,
      "step": 415
    },
    {
      "epoch": 0.07540733426096324,
      "grad_norm": 4.28125,
      "learning_rate": 1.643154656157201e-05,
      "logits/chosen": -1.2414597272872925,
      "logits/rejected": -1.2512853145599365,
      "logps/chosen": -301.22174072265625,
      "logps/rejected": -416.2430725097656,
      "loss": 0.0651,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -9.233153343200684,
      "rewards/margins": 9.558819770812988,
      "rewards/rejected": -18.79197120666504,
      "step": 420
    },
    {
      "epoch": 0.07630504062121281,
      "grad_norm": 2.984375,
      "learning_rate": 1.6431123702243618e-05,
      "logits/chosen": -1.2505871057510376,
      "logits/rejected": -1.2604036331176758,
      "logps/chosen": -319.97369384765625,
      "logps/rejected": -420.20184326171875,
      "loss": 0.0256,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -9.754773139953613,
      "rewards/margins": 7.977179527282715,
      "rewards/rejected": -17.731952667236328,
      "step": 425
    },
    {
      "epoch": 0.07720274698146236,
      "grad_norm": 2.75,
      "learning_rate": 1.6430695382412714e-05,
      "logits/chosen": -1.2662450075149536,
      "logits/rejected": -1.2877540588378906,
      "logps/chosen": -322.77532958984375,
      "logps/rejected": -426.742919921875,
      "loss": 0.0443,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -10.8510103225708,
      "rewards/margins": 8.357492446899414,
      "rewards/rejected": -19.208499908447266,
      "step": 430
    },
    {
      "epoch": 0.07810045334171192,
      "grad_norm": 0.039794921875,
      "learning_rate": 1.6430261602459523e-05,
      "logits/chosen": -1.291669487953186,
      "logits/rejected": -1.3137457370758057,
      "logps/chosen": -338.3268127441406,
      "logps/rejected": -457.73077392578125,
      "loss": 0.0177,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -12.022209167480469,
      "rewards/margins": 10.057284355163574,
      "rewards/rejected": -22.07949447631836,
      "step": 435
    },
    {
      "epoch": 0.07899815970196149,
      "grad_norm": 7.125,
      "learning_rate": 1.6429822362769104e-05,
      "logits/chosen": -1.2740453481674194,
      "logits/rejected": -1.2928683757781982,
      "logps/chosen": -351.74859619140625,
      "logps/rejected": -467.54345703125,
      "loss": 0.0661,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.225227355957031,
      "rewards/margins": 9.72153377532959,
      "rewards/rejected": -22.946762084960938,
      "step": 440
    },
    {
      "epoch": 0.07989586606221105,
      "grad_norm": 5.0625,
      "learning_rate": 1.642937766373137e-05,
      "logits/chosen": -1.2580888271331787,
      "logits/rejected": -1.2826154232025146,
      "logps/chosen": -356.80303955078125,
      "logps/rejected": -462.6808166503906,
      "loss": 0.05,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -13.879257202148438,
      "rewards/margins": 8.64616584777832,
      "rewards/rejected": -22.525421142578125,
      "step": 445
    },
    {
      "epoch": 0.08079357242246062,
      "grad_norm": 3.53125,
      "learning_rate": 1.6428927505741077e-05,
      "logits/chosen": -1.3274773359298706,
      "logits/rejected": -1.3538029193878174,
      "logps/chosen": -351.6380920410156,
      "logps/rejected": -460.96075439453125,
      "loss": 0.0209,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -12.758635520935059,
      "rewards/margins": 9.032510757446289,
      "rewards/rejected": -21.79114532470703,
      "step": 450
    },
    {
      "epoch": 0.08169127878271018,
      "grad_norm": 2.140625,
      "learning_rate": 1.642847188919783e-05,
      "logits/chosen": -1.3372552394866943,
      "logits/rejected": -1.3623110055923462,
      "logps/chosen": -347.4330139160156,
      "logps/rejected": -465.4501037597656,
      "loss": 0.0277,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -12.52314281463623,
      "rewards/margins": 9.909950256347656,
      "rewards/rejected": -22.43309211730957,
      "step": 455
    },
    {
      "epoch": 0.08258898514295973,
      "grad_norm": 5.5,
      "learning_rate": 1.6428010814506082e-05,
      "logits/chosen": -1.3123576641082764,
      "logits/rejected": -1.339634895324707,
      "logps/chosen": -338.0999450683594,
      "logps/rejected": -456.74310302734375,
      "loss": 0.0419,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -13.015867233276367,
      "rewards/margins": 9.875136375427246,
      "rewards/rejected": -22.891002655029297,
      "step": 460
    },
    {
      "epoch": 0.0834866915032093,
      "grad_norm": 1.703125,
      "learning_rate": 1.6427544282075123e-05,
      "logits/chosen": -1.3849332332611084,
      "logits/rejected": -1.4038164615631104,
      "logps/chosen": -353.74908447265625,
      "logps/rejected": -478.51287841796875,
      "loss": 0.0462,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -13.985809326171875,
      "rewards/margins": 10.343815803527832,
      "rewards/rejected": -24.32962417602539,
      "step": 465
    },
    {
      "epoch": 0.08438439786345886,
      "grad_norm": 0.1669921875,
      "learning_rate": 1.642707229231909e-05,
      "logits/chosen": -1.3579334020614624,
      "logits/rejected": -1.3771896362304688,
      "logps/chosen": -369.145751953125,
      "logps/rejected": -490.97100830078125,
      "loss": 0.0146,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -15.570086479187012,
      "rewards/margins": 9.97675895690918,
      "rewards/rejected": -25.546846389770508,
      "step": 470
    },
    {
      "epoch": 0.08528210422370842,
      "grad_norm": 1.8203125,
      "learning_rate": 1.6426594845656973e-05,
      "logits/chosen": -1.355943202972412,
      "logits/rejected": -1.3650095462799072,
      "logps/chosen": -376.259521484375,
      "logps/rejected": -499.44970703125,
      "loss": 0.0148,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -15.661959648132324,
      "rewards/margins": 10.384978294372559,
      "rewards/rejected": -26.04693603515625,
      "step": 475
    },
    {
      "epoch": 0.08617981058395799,
      "grad_norm": 2.265625,
      "learning_rate": 1.642611194251259e-05,
      "logits/chosen": -1.355452299118042,
      "logits/rejected": -1.3569139242172241,
      "logps/chosen": -376.9379577636719,
      "logps/rejected": -506.89666748046875,
      "loss": 0.0385,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -15.602932929992676,
      "rewards/margins": 10.967208862304688,
      "rewards/rejected": -26.570140838623047,
      "step": 480
    },
    {
      "epoch": 0.08707751694420755,
      "grad_norm": 1.71875,
      "learning_rate": 1.642562358331462e-05,
      "logits/chosen": -1.3472042083740234,
      "logits/rejected": -1.3535155057907104,
      "logps/chosen": -362.6130676269531,
      "logps/rejected": -499.9092712402344,
      "loss": 0.0131,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -14.739013671875,
      "rewards/margins": 11.621049880981445,
      "rewards/rejected": -26.360065460205078,
      "step": 485
    },
    {
      "epoch": 0.08797522330445712,
      "grad_norm": 1.1171875,
      "learning_rate": 1.6425129768496577e-05,
      "logits/chosen": -1.3245633840560913,
      "logits/rejected": -1.3288484811782837,
      "logps/chosen": -364.94903564453125,
      "logps/rejected": -503.7896423339844,
      "loss": 0.0225,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -14.195643424987793,
      "rewards/margins": 11.918843269348145,
      "rewards/rejected": -26.114486694335938,
      "step": 490
    },
    {
      "epoch": 0.08887292966470667,
      "grad_norm": 3.65625,
      "learning_rate": 1.6424630498496813e-05,
      "logits/chosen": -1.3164643049240112,
      "logits/rejected": -1.3220335245132446,
      "logps/chosen": -365.0807189941406,
      "logps/rejected": -510.30340576171875,
      "loss": 0.0186,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -14.422874450683594,
      "rewards/margins": 12.447381973266602,
      "rewards/rejected": -26.870258331298828,
      "step": 495
    },
    {
      "epoch": 0.08977063602495623,
      "grad_norm": 2.203125,
      "learning_rate": 1.6424125773758535e-05,
      "logits/chosen": -1.418001413345337,
      "logits/rejected": -1.4166367053985596,
      "logps/chosen": -361.4501037597656,
      "logps/rejected": -499.55902099609375,
      "loss": 0.0316,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -14.319913864135742,
      "rewards/margins": 11.92530632019043,
      "rewards/rejected": -26.245220184326172,
      "step": 500
    },
    {
      "epoch": 0.08977063602495623,
      "eval_logits/chosen": -1.3433603048324585,
      "eval_logits/rejected": -1.36442232131958,
      "eval_logps/chosen": -369.4107666015625,
      "eval_logps/rejected": -511.26239013671875,
      "eval_loss": 0.014039273373782635,
      "eval_rewards/accuracies": 0.9900000095367432,
      "eval_rewards/chosen": -13.992281913757324,
      "eval_rewards/margins": 12.298800468444824,
      "eval_rewards/rejected": -26.29108238220215,
      "eval_runtime": 10.3033,
      "eval_samples_per_second": 19.411,
      "eval_steps_per_second": 19.411,
      "step": 500
    }
  ],
  "logging_steps": 5,
  "max_steps": 16707,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}