{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1833,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016366612111292964,
      "grad_norm": NaN,
      "learning_rate": 1.6304347826086957e-06,
      "logits/chosen": -1.2515310049057007,
      "logits/rejected": -0.8573959469795227,
      "logps/chosen": -455.1468811035156,
      "logps/rejected": -387.0008239746094,
      "loss": 0.6923,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0002114105736836791,
      "rewards/margins": 0.00180077797267586,
      "rewards/rejected": -0.0015893673989921808,
      "step": 10
    },
    {
      "epoch": 0.03273322422258593,
      "grad_norm": 10.154256820678711,
      "learning_rate": 3.2608695652173914e-06,
      "logits/chosen": -1.145971417427063,
      "logits/rejected": -1.082183837890625,
      "logps/chosen": -446.249755859375,
      "logps/rejected": -426.87237548828125,
      "loss": 0.6942,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0005732345744036138,
      "rewards/margins": -0.002082281280308962,
      "rewards/rejected": 0.0015090465312823653,
      "step": 20
    },
    {
      "epoch": 0.049099836333878884,
      "grad_norm": 8.393162727355957,
      "learning_rate": 5.4347826086956525e-06,
      "logits/chosen": -0.9614235162734985,
      "logits/rejected": -0.8527517318725586,
      "logps/chosen": -460.367431640625,
      "logps/rejected": -433.61285400390625,
      "loss": 0.6934,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0062791816890239716,
      "rewards/margins": -0.00045429245801642537,
      "rewards/rejected": 0.006733474787324667,
      "step": 30
    },
    {
      "epoch": 0.06546644844517185,
      "grad_norm": 9.398829460144043,
      "learning_rate": 8.15217391304348e-06,
      "logits/chosen": -1.171631097793579,
      "logits/rejected": -1.0616101026535034,
      "logps/chosen": -421.583984375,
      "logps/rejected": -424.787109375,
      "loss": 0.6902,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.020799484103918076,
      "rewards/margins": 0.006221642717719078,
      "rewards/rejected": 0.014577841386198997,
      "step": 40
    },
    {
      "epoch": 0.08183306055646482,
      "grad_norm": 9.43005657196045,
      "learning_rate": 1.0869565217391305e-05,
      "logits/chosen": -1.2450587749481201,
      "logits/rejected": -1.1417832374572754,
      "logps/chosen": -476.83538818359375,
      "logps/rejected": -444.6556091308594,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.042164258658885956,
      "rewards/margins": 0.010725794360041618,
      "rewards/rejected": 0.03143846243619919,
      "step": 50
    },
    {
      "epoch": 0.09819967266775777,
      "grad_norm": 8.825221061706543,
      "learning_rate": 1.331521739130435e-05,
      "logits/chosen": -1.3223823308944702,
      "logits/rejected": NaN,
      "logps/chosen": -383.4811706542969,
      "logps/rejected": -454.43780517578125,
      "loss": 0.6965,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.03668512776494026,
      "rewards/margins": -0.003995636478066444,
      "rewards/rejected": 0.04068076238036156,
      "step": 60
    },
    {
      "epoch": 0.11456628477905073,
      "grad_norm": 9.5093994140625,
      "learning_rate": 1.6032608695652173e-05,
      "logits/chosen": -1.2538468837738037,
      "logits/rejected": -1.2771422863006592,
      "logps/chosen": -468.79400634765625,
      "logps/rejected": -402.565673828125,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04952714219689369,
      "rewards/margins": 0.01824185810983181,
      "rewards/rejected": 0.03128528222441673,
      "step": 70
    },
    {
      "epoch": 0.1309328968903437,
      "grad_norm": 7.945922374725342,
      "learning_rate": 1.8750000000000002e-05,
      "logits/chosen": -1.386731743812561,
      "logits/rejected": -0.8966825604438782,
      "logps/chosen": -446.54736328125,
      "logps/rejected": -430.45025634765625,
      "loss": 0.6719,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.06767664849758148,
      "rewards/margins": 0.048766594380140305,
      "rewards/rejected": 0.01891004480421543,
      "step": 80
    },
    {
      "epoch": 0.14729950900163666,
      "grad_norm": 8.496461868286133,
      "learning_rate": 2.1467391304347828e-05,
      "logits/chosen": -1.3730494976043701,
      "logits/rejected": -1.1826181411743164,
      "logps/chosen": -413.87835693359375,
      "logps/rejected": -385.1138916015625,
      "loss": 0.69,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.07122664153575897,
      "rewards/margins": 0.014941570349037647,
      "rewards/rejected": 0.0562850646674633,
      "step": 90
    },
    {
      "epoch": 0.16366612111292964,
      "grad_norm": 10.744088172912598,
      "learning_rate": 2.4184782608695653e-05,
      "logits/chosen": -1.2254825830459595,
      "logits/rejected": -1.1136434078216553,
      "logps/chosen": -495.34295654296875,
      "logps/rejected": -435.0003967285156,
      "loss": 0.6618,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.1682208627462387,
      "rewards/margins": 0.08198796212673187,
      "rewards/rejected": 0.08623288571834564,
      "step": 100
    },
    {
      "epoch": 0.18003273322422259,
      "grad_norm": 11.016289710998535,
      "learning_rate": 2.6902173913043476e-05,
      "logits/chosen": -1.3138881921768188,
      "logits/rejected": -1.0289314985275269,
      "logps/chosen": -476.26153564453125,
      "logps/rejected": -426.837646484375,
      "loss": 0.6355,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.21744687855243683,
      "rewards/margins": 0.15108919143676758,
      "rewards/rejected": 0.06635768711566925,
      "step": 110
    },
    {
      "epoch": 0.19639934533551553,
      "grad_norm": 11.619248390197754,
      "learning_rate": 2.9619565217391305e-05,
      "logits/chosen": -1.2751747369766235,
      "logits/rejected": -1.1858090162277222,
      "logps/chosen": -444.4823303222656,
      "logps/rejected": -404.58123779296875,
      "loss": 0.6716,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.2203199863433838,
      "rewards/margins": 0.06618332862854004,
      "rewards/rejected": 0.15413665771484375,
      "step": 120
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 8.900009155273438,
      "learning_rate": 3.233695652173913e-05,
      "logits/chosen": -1.2365785837173462,
      "logits/rejected": -1.2031922340393066,
      "logps/chosen": -477.8089294433594,
      "logps/rejected": -456.80120849609375,
      "loss": 0.6794,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.20705166459083557,
      "rewards/margins": 0.060810189694166183,
      "rewards/rejected": 0.1462414562702179,
      "step": 130
    },
    {
      "epoch": 0.22913256955810146,
      "grad_norm": 8.928816795349121,
      "learning_rate": 3.505434782608696e-05,
      "logits/chosen": -1.2421447038650513,
      "logits/rejected": -1.136252522468567,
      "logps/chosen": -547.4909057617188,
      "logps/rejected": -460.38275146484375,
      "loss": 0.6047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.21343636512756348,
      "rewards/margins": 0.24689535796642303,
      "rewards/rejected": -0.033459004014730453,
      "step": 140
    },
    {
      "epoch": 0.24549918166939444,
      "grad_norm": 11.977150917053223,
      "learning_rate": 3.777173913043478e-05,
      "logits/chosen": -1.338332176208496,
      "logits/rejected": -1.1015567779541016,
      "logps/chosen": -432.2693786621094,
      "logps/rejected": -423.0946350097656,
      "loss": 0.6946,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.15119442343711853,
      "rewards/margins": 0.07870660722255707,
      "rewards/rejected": 0.07248781621456146,
      "step": 150
    },
    {
      "epoch": 0.2618657937806874,
      "grad_norm": 13.040871620178223,
      "learning_rate": 4.0489130434782615e-05,
      "logits/chosen": -1.0111101865768433,
      "logits/rejected": NaN,
      "logps/chosen": -472.6917419433594,
      "logps/rejected": -426.6333923339844,
      "loss": 0.6598,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.24765905737876892,
      "rewards/margins": 0.13647916913032532,
      "rewards/rejected": 0.11117993295192719,
      "step": 160
    },
    {
      "epoch": 0.27823240589198034,
      "grad_norm": 8.028325080871582,
      "learning_rate": 4.3206521739130434e-05,
      "logits/chosen": -0.9152072668075562,
      "logits/rejected": -0.8340185284614563,
      "logps/chosen": -438.38555908203125,
      "logps/rejected": -392.93426513671875,
      "loss": 0.6272,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.3285876512527466,
      "rewards/margins": 0.20840699970722198,
      "rewards/rejected": 0.1201806515455246,
      "step": 170
    },
    {
      "epoch": 0.2945990180032733,
      "grad_norm": 9.436712265014648,
      "learning_rate": 4.592391304347826e-05,
      "logits/chosen": -1.0703661441802979,
      "logits/rejected": -1.0127884149551392,
      "logps/chosen": -449.83905029296875,
      "logps/rejected": -437.92822265625,
      "loss": 0.6684,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.34186336398124695,
      "rewards/margins": 0.14258861541748047,
      "rewards/rejected": 0.19927474856376648,
      "step": 180
    },
    {
      "epoch": 0.3109656301145663,
      "grad_norm": 8.914007186889648,
      "learning_rate": 4.8641304347826086e-05,
      "logits/chosen": -0.8762575387954712,
      "logits/rejected": -0.5631248354911804,
      "logps/chosen": -473.806396484375,
      "logps/rejected": -443.6448669433594,
      "loss": 0.6243,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.3772050440311432,
      "rewards/margins": 0.24287982285022736,
      "rewards/rejected": 0.13432522118091583,
      "step": 190
    },
    {
      "epoch": 0.32733224222585927,
      "grad_norm": 14.76502513885498,
      "learning_rate": 4.98483929654336e-05,
      "logits/chosen": NaN,
      "logits/rejected": -0.9821416139602661,
      "logps/chosen": -447.65386962890625,
      "logps/rejected": -400.1768493652344,
      "loss": 0.6805,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.2871859073638916,
      "rewards/margins": 0.17125040292739868,
      "rewards/rejected": 0.11593550443649292,
      "step": 200
    },
    {
      "epoch": 0.3436988543371522,
      "grad_norm": 11.80213451385498,
      "learning_rate": 4.954517889630079e-05,
      "logits/chosen": -1.2535641193389893,
      "logits/rejected": -1.0489479303359985,
      "logps/chosen": -457.010986328125,
      "logps/rejected": -432.8558044433594,
      "loss": 0.6769,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.24471895396709442,
      "rewards/margins": 0.16205154359340668,
      "rewards/rejected": 0.08266742527484894,
      "step": 210
    },
    {
      "epoch": 0.36006546644844517,
      "grad_norm": 7.534430980682373,
      "learning_rate": 4.924196482716799e-05,
      "logits/chosen": -1.5050650835037231,
      "logits/rejected": -1.5133517980575562,
      "logps/chosen": -445.3924865722656,
      "logps/rejected": -420.99114990234375,
      "loss": 0.7003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.18017885088920593,
      "rewards/margins": 0.1387586146593094,
      "rewards/rejected": 0.04142025485634804,
      "step": 220
    },
    {
      "epoch": 0.37643207855973815,
      "grad_norm": 7.207632541656494,
      "learning_rate": 4.893875075803517e-05,
      "logits/chosen": -1.3496973514556885,
      "logits/rejected": -1.2030962705612183,
      "logps/chosen": -436.5039978027344,
      "logps/rejected": -396.91534423828125,
      "loss": 0.6192,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.19479091465473175,
      "rewards/margins": 0.353401243686676,
      "rewards/rejected": -0.15861034393310547,
      "step": 230
    },
    {
      "epoch": 0.39279869067103107,
      "grad_norm": 9.134820938110352,
      "learning_rate": 4.8665858095815645e-05,
      "logits/chosen": -1.2978178262710571,
      "logits/rejected": -1.1864020824432373,
      "logps/chosen": -486.7569885253906,
      "logps/rejected": -457.94586181640625,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.29108190536499023,
      "rewards/margins": 0.15347807109355927,
      "rewards/rejected": 0.13760384917259216,
      "step": 240
    },
    {
      "epoch": 0.40916530278232405,
      "grad_norm": 10.825005531311035,
      "learning_rate": 4.836264402668284e-05,
      "logits/chosen": -1.3271738290786743,
      "logits/rejected": -1.030890703201294,
      "logps/chosen": -421.89349365234375,
      "logps/rejected": -400.1674499511719,
      "loss": 0.6536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.42532652616500854,
      "rewards/margins": 0.34351587295532227,
      "rewards/rejected": 0.08181065320968628,
      "step": 250
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 9.450189590454102,
      "learning_rate": 4.8059429957550034e-05,
      "logits/chosen": -1.0212231874465942,
      "logits/rejected": -1.0058345794677734,
      "logps/chosen": -422.1316833496094,
      "logps/rejected": -390.0425109863281,
      "loss": 0.6384,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.490867555141449,
      "rewards/margins": 0.35965341329574585,
      "rewards/rejected": 0.13121409714221954,
      "step": 260
    },
    {
      "epoch": 0.44189852700491,
      "grad_norm": 8.231124877929688,
      "learning_rate": 4.7756215888417225e-05,
      "logits/chosen": -0.9735445976257324,
      "logits/rejected": -0.937743067741394,
      "logps/chosen": -552.4168701171875,
      "logps/rejected": -501.0535583496094,
      "loss": 0.6694,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.4536367952823639,
      "rewards/margins": 0.19916944205760956,
      "rewards/rejected": 0.2544674277305603,
      "step": 270
    },
    {
      "epoch": 0.4582651391162029,
      "grad_norm": 9.207930564880371,
      "learning_rate": 4.7453001819284416e-05,
      "logits/chosen": -0.9774928092956543,
      "logits/rejected": -0.8485246896743774,
      "logps/chosen": -444.54827880859375,
      "logps/rejected": -419.5541076660156,
      "loss": 0.6607,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.3690536618232727,
      "rewards/margins": 0.22144463658332825,
      "rewards/rejected": 0.14760902523994446,
      "step": 280
    },
    {
      "epoch": 0.4746317512274959,
      "grad_norm": 6.876308441162109,
      "learning_rate": 4.714978775015161e-05,
      "logits/chosen": -1.2095867395401,
      "logits/rejected": -0.8773800730705261,
      "logps/chosen": -437.869384765625,
      "logps/rejected": -419.9366149902344,
      "loss": 0.5867,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.5094749331474304,
      "rewards/margins": 0.5250319242477417,
      "rewards/rejected": -0.015556998550891876,
      "step": 290
    },
    {
      "epoch": 0.4909983633387889,
      "grad_norm": 12.918950080871582,
      "learning_rate": 4.68465736810188e-05,
      "logits/chosen": -0.9076827764511108,
      "logits/rejected": -0.7181829214096069,
      "logps/chosen": -469.4864196777344,
      "logps/rejected": -420.38885498046875,
      "loss": 0.6833,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.5444287657737732,
      "rewards/margins": 0.2743019461631775,
      "rewards/rejected": 0.2701267600059509,
      "step": 300
    },
    {
      "epoch": 0.5073649754500819,
      "grad_norm": 4.9246110916137695,
      "learning_rate": 4.6543359611885996e-05,
      "logits/chosen": -1.214247226715088,
      "logits/rejected": -0.9910456538200378,
      "logps/chosen": -385.4250183105469,
      "logps/rejected": -390.26043701171875,
      "loss": 0.6419,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.4199371337890625,
      "rewards/margins": 0.34462255239486694,
      "rewards/rejected": 0.07531457394361496,
      "step": 310
    },
    {
      "epoch": 0.5237315875613748,
      "grad_norm": 8.829249382019043,
      "learning_rate": 4.624014554275319e-05,
      "logits/chosen": -1.0396336317062378,
      "logits/rejected": -0.9439867734909058,
      "logps/chosen": -452.1678161621094,
      "logps/rejected": -379.70745849609375,
      "loss": 0.6396,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.472791850566864,
      "rewards/margins": 0.23509594798088074,
      "rewards/rejected": 0.23769588768482208,
      "step": 320
    },
    {
      "epoch": 0.5400981996726678,
      "grad_norm": 13.249287605285645,
      "learning_rate": 4.593693147362038e-05,
      "logits/chosen": -0.927478015422821,
      "logits/rejected": -0.7090943455696106,
      "logps/chosen": -488.009765625,
      "logps/rejected": -473.47833251953125,
      "loss": 0.6833,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.4911763668060303,
      "rewards/margins": 0.25351986289024353,
      "rewards/rejected": 0.23765654861927032,
      "step": 330
    },
    {
      "epoch": 0.5564648117839607,
      "grad_norm": 9.580172538757324,
      "learning_rate": 4.566403881140085e-05,
      "logits/chosen": -0.9790302515029907,
      "logits/rejected": -0.8282285928726196,
      "logps/chosen": -481.6971740722656,
      "logps/rejected": -441.3619689941406,
      "loss": 0.6922,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.4195849299430847,
      "rewards/margins": 0.210377499461174,
      "rewards/rejected": 0.20920750498771667,
      "step": 340
    },
    {
      "epoch": 0.5728314238952537,
      "grad_norm": 6.872241973876953,
      "learning_rate": 4.536082474226804e-05,
      "logits/chosen": -0.9942235946655273,
      "logits/rejected": -0.824662983417511,
      "logps/chosen": -437.9869079589844,
      "logps/rejected": -417.3473205566406,
      "loss": 0.6071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.49881643056869507,
      "rewards/margins": 0.41778793931007385,
      "rewards/rejected": 0.08102855831384659,
      "step": 350
    },
    {
      "epoch": 0.5891980360065466,
      "grad_norm": 7.748345375061035,
      "learning_rate": 4.5057610673135234e-05,
      "logits/chosen": -1.0704625844955444,
      "logits/rejected": -0.7879603505134583,
      "logps/chosen": -428.53021240234375,
      "logps/rejected": -435.9026794433594,
      "loss": 0.6084,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.4484317898750305,
      "rewards/margins": 0.4125775396823883,
      "rewards/rejected": 0.035854239016771317,
      "step": 360
    },
    {
      "epoch": 0.6055646481178396,
      "grad_norm": 7.428864479064941,
      "learning_rate": 4.475439660400243e-05,
      "logits/chosen": -1.098026990890503,
      "logits/rejected": -0.9232645034790039,
      "logps/chosen": -441.7071228027344,
      "logps/rejected": -402.95770263671875,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.37514036893844604,
      "rewards/margins": 0.2525361478328705,
      "rewards/rejected": 0.12260414659976959,
      "step": 370
    },
    {
      "epoch": 0.6219312602291326,
      "grad_norm": 9.591557502746582,
      "learning_rate": 4.445118253486962e-05,
      "logits/chosen": -1.2409312725067139,
      "logits/rejected": -0.8442489504814148,
      "logps/chosen": -466.13311767578125,
      "logps/rejected": -420.9102478027344,
      "loss": 0.7684,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.2286566197872162,
      "rewards/margins": 0.11457403004169464,
      "rewards/rejected": 0.11408257484436035,
      "step": 380
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 7.025293350219727,
      "learning_rate": 4.414796846573681e-05,
      "logits/chosen": -1.1844158172607422,
      "logits/rejected": -1.1069843769073486,
      "logps/chosen": -428.5450134277344,
      "logps/rejected": -371.85614013671875,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.2637328803539276,
      "rewards/margins": 0.10889965295791626,
      "rewards/rejected": 0.15483322739601135,
      "step": 390
    },
    {
      "epoch": 0.6546644844517185,
      "grad_norm": 19.2535343170166,
      "learning_rate": 4.3844754396604006e-05,
      "logits/chosen": -1.1141570806503296,
      "logits/rejected": -0.8798454999923706,
      "logps/chosen": -493.08197021484375,
      "logps/rejected": -465.0943298339844,
      "loss": 0.6545,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.4174181818962097,
      "rewards/margins": 0.25259822607040405,
      "rewards/rejected": 0.16481998562812805,
      "step": 400
    },
    {
      "epoch": 0.6710310965630114,
      "grad_norm": 8.808697700500488,
      "learning_rate": 4.35415403274712e-05,
      "logits/chosen": -0.9878439903259277,
      "logits/rejected": -0.7299278974533081,
      "logps/chosen": -523.9139404296875,
      "logps/rejected": -482.0597229003906,
      "loss": 0.552,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.4945390820503235,
      "rewards/margins": 0.6175243854522705,
      "rewards/rejected": -0.1229853481054306,
      "step": 410
    },
    {
      "epoch": 0.6873977086743044,
      "grad_norm": 9.99456787109375,
      "learning_rate": 4.323832625833839e-05,
      "logits/chosen": -1.0902565717697144,
      "logits/rejected": -0.9221500158309937,
      "logps/chosen": -443.0076599121094,
      "logps/rejected": -375.8013610839844,
      "loss": 0.5684,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.46177297830581665,
      "rewards/margins": 0.509793221950531,
      "rewards/rejected": -0.048020295798778534,
      "step": 420
    },
    {
      "epoch": 0.7037643207855974,
      "grad_norm": 5.767345428466797,
      "learning_rate": 4.293511218920558e-05,
      "logits/chosen": -1.0450439453125,
      "logits/rejected": -0.7902520298957825,
      "logps/chosen": -458.2330017089844,
      "logps/rejected": -382.6787109375,
      "loss": 0.6126,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.5793684124946594,
      "rewards/margins": 0.35584205389022827,
      "rewards/rejected": 0.22352632880210876,
      "step": 430
    },
    {
      "epoch": 0.7201309328968903,
      "grad_norm": 11.51710033416748,
      "learning_rate": 4.263189812007277e-05,
      "logits/chosen": -0.8710979223251343,
      "logits/rejected": -0.783151388168335,
      "logps/chosen": -451.32122802734375,
      "logps/rejected": -416.0005798339844,
      "loss": 0.6751,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.5284687280654907,
      "rewards/margins": 0.2338237315416336,
      "rewards/rejected": 0.2946449816226959,
      "step": 440
    },
    {
      "epoch": 0.7364975450081833,
      "grad_norm": 8.408599853515625,
      "learning_rate": 4.232868405093997e-05,
      "logits/chosen": -0.9611997604370117,
      "logits/rejected": -0.8005222082138062,
      "logps/chosen": -390.06817626953125,
      "logps/rejected": -376.57183837890625,
      "loss": 0.723,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.35626164078712463,
      "rewards/margins": 0.07882154732942581,
      "rewards/rejected": 0.27744007110595703,
      "step": 450
    },
    {
      "epoch": 0.7528641571194763,
      "grad_norm": 7.085683822631836,
      "learning_rate": 4.202546998180716e-05,
      "logits/chosen": -1.181065559387207,
      "logits/rejected": -0.9188621640205383,
      "logps/chosen": -446.1695861816406,
      "logps/rejected": -461.0537109375,
      "loss": 0.6376,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.6099992990493774,
      "rewards/margins": 0.40477481484413147,
      "rewards/rejected": 0.20522448420524597,
      "step": 460
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 7.778029918670654,
      "learning_rate": 4.172225591267435e-05,
      "logits/chosen": -1.1612136363983154,
      "logits/rejected": -0.7909379601478577,
      "logps/chosen": -415.356201171875,
      "logps/rejected": -405.0163879394531,
      "loss": 0.639,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.5554541349411011,
      "rewards/margins": 0.49387890100479126,
      "rewards/rejected": 0.06157524511218071,
      "step": 470
    },
    {
      "epoch": 0.7855973813420621,
      "grad_norm": 9.144447326660156,
      "learning_rate": 4.141904184354154e-05,
      "logits/chosen": -1.1227318048477173,
      "logits/rejected": -0.8980533480644226,
      "logps/chosen": -470.1067810058594,
      "logps/rejected": -385.7353210449219,
      "loss": 0.6301,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.5011923909187317,
      "rewards/margins": 0.36913150548934937,
      "rewards/rejected": 0.1320609152317047,
      "step": 480
    },
    {
      "epoch": 0.8019639934533551,
      "grad_norm": 9.528294563293457,
      "learning_rate": 4.111582777440873e-05,
      "logits/chosen": -1.4198505878448486,
      "logits/rejected": -1.1528406143188477,
      "logps/chosen": -430.28668212890625,
      "logps/rejected": -385.2845764160156,
      "loss": 0.634,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.6369161605834961,
      "rewards/margins": 0.4485607147216797,
      "rewards/rejected": 0.18835540115833282,
      "step": 490
    },
    {
      "epoch": 0.8183306055646481,
      "grad_norm": 6.964690208435059,
      "learning_rate": 4.0812613705275924e-05,
      "logits/chosen": -1.1329933404922485,
      "logits/rejected": -1.066542625427246,
      "logps/chosen": -423.4967346191406,
      "logps/rejected": -376.92803955078125,
      "loss": 0.5904,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.6795145273208618,
      "rewards/margins": 0.3617789149284363,
      "rewards/rejected": 0.3177356719970703,
      "step": 500
    },
    {
      "epoch": 0.8346972176759411,
      "grad_norm": 11.78022289276123,
      "learning_rate": 4.050939963614312e-05,
      "logits/chosen": -1.1565020084381104,
      "logits/rejected": -1.064300298690796,
      "logps/chosen": -465.0497131347656,
      "logps/rejected": -438.4883728027344,
      "loss": 0.6384,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.6123272180557251,
      "rewards/margins": 0.3706541657447815,
      "rewards/rejected": 0.241673082113266,
      "step": 510
    },
    {
      "epoch": 0.851063829787234,
      "grad_norm": 6.378419876098633,
      "learning_rate": 4.020618556701031e-05,
      "logits/chosen": -1.0936065912246704,
      "logits/rejected": -1.157414436340332,
      "logps/chosen": -431.05657958984375,
      "logps/rejected": -380.1803283691406,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.514751672744751,
      "rewards/margins": 0.2915990948677063,
      "rewards/rejected": 0.22315259277820587,
      "step": 520
    },
    {
      "epoch": 0.867430441898527,
      "grad_norm": 6.811694145202637,
      "learning_rate": 3.9902971497877504e-05,
      "logits/chosen": -1.0564637184143066,
      "logits/rejected": -0.8537761569023132,
      "logps/chosen": -400.8801574707031,
      "logps/rejected": -400.9678039550781,
      "loss": 0.6036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5905786156654358,
      "rewards/margins": 0.49240899085998535,
      "rewards/rejected": 0.09816964715719223,
      "step": 530
    },
    {
      "epoch": 0.88379705400982,
      "grad_norm": 8.516433715820312,
      "learning_rate": 3.9599757428744695e-05,
      "logits/chosen": -1.0425868034362793,
      "logits/rejected": -1.0106391906738281,
      "logps/chosen": -494.23681640625,
      "logps/rejected": -451.3269958496094,
      "loss": 0.7295,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.480445921421051,
      "rewards/margins": 0.12438355386257172,
      "rewards/rejected": 0.3560623824596405,
      "step": 540
    },
    {
      "epoch": 0.900163666121113,
      "grad_norm": 9.584715843200684,
      "learning_rate": 3.9296543359611886e-05,
      "logits/chosen": -1.1160447597503662,
      "logits/rejected": -0.898239016532898,
      "logps/chosen": -471.17718505859375,
      "logps/rejected": -404.9838562011719,
      "loss": 0.5973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4906229078769684,
      "rewards/margins": 0.3674588203430176,
      "rewards/rejected": 0.1231641173362732,
      "step": 550
    },
    {
      "epoch": 0.9165302782324058,
      "grad_norm": 9.050358772277832,
      "learning_rate": 3.899332929047908e-05,
      "logits/chosen": -1.0188591480255127,
      "logits/rejected": NaN,
      "logps/chosen": -505.81146240234375,
      "logps/rejected": -455.074462890625,
      "loss": 0.672,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.5001901388168335,
      "rewards/margins": 0.3212779462337494,
      "rewards/rejected": 0.1789121776819229,
      "step": 560
    },
    {
      "epoch": 0.9328968903436988,
      "grad_norm": 10.636481285095215,
      "learning_rate": 3.8690115221346276e-05,
      "logits/chosen": -1.0234591960906982,
      "logits/rejected": -0.9390643835067749,
      "logps/chosen": -423.03594970703125,
      "logps/rejected": -383.97369384765625,
      "loss": 0.6515,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.40482932329177856,
      "rewards/margins": 0.20538429915905,
      "rewards/rejected": 0.19944503903388977,
      "step": 570
    },
    {
      "epoch": 0.9492635024549918,
      "grad_norm": 12.001391410827637,
      "learning_rate": 3.838690115221347e-05,
      "logits/chosen": -1.254690170288086,
      "logits/rejected": -1.098749041557312,
      "logps/chosen": -400.72021484375,
      "logps/rejected": -348.39678955078125,
      "loss": 0.6332,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.5642321705818176,
      "rewards/margins": 0.34682947397232056,
      "rewards/rejected": 0.21740269660949707,
      "step": 580
    },
    {
      "epoch": 0.9656301145662848,
      "grad_norm": 9.584880828857422,
      "learning_rate": 3.808368708308066e-05,
      "logits/chosen": -0.9567238092422485,
      "logits/rejected": -0.8445194959640503,
      "logps/chosen": -475.3368225097656,
      "logps/rejected": -418.3975524902344,
      "loss": 0.609,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.7818068861961365,
      "rewards/margins": 0.4438362717628479,
      "rewards/rejected": 0.3379705548286438,
      "step": 590
    },
    {
      "epoch": 0.9819967266775778,
      "grad_norm": 8.225208282470703,
      "learning_rate": 3.778047301394785e-05,
      "logits/chosen": -0.8194648623466492,
      "logits/rejected": -0.6516283750534058,
      "logps/chosen": -464.95648193359375,
      "logps/rejected": -454.74896240234375,
      "loss": 0.6792,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.673040509223938,
      "rewards/margins": 0.3073888421058655,
      "rewards/rejected": 0.3656516671180725,
      "step": 600
    },
    {
      "epoch": 0.9983633387888707,
      "grad_norm": 7.938261032104492,
      "learning_rate": 3.747725894481504e-05,
      "logits/chosen": -0.9632278680801392,
      "logits/rejected": -0.7192381620407104,
      "logps/chosen": -428.2535095214844,
      "logps/rejected": -426.05780029296875,
      "loss": 0.5784,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.778376042842865,
      "rewards/margins": 0.44757041335105896,
      "rewards/rejected": 0.33080559968948364,
      "step": 610
    },
    {
      "epoch": 1.0147299509001637,
      "grad_norm": 6.6163482666015625,
      "learning_rate": 3.717404487568223e-05,
      "logits/chosen": -0.8798831105232239,
      "logits/rejected": -0.716325581073761,
      "logps/chosen": -401.0724182128906,
      "logps/rejected": -393.04925537109375,
      "loss": 0.5397,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.6812083721160889,
      "rewards/margins": 0.6590946912765503,
      "rewards/rejected": 0.022113658487796783,
      "step": 620
    },
    {
      "epoch": 1.0310965630114566,
      "grad_norm": 4.5942792892456055,
      "learning_rate": 3.687083080654943e-05,
      "logits/chosen": -0.595669150352478,
      "logits/rejected": -0.6483780741691589,
      "logps/chosen": -519.7013549804688,
      "logps/rejected": -459.6736755371094,
      "loss": 0.3798,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9690455198287964,
      "rewards/margins": 1.2637821435928345,
      "rewards/rejected": -0.2947366237640381,
      "step": 630
    },
    {
      "epoch": 1.0474631751227497,
      "grad_norm": 5.822299003601074,
      "learning_rate": 3.6567616737416614e-05,
      "logits/chosen": -1.1080423593521118,
      "logits/rejected": -0.8690654635429382,
      "logps/chosen": -430.8399963378906,
      "logps/rejected": -382.16131591796875,
      "loss": 0.3725,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.926641583442688,
      "rewards/margins": 1.1296237707138062,
      "rewards/rejected": -0.20298215746879578,
      "step": 640
    },
    {
      "epoch": 1.0638297872340425,
      "grad_norm": 5.2786078453063965,
      "learning_rate": 3.626440266828381e-05,
      "logits/chosen": -0.986005961894989,
      "logits/rejected": -0.8763575553894043,
      "logps/chosen": -407.85260009765625,
      "logps/rejected": -399.2723693847656,
      "loss": 0.4381,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.8758468627929688,
      "rewards/margins": 0.9750221371650696,
      "rewards/rejected": -0.09917531907558441,
      "step": 650
    },
    {
      "epoch": 1.0801963993453354,
      "grad_norm": 11.273237228393555,
      "learning_rate": 3.5961188599151e-05,
      "logits/chosen": -1.3430869579315186,
      "logits/rejected": -1.056379795074463,
      "logps/chosen": -410.79791259765625,
      "logps/rejected": -434.85845947265625,
      "loss": 0.4005,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9130380749702454,
      "rewards/margins": 0.9950882196426392,
      "rewards/rejected": -0.08205002546310425,
      "step": 660
    },
    {
      "epoch": 1.0965630114566285,
      "grad_norm": 7.220339775085449,
      "learning_rate": 3.5657974530018194e-05,
      "logits/chosen": -1.0506983995437622,
      "logits/rejected": -1.0361990928649902,
      "logps/chosen": -408.18035888671875,
      "logps/rejected": -391.70562744140625,
      "loss": 0.4471,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7731181979179382,
      "rewards/margins": 0.9366775751113892,
      "rewards/rejected": -0.16355939209461212,
      "step": 670
    },
    {
      "epoch": 1.1129296235679214,
      "grad_norm": 5.74124813079834,
      "learning_rate": 3.535476046088539e-05,
      "logits/chosen": -1.206479787826538,
      "logits/rejected": -1.1451863050460815,
      "logps/chosen": -444.80438232421875,
      "logps/rejected": -439.0628356933594,
      "loss": 0.4043,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8405340313911438,
      "rewards/margins": 1.1449936628341675,
      "rewards/rejected": -0.30445969104766846,
      "step": 680
    },
    {
      "epoch": 1.1292962356792144,
      "grad_norm": 7.32386589050293,
      "learning_rate": 3.5051546391752576e-05,
      "logits/chosen": -1.0227863788604736,
      "logits/rejected": -0.9067742228507996,
      "logps/chosen": -476.1012268066406,
      "logps/rejected": -466.14971923828125,
      "loss": 0.4261,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8802071809768677,
      "rewards/margins": 1.1421091556549072,
      "rewards/rejected": -0.2619021236896515,
      "step": 690
    },
    {
      "epoch": 1.1456628477905073,
      "grad_norm": 4.43737268447876,
      "learning_rate": 3.474833232261977e-05,
      "logits/chosen": -1.135181188583374,
      "logits/rejected": -1.0297483205795288,
      "logps/chosen": -506.5615234375,
      "logps/rejected": -483.1039123535156,
      "loss": 0.3875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.5547564029693604,
      "rewards/margins": 1.281551480293274,
      "rewards/rejected": -0.7267951369285583,
      "step": 700
    },
    {
      "epoch": 1.1620294599018004,
      "grad_norm": 2.298978328704834,
      "learning_rate": 3.4445118253486965e-05,
      "logits/chosen": -1.426138162612915,
      "logits/rejected": -1.091489553451538,
      "logps/chosen": -400.107177734375,
      "logps/rejected": -401.62109375,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6946335434913635,
      "rewards/margins": 1.441784143447876,
      "rewards/rejected": -0.7471505999565125,
      "step": 710
    },
    {
      "epoch": 1.1783960720130933,
      "grad_norm": 4.7108154296875,
      "learning_rate": 3.4141904184354156e-05,
      "logits/chosen": -1.1755582094192505,
      "logits/rejected": -0.901213526725769,
      "logps/chosen": -425.28759765625,
      "logps/rejected": -402.042236328125,
      "loss": 0.4615,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.6500431299209595,
      "rewards/margins": 0.9347268342971802,
      "rewards/rejected": -0.2846837639808655,
      "step": 720
    },
    {
      "epoch": 1.1947626841243864,
      "grad_norm": 12.316296577453613,
      "learning_rate": 3.383869011522135e-05,
      "logits/chosen": -1.284589409828186,
      "logits/rejected": -1.0795433521270752,
      "logps/chosen": -413.8037109375,
      "logps/rejected": -491.56146240234375,
      "loss": 0.4385,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.6515102386474609,
      "rewards/margins": 1.2281266450881958,
      "rewards/rejected": -0.5766164660453796,
      "step": 730
    },
    {
      "epoch": 1.2111292962356792,
      "grad_norm": 9.386224746704102,
      "learning_rate": 3.353547604608854e-05,
      "logits/chosen": -1.257037878036499,
      "logits/rejected": -1.0705680847167969,
      "logps/chosen": -505.6416931152344,
      "logps/rejected": -459.56805419921875,
      "loss": 0.3726,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7937537431716919,
      "rewards/margins": 1.4233770370483398,
      "rewards/rejected": -0.629623293876648,
      "step": 740
    },
    {
      "epoch": 1.227495908346972,
      "grad_norm": 6.265888214111328,
      "learning_rate": 3.323226197695573e-05,
      "logits/chosen": -1.332099199295044,
      "logits/rejected": -1.0086009502410889,
      "logps/chosen": -457.0177307128906,
      "logps/rejected": -449.7435607910156,
      "loss": 0.4034,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.790605366230011,
      "rewards/margins": 1.2190719842910767,
      "rewards/rejected": -0.4284667372703552,
      "step": 750
    },
    {
      "epoch": 1.2438625204582652,
      "grad_norm": 6.070497035980225,
      "learning_rate": 3.292904790782292e-05,
      "logits/chosen": -1.2865736484527588,
      "logits/rejected": -1.072819471359253,
      "logps/chosen": -432.710693359375,
      "logps/rejected": -467.6375427246094,
      "loss": 0.3773,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.7443059682846069,
      "rewards/margins": 1.3002400398254395,
      "rewards/rejected": -0.5559341907501221,
      "step": 760
    },
    {
      "epoch": 1.260229132569558,
      "grad_norm": 5.7988128662109375,
      "learning_rate": 3.262583383869012e-05,
      "logits/chosen": -1.0803556442260742,
      "logits/rejected": -0.8561310768127441,
      "logps/chosen": -514.7871704101562,
      "logps/rejected": -503.19500732421875,
      "loss": 0.3216,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7713959217071533,
      "rewards/margins": 1.6244882345199585,
      "rewards/rejected": -0.8530920743942261,
      "step": 770
    },
    {
      "epoch": 1.2765957446808511,
      "grad_norm": 4.323042869567871,
      "learning_rate": 3.232261976955731e-05,
      "logits/chosen": -1.408276915550232,
      "logits/rejected": -0.9835014343261719,
      "logps/chosen": -456.21142578125,
      "logps/rejected": -429.10467529296875,
      "loss": 0.3832,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.5816126465797424,
      "rewards/margins": 1.3987071514129639,
      "rewards/rejected": -0.8170945048332214,
      "step": 780
    },
    {
      "epoch": 1.292962356792144,
      "grad_norm": 7.249789237976074,
      "learning_rate": 3.20194057004245e-05,
      "logits/chosen": -1.1671850681304932,
      "logits/rejected": -0.653769314289093,
      "logps/chosen": -481.74859619140625,
      "logps/rejected": -441.5457458496094,
      "loss": 0.3293,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5319666862487793,
      "rewards/margins": 1.7422034740447998,
      "rewards/rejected": -1.21023690700531,
      "step": 790
    },
    {
      "epoch": 1.3093289689034369,
      "grad_norm": 9.989459037780762,
      "learning_rate": 3.171619163129169e-05,
      "logits/chosen": -1.2330873012542725,
      "logits/rejected": -1.1575677394866943,
      "logps/chosen": -462.0821228027344,
      "logps/rejected": -419.34149169921875,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7070258855819702,
      "rewards/margins": 1.2338263988494873,
      "rewards/rejected": -0.5268004536628723,
      "step": 800
    },
    {
      "epoch": 1.32569558101473,
      "grad_norm": 5.914211750030518,
      "learning_rate": 3.141297756215888e-05,
      "logits/chosen": -1.257925271987915,
      "logits/rejected": -1.1667311191558838,
      "logps/chosen": -492.0726623535156,
      "logps/rejected": -463.2273864746094,
      "loss": 0.3597,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7776979207992554,
      "rewards/margins": 1.3328384160995483,
      "rewards/rejected": -0.555140495300293,
      "step": 810
    },
    {
      "epoch": 1.342062193126023,
      "grad_norm": 3.587632179260254,
      "learning_rate": 3.110976349302608e-05,
      "logits/chosen": -1.2870782613754272,
      "logits/rejected": -0.9442241787910461,
      "logps/chosen": -493.3888244628906,
      "logps/rejected": -464.4835510253906,
      "loss": 0.3151,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.9334325790405273,
      "rewards/margins": 1.9022817611694336,
      "rewards/rejected": -0.9688493013381958,
      "step": 820
    },
    {
      "epoch": 1.358428805237316,
      "grad_norm": 6.462862968444824,
      "learning_rate": 3.080654942389327e-05,
      "logits/chosen": -1.4893635511398315,
      "logits/rejected": -1.0626758337020874,
      "logps/chosen": -373.52288818359375,
      "logps/rejected": -411.831298828125,
      "loss": 0.438,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.5565814971923828,
      "rewards/margins": 1.1154134273529053,
      "rewards/rejected": -0.5588318705558777,
      "step": 830
    },
    {
      "epoch": 1.3747954173486088,
      "grad_norm": 6.482163429260254,
      "learning_rate": 3.0503335354760464e-05,
      "logits/chosen": -1.2426273822784424,
      "logits/rejected": -1.0962947607040405,
      "logps/chosen": -418.48577880859375,
      "logps/rejected": -414.7897033691406,
      "loss": 0.4516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7719891667366028,
      "rewards/margins": 1.0017683506011963,
      "rewards/rejected": -0.22977924346923828,
      "step": 840
    },
    {
      "epoch": 1.3911620294599019,
      "grad_norm": 8.918207168579102,
      "learning_rate": 3.020012128562765e-05,
      "logits/chosen": -1.3620984554290771,
      "logits/rejected": -1.1984447240829468,
      "logps/chosen": -465.46270751953125,
      "logps/rejected": -446.5343322753906,
      "loss": 0.4053,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6932386159896851,
      "rewards/margins": 1.3631744384765625,
      "rewards/rejected": -0.6699355840682983,
      "step": 850
    },
    {
      "epoch": 1.4075286415711947,
      "grad_norm": 19.940839767456055,
      "learning_rate": 2.9896907216494846e-05,
      "logits/chosen": -1.2039759159088135,
      "logits/rejected": -0.9377444386482239,
      "logps/chosen": -574.588623046875,
      "logps/rejected": -502.51214599609375,
      "loss": 0.4461,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8642657399177551,
      "rewards/margins": 1.3982216119766235,
      "rewards/rejected": -0.5339558124542236,
      "step": 860
    },
    {
      "epoch": 1.4238952536824878,
      "grad_norm": 6.069481372833252,
      "learning_rate": 2.959369314736204e-05,
      "logits/chosen": -1.3913848400115967,
      "logits/rejected": -1.2518432140350342,
      "logps/chosen": -458.3941345214844,
      "logps/rejected": -452.67352294921875,
      "loss": 0.3604,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.043166160583496,
      "rewards/margins": 1.7257499694824219,
      "rewards/rejected": -0.6825838685035706,
      "step": 870
    },
    {
      "epoch": 1.4402618657937807,
      "grad_norm": 3.334609031677246,
      "learning_rate": 2.929047907822923e-05,
      "logits/chosen": -1.3935604095458984,
      "logits/rejected": NaN,
      "logps/chosen": -490.9991149902344,
      "logps/rejected": -452.0387268066406,
      "loss": 0.3173,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0780329704284668,
      "rewards/margins": 1.919669508934021,
      "rewards/rejected": -0.8416364789009094,
      "step": 880
    },
    {
      "epoch": 1.4566284779050735,
      "grad_norm": 5.730930805206299,
      "learning_rate": 2.8987265009096426e-05,
      "logits/chosen": -1.3444249629974365,
      "logits/rejected": -1.124793291091919,
      "logps/chosen": -409.4859619140625,
      "logps/rejected": -429.79150390625,
      "loss": 0.4307,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.8416284322738647,
      "rewards/margins": 1.227608561515808,
      "rewards/rejected": -0.3859800696372986,
      "step": 890
    },
    {
      "epoch": 1.4729950900163666,
      "grad_norm": 4.7208333015441895,
      "learning_rate": 2.8684050939963614e-05,
      "logits/chosen": -1.2483845949172974,
      "logits/rejected": -1.162945032119751,
      "logps/chosen": -438.02978515625,
      "logps/rejected": -483.64776611328125,
      "loss": 0.4359,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6855508089065552,
      "rewards/margins": 1.3711297512054443,
      "rewards/rejected": -0.6855790019035339,
      "step": 900
    },
    {
      "epoch": 1.4893617021276595,
      "grad_norm": 4.603919982910156,
      "learning_rate": 2.838083687083081e-05,
      "logits/chosen": -1.3359647989273071,
      "logits/rejected": -1.2829469442367554,
      "logps/chosen": -478.17022705078125,
      "logps/rejected": -408.0987243652344,
      "loss": 0.431,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.9056881666183472,
      "rewards/margins": 1.124969244003296,
      "rewards/rejected": -0.21928110718727112,
      "step": 910
    },
    {
      "epoch": 1.5057283142389526,
      "grad_norm": 7.60992431640625,
      "learning_rate": 2.8077622801698e-05,
      "logits/chosen": -1.4223133325576782,
      "logits/rejected": -1.2066584825515747,
      "logps/chosen": -428.1543884277344,
      "logps/rejected": -370.4458923339844,
      "loss": 0.4003,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.8528293371200562,
      "rewards/margins": 1.363084077835083,
      "rewards/rejected": -0.5102548599243164,
      "step": 920
    },
    {
      "epoch": 1.5220949263502455,
      "grad_norm": 4.768836498260498,
      "learning_rate": 2.7774408732565194e-05,
      "logits/chosen": -1.2342239618301392,
      "logits/rejected": -1.1602811813354492,
      "logps/chosen": -440.5987854003906,
      "logps/rejected": -405.0908508300781,
      "loss": 0.3879,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.9851611256599426,
      "rewards/margins": 1.1822245121002197,
      "rewards/rejected": -0.19706343114376068,
      "step": 930
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 8.149114608764648,
      "learning_rate": 2.7471194663432385e-05,
      "logits/chosen": -1.1173126697540283,
      "logits/rejected": -1.014575481414795,
      "logps/chosen": -402.3595275878906,
      "logps/rejected": -410.60003662109375,
      "loss": 0.4529,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7943775653839111,
      "rewards/margins": 1.2599968910217285,
      "rewards/rejected": -0.4656193256378174,
      "step": 940
    },
    {
      "epoch": 1.5548281505728314,
      "grad_norm": 7.4743852615356445,
      "learning_rate": 2.7167980594299573e-05,
      "logits/chosen": NaN,
      "logits/rejected": -1.143424391746521,
      "logps/chosen": -446.26629638671875,
      "logps/rejected": -469.7017517089844,
      "loss": 0.491,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.797395646572113,
      "rewards/margins": 1.1872708797454834,
      "rewards/rejected": -0.38987529277801514,
      "step": 950
    },
    {
      "epoch": 1.5711947626841245,
      "grad_norm": 8.159062385559082,
      "learning_rate": 2.6864766525166768e-05,
      "logits/chosen": -1.14289128780365,
      "logits/rejected": NaN,
      "logps/chosen": -450.77117919921875,
      "logps/rejected": -438.8096618652344,
      "loss": 0.4282,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.7298113107681274,
      "rewards/margins": 1.092790126800537,
      "rewards/rejected": -0.3629787862300873,
      "step": 960
    },
    {
      "epoch": 1.5875613747954174,
      "grad_norm": 9.274547576904297,
      "learning_rate": 2.6561552456033962e-05,
      "logits/chosen": -0.9688614010810852,
      "logits/rejected": -0.897729754447937,
      "logps/chosen": -515.4688720703125,
      "logps/rejected": -436.51275634765625,
      "loss": 0.4169,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7370742559432983,
      "rewards/margins": 1.1543539762496948,
      "rewards/rejected": -0.4172796607017517,
      "step": 970
    },
    {
      "epoch": 1.6039279869067102,
      "grad_norm": 5.051360130310059,
      "learning_rate": 2.6258338386901153e-05,
      "logits/chosen": -1.0737905502319336,
      "logits/rejected": -0.9251714944839478,
      "logps/chosen": -471.31146240234375,
      "logps/rejected": -432.572509765625,
      "loss": 0.3841,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.8669958114624023,
      "rewards/margins": 1.4004991054534912,
      "rewards/rejected": -0.5335032939910889,
      "step": 980
    },
    {
      "epoch": 1.6202945990180033,
      "grad_norm": 6.768540859222412,
      "learning_rate": 2.5955124317768348e-05,
      "logits/chosen": -1.1147942543029785,
      "logits/rejected": -1.1377754211425781,
      "logps/chosen": -462.5465393066406,
      "logps/rejected": -390.2607727050781,
      "loss": 0.3919,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8462246060371399,
      "rewards/margins": 1.2740905284881592,
      "rewards/rejected": -0.4278659224510193,
      "step": 990
    },
    {
      "epoch": 1.6366612111292962,
      "grad_norm": 8.313865661621094,
      "learning_rate": 2.5651910248635535e-05,
      "logits/chosen": -1.0041131973266602,
      "logits/rejected": -0.8046473264694214,
      "logps/chosen": -461.7525939941406,
      "logps/rejected": -443.0941467285156,
      "loss": 0.4222,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8126258850097656,
      "rewards/margins": 1.1657203435897827,
      "rewards/rejected": -0.3530944585800171,
      "step": 1000
    },
    {
      "epoch": 1.6530278232405893,
      "grad_norm": 11.539511680603027,
      "learning_rate": 2.534869617950273e-05,
      "logits/chosen": -1.2518326044082642,
      "logits/rejected": -0.9322371482849121,
      "logps/chosen": -477.0799255371094,
      "logps/rejected": -450.06976318359375,
      "loss": 0.3355,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.9717556834220886,
      "rewards/margins": 1.7565772533416748,
      "rewards/rejected": -0.784821629524231,
      "step": 1010
    },
    {
      "epoch": 1.6693944353518821,
      "grad_norm": 8.115518569946289,
      "learning_rate": 2.504548211036992e-05,
      "logits/chosen": -1.4984604120254517,
      "logits/rejected": -1.33262038230896,
      "logps/chosen": -469.9319763183594,
      "logps/rejected": -408.9674072265625,
      "loss": 0.3737,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8488012552261353,
      "rewards/margins": 1.4102153778076172,
      "rewards/rejected": -0.5614141225814819,
      "step": 1020
    },
    {
      "epoch": 1.685761047463175,
      "grad_norm": 7.375257968902588,
      "learning_rate": 2.4742268041237116e-05,
      "logits/chosen": -1.078918695449829,
      "logits/rejected": -0.8934460878372192,
      "logps/chosen": -492.44769287109375,
      "logps/rejected": -451.58795166015625,
      "loss": 0.3332,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.0642842054367065,
      "rewards/margins": 1.744214415550232,
      "rewards/rejected": -0.6799300909042358,
      "step": 1030
    },
    {
      "epoch": 1.702127659574468,
      "grad_norm": 3.4032139778137207,
      "learning_rate": 2.4439053972104307e-05,
      "logits/chosen": -1.1205917596817017,
      "logits/rejected": -0.8298323750495911,
      "logps/chosen": -420.125,
      "logps/rejected": -380.03729248046875,
      "loss": 0.4017,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.7596099376678467,
      "rewards/margins": 1.2399921417236328,
      "rewards/rejected": -0.48038214445114136,
      "step": 1040
    },
    {
      "epoch": 1.7184942716857612,
      "grad_norm": 3.1656646728515625,
      "learning_rate": 2.4135839902971498e-05,
      "logits/chosen": -1.0980288982391357,
      "logits/rejected": -0.9042747616767883,
      "logps/chosen": -448.61029052734375,
      "logps/rejected": -430.3683166503906,
      "loss": 0.3857,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.9228833317756653,
      "rewards/margins": 1.3001716136932373,
      "rewards/rejected": -0.37728843092918396,
      "step": 1050
    },
    {
      "epoch": 1.734860883797054,
      "grad_norm": 6.291409015655518,
      "learning_rate": 2.3832625833838693e-05,
      "logits/chosen": -1.2553077936172485,
      "logits/rejected": -0.9304355382919312,
      "logps/chosen": -443.744140625,
      "logps/rejected": -375.67840576171875,
      "loss": 0.4043,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7896615862846375,
      "rewards/margins": 1.1822974681854248,
      "rewards/rejected": -0.39263591170310974,
      "step": 1060
    },
    {
      "epoch": 1.751227495908347,
      "grad_norm": 4.611414909362793,
      "learning_rate": 2.3529411764705884e-05,
      "logits/chosen": -1.3921210765838623,
      "logits/rejected": -1.2024405002593994,
      "logps/chosen": -438.8999938964844,
      "logps/rejected": -379.8556213378906,
      "loss": 0.3666,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7746474742889404,
      "rewards/margins": 1.3105146884918213,
      "rewards/rejected": -0.5358672142028809,
      "step": 1070
    },
    {
      "epoch": 1.7675941080196398,
      "grad_norm": 2.6192657947540283,
      "learning_rate": 2.3226197695573075e-05,
      "logits/chosen": -1.2743886709213257,
      "logits/rejected": -1.193303108215332,
      "logps/chosen": -469.51220703125,
      "logps/rejected": -446.6626892089844,
      "loss": 0.4615,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.9081875085830688,
      "rewards/margins": 1.3380632400512695,
      "rewards/rejected": -0.4298757016658783,
      "step": 1080
    },
    {
      "epoch": 1.7839607201309329,
      "grad_norm": 5.901863098144531,
      "learning_rate": 2.2922983626440266e-05,
      "logits/chosen": -1.249284267425537,
      "logits/rejected": -1.0896720886230469,
      "logps/chosen": -388.6412048339844,
      "logps/rejected": -381.84747314453125,
      "loss": 0.4718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.62062668800354,
      "rewards/margins": 1.1487479209899902,
      "rewards/rejected": -0.5281211733818054,
      "step": 1090
    },
    {
      "epoch": 1.800327332242226,
      "grad_norm": 4.81393575668335,
      "learning_rate": 2.261976955730746e-05,
      "logits/chosen": -1.2179973125457764,
      "logits/rejected": -1.1146996021270752,
      "logps/chosen": -432.94970703125,
      "logps/rejected": -404.9814453125,
      "loss": 0.4061,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.49649685621261597,
      "rewards/margins": 1.0791733264923096,
      "rewards/rejected": -0.5826765298843384,
      "step": 1100
    },
    {
      "epoch": 1.8166939443535188,
      "grad_norm": 7.300076484680176,
      "learning_rate": 2.2316555488174655e-05,
      "logits/chosen": -1.4200313091278076,
      "logits/rejected": -1.3520318269729614,
      "logps/chosen": -398.1583557128906,
      "logps/rejected": -411.41864013671875,
      "loss": 0.4397,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.8094931840896606,
      "rewards/margins": 1.2237316370010376,
      "rewards/rejected": -0.41423845291137695,
      "step": 1110
    },
    {
      "epoch": 1.8330605564648117,
      "grad_norm": 5.185066223144531,
      "learning_rate": 2.2013341419041843e-05,
      "logits/chosen": -1.282606840133667,
      "logits/rejected": -1.3600891828536987,
      "logps/chosen": -397.7345275878906,
      "logps/rejected": -368.4806823730469,
      "loss": 0.4605,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.7671560049057007,
      "rewards/margins": 0.9713778495788574,
      "rewards/rejected": -0.20422179996967316,
      "step": 1120
    },
    {
      "epoch": 1.8494271685761048,
      "grad_norm": 6.64644718170166,
      "learning_rate": 2.1710127349909037e-05,
      "logits/chosen": -1.309072732925415,
      "logits/rejected": -1.237240195274353,
      "logps/chosen": -508.67864990234375,
      "logps/rejected": -486.6451110839844,
      "loss": 0.3327,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.003493070602417,
      "rewards/margins": 1.6196180582046509,
      "rewards/rejected": -0.6161248683929443,
      "step": 1130
    },
    {
      "epoch": 1.8657937806873979,
      "grad_norm": 8.295884132385254,
      "learning_rate": 2.140691328077623e-05,
      "logits/chosen": -1.2569806575775146,
      "logits/rejected": -1.0191644430160522,
      "logps/chosen": -450.56884765625,
      "logps/rejected": -396.7924499511719,
      "loss": 0.4093,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.6329465508460999,
      "rewards/margins": 1.3668501377105713,
      "rewards/rejected": -0.7339035868644714,
      "step": 1140
    },
    {
      "epoch": 1.8821603927986907,
      "grad_norm": 5.191346168518066,
      "learning_rate": 2.110369921164342e-05,
      "logits/chosen": -1.2032971382141113,
      "logits/rejected": -0.9198576211929321,
      "logps/chosen": -401.7949523925781,
      "logps/rejected": -387.2808532714844,
      "loss": 0.3817,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5356651544570923,
      "rewards/margins": 1.356689214706421,
      "rewards/rejected": -0.8210241198539734,
      "step": 1150
    },
    {
      "epoch": 1.8985270049099836,
      "grad_norm": 4.5822367668151855,
      "learning_rate": 2.0800485142510614e-05,
      "logits/chosen": -1.16325843334198,
      "logits/rejected": -0.8339241147041321,
      "logps/chosen": -385.2162170410156,
      "logps/rejected": -399.42401123046875,
      "loss": 0.4329,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.5546852946281433,
      "rewards/margins": 1.410249948501587,
      "rewards/rejected": -0.8555648922920227,
      "step": 1160
    },
    {
      "epoch": 1.9148936170212765,
      "grad_norm": 7.598694324493408,
      "learning_rate": 2.0497271073377805e-05,
      "logits/chosen": -1.2841596603393555,
      "logits/rejected": -1.0715458393096924,
      "logps/chosen": -426.3335876464844,
      "logps/rejected": -392.77947998046875,
      "loss": 0.3999,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.6999267339706421,
      "rewards/margins": 1.2335584163665771,
      "rewards/rejected": -0.5336317420005798,
      "step": 1170
    },
    {
      "epoch": 1.9312602291325696,
      "grad_norm": 12.754505157470703,
      "learning_rate": 2.0194057004245e-05,
      "logits/chosen": -1.2506614923477173,
      "logits/rejected": -1.1044342517852783,
      "logps/chosen": -464.2216796875,
      "logps/rejected": -420.13336181640625,
      "loss": 0.4258,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.7730525135993958,
      "rewards/margins": 1.1193318367004395,
      "rewards/rejected": -0.3462793827056885,
      "step": 1180
    },
    {
      "epoch": 1.9476268412438626,
      "grad_norm": 5.040411949157715,
      "learning_rate": 1.9890842935112188e-05,
      "logits/chosen": -1.5200366973876953,
      "logits/rejected": -1.453028917312622,
      "logps/chosen": -471.79180908203125,
      "logps/rejected": -421.40899658203125,
      "loss": 0.3997,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.7299527525901794,
      "rewards/margins": 1.2998378276824951,
      "rewards/rejected": -0.56988525390625,
      "step": 1190
    },
    {
      "epoch": 1.9639934533551555,
      "grad_norm": 12.064972877502441,
      "learning_rate": 1.9587628865979382e-05,
      "logits/chosen": -1.2272264957427979,
      "logits/rejected": -0.9471932649612427,
      "logps/chosen": -501.19842529296875,
      "logps/rejected": -451.31658935546875,
      "loss": 0.4069,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8413068056106567,
      "rewards/margins": 1.3548190593719482,
      "rewards/rejected": -0.5135123133659363,
      "step": 1200
    },
    {
      "epoch": 1.9803600654664484,
      "grad_norm": 7.827969551086426,
      "learning_rate": 1.9284414796846577e-05,
      "logits/chosen": -1.1897993087768555,
      "logits/rejected": -1.085766077041626,
      "logps/chosen": -467.1263122558594,
      "logps/rejected": -420.46417236328125,
      "loss": 0.4029,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8640726208686829,
      "rewards/margins": 1.2383184432983398,
      "rewards/rejected": -0.3742457330226898,
      "step": 1210
    },
    {
      "epoch": 1.9967266775777412,
      "grad_norm": 4.033712863922119,
      "learning_rate": 1.8981200727713764e-05,
      "logits/chosen": -1.1938722133636475,
      "logits/rejected": -0.7925646305084229,
      "logps/chosen": -434.82916259765625,
      "logps/rejected": -415.64178466796875,
      "loss": 0.4062,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.6897398233413696,
      "rewards/margins": 1.3555059432983398,
      "rewards/rejected": -0.6657661199569702,
      "step": 1220
    },
    {
      "epoch": 2.0130932896890346,
      "grad_norm": 2.060443162918091,
      "learning_rate": 1.867798665858096e-05,
      "logits/chosen": NaN,
      "logits/rejected": -1.3225266933441162,
      "logps/chosen": -402.45654296875,
      "logps/rejected": -432.16448974609375,
      "loss": 0.3632,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.8448646664619446,
      "rewards/margins": 1.4718005657196045,
      "rewards/rejected": -0.6269359588623047,
      "step": 1230
    },
    {
      "epoch": 2.0294599018003274,
      "grad_norm": 2.7568705081939697,
      "learning_rate": 1.8374772589448153e-05,
      "logits/chosen": -1.589988112449646,
      "logits/rejected": -1.4496564865112305,
      "logps/chosen": -415.6414489746094,
      "logps/rejected": -382.269287109375,
      "loss": 0.3286,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.015729308128357,
      "rewards/margins": 1.6888082027435303,
      "rewards/rejected": -0.6730789542198181,
      "step": 1240
    },
    {
      "epoch": 2.0458265139116203,
      "grad_norm": 7.693350315093994,
      "learning_rate": 1.8071558520315345e-05,
      "logits/chosen": -1.2396166324615479,
      "logits/rejected": -1.0914833545684814,
      "logps/chosen": -428.75146484375,
      "logps/rejected": -439.94537353515625,
      "loss": 0.3436,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.9149911999702454,
      "rewards/margins": 1.669695496559143,
      "rewards/rejected": -0.7547044157981873,
      "step": 1250
    },
    {
      "epoch": 2.062193126022913,
      "grad_norm": 6.800998210906982,
      "learning_rate": 1.7768344451182536e-05,
      "logits/chosen": -1.3096355199813843,
      "logits/rejected": -1.2225430011749268,
      "logps/chosen": -544.235595703125,
      "logps/rejected": -471.3172302246094,
      "loss": 0.2387,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1924231052398682,
      "rewards/margins": 1.9907211065292358,
      "rewards/rejected": -0.7982980608940125,
      "step": 1260
    },
    {
      "epoch": 2.078559738134206,
      "grad_norm": 2.9473204612731934,
      "learning_rate": 1.7465130382049727e-05,
      "logits/chosen": -1.5475558042526245,
      "logits/rejected": -1.2119872570037842,
      "logps/chosen": -425.4368591308594,
      "logps/rejected": -431.3377990722656,
      "loss": 0.1952,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1395524740219116,
      "rewards/margins": 2.476022243499756,
      "rewards/rejected": -1.3364696502685547,
      "step": 1270
    },
    {
      "epoch": 2.0949263502454993,
      "grad_norm": 5.591909885406494,
      "learning_rate": 1.716191631291692e-05,
      "logits/chosen": -1.5607810020446777,
      "logits/rejected": -1.4012185335159302,
      "logps/chosen": -413.8076171875,
      "logps/rejected": -411.9990234375,
      "loss": 0.2864,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.065088152885437,
      "rewards/margins": 1.9613841772079468,
      "rewards/rejected": -0.8962962031364441,
      "step": 1280
    },
    {
      "epoch": 2.111292962356792,
      "grad_norm": 4.462930679321289,
      "learning_rate": 1.6858702243784113e-05,
      "logits/chosen": -1.4985883235931396,
      "logits/rejected": -1.2055768966674805,
      "logps/chosen": -402.1160583496094,
      "logps/rejected": -397.81402587890625,
      "loss": 0.2957,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8168094754219055,
      "rewards/margins": 1.9579219818115234,
      "rewards/rejected": -1.1411124467849731,
      "step": 1290
    },
    {
      "epoch": 2.127659574468085,
      "grad_norm": 3.3461086750030518,
      "learning_rate": 1.6555488174651304e-05,
      "logits/chosen": -1.5269520282745361,
      "logits/rejected": -1.0539462566375732,
      "logps/chosen": -507.36834716796875,
      "logps/rejected": -461.356689453125,
      "loss": 0.2565,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6355259418487549,
      "rewards/margins": 2.068328380584717,
      "rewards/rejected": -1.432802438735962,
      "step": 1300
    },
    {
      "epoch": 2.144026186579378,
      "grad_norm": 4.208719253540039,
      "learning_rate": 1.6252274105518498e-05,
      "logits/chosen": -1.403276801109314,
      "logits/rejected": -1.199331283569336,
      "logps/chosen": -427.9095153808594,
      "logps/rejected": -373.31756591796875,
      "loss": 0.2796,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.8879494667053223,
      "rewards/margins": 1.912683129310608,
      "rewards/rejected": -1.0247336626052856,
      "step": 1310
    },
    {
      "epoch": 2.160392798690671,
      "grad_norm": 3.667640447616577,
      "learning_rate": 1.594906003638569e-05,
      "logits/chosen": -1.4091194868087769,
      "logits/rejected": -1.123011827468872,
      "logps/chosen": -457.79461669921875,
      "logps/rejected": -444.686279296875,
      "loss": 0.2564,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.8721135258674622,
      "rewards/margins": 2.0980000495910645,
      "rewards/rejected": -1.2258864641189575,
      "step": 1320
    },
    {
      "epoch": 2.176759410801964,
      "grad_norm": 2.9427664279937744,
      "learning_rate": 1.564584596725288e-05,
      "logits/chosen": -1.4273817539215088,
      "logits/rejected": -1.2326085567474365,
      "logps/chosen": -431.9471130371094,
      "logps/rejected": -421.22747802734375,
      "loss": 0.3017,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.7032436728477478,
      "rewards/margins": 1.7488292455673218,
      "rewards/rejected": -1.0455853939056396,
      "step": 1330
    },
    {
      "epoch": 2.193126022913257,
      "grad_norm": 3.592177629470825,
      "learning_rate": 1.5342631898120075e-05,
      "logits/chosen": -1.3208458423614502,
      "logits/rejected": -1.1550036668777466,
      "logps/chosen": -442.4117736816406,
      "logps/rejected": -370.07708740234375,
      "loss": 0.3644,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.9467415809631348,
      "rewards/margins": 1.7952324151992798,
      "rewards/rejected": -0.8484910726547241,
      "step": 1340
    },
    {
      "epoch": 2.20949263502455,
      "grad_norm": 2.6388635635375977,
      "learning_rate": 1.5039417828987265e-05,
      "logits/chosen": -1.305037021636963,
      "logits/rejected": -1.2544772624969482,
      "logps/chosen": -438.3041076660156,
      "logps/rejected": -412.14923095703125,
      "loss": 0.2553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.2883623838424683,
      "rewards/margins": 1.9844329357147217,
      "rewards/rejected": -0.6960704326629639,
      "step": 1350
    },
    {
      "epoch": 2.2258592471358427,
      "grad_norm": 5.4221954345703125,
      "learning_rate": 1.4736203759854459e-05,
      "logits/chosen": -1.3623788356781006,
      "logits/rejected": -1.030826210975647,
      "logps/chosen": -437.98150634765625,
      "logps/rejected": -413.3927307128906,
      "loss": 0.3294,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8824920654296875,
      "rewards/margins": 1.7985508441925049,
      "rewards/rejected": -0.9160588383674622,
      "step": 1360
    },
    {
      "epoch": 2.242225859247136,
      "grad_norm": 1.4540634155273438,
      "learning_rate": 1.4432989690721649e-05,
      "logits/chosen": -1.4040526151657104,
      "logits/rejected": -1.2125266790390015,
      "logps/chosen": -444.2418518066406,
      "logps/rejected": -445.39154052734375,
      "loss": 0.2202,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2369893789291382,
      "rewards/margins": 2.244361162185669,
      "rewards/rejected": -1.0073716640472412,
      "step": 1370
    },
    {
      "epoch": 2.258592471358429,
      "grad_norm": 5.328054904937744,
      "learning_rate": 1.4129775621588841e-05,
      "logits/chosen": -1.1481372117996216,
      "logits/rejected": -1.1810128688812256,
      "logps/chosen": -483.92901611328125,
      "logps/rejected": -435.12408447265625,
      "loss": 0.2465,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3297772407531738,
      "rewards/margins": 2.0740952491760254,
      "rewards/rejected": -0.7443181872367859,
      "step": 1380
    },
    {
      "epoch": 2.2749590834697218,
      "grad_norm": 5.371053695678711,
      "learning_rate": 1.3826561552456036e-05,
      "logits/chosen": -1.341086506843567,
      "logits/rejected": -1.2301025390625,
      "logps/chosen": -441.9823303222656,
      "logps/rejected": -371.228271484375,
      "loss": 0.2796,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.0860683917999268,
      "rewards/margins": 1.9399855136871338,
      "rewards/rejected": -0.853917121887207,
      "step": 1390
    },
    {
      "epoch": 2.2913256955810146,
      "grad_norm": 4.199862480163574,
      "learning_rate": 1.3523347483323225e-05,
      "logits/chosen": -1.2742938995361328,
      "logits/rejected": -1.0367982387542725,
      "logps/chosen": -441.23193359375,
      "logps/rejected": -432.3246154785156,
      "loss": 0.2161,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.0744221210479736,
      "rewards/margins": 2.1862921714782715,
      "rewards/rejected": -1.1118695735931396,
      "step": 1400
    },
    {
      "epoch": 2.3076923076923075,
      "grad_norm": 4.229262351989746,
      "learning_rate": 1.322013341419042e-05,
      "logits/chosen": -1.4364268779754639,
      "logits/rejected": -1.0992625951766968,
      "logps/chosen": -425.8427734375,
      "logps/rejected": -459.83642578125,
      "loss": 0.2952,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9090694189071655,
      "rewards/margins": 1.8446950912475586,
      "rewards/rejected": -0.9356255531311035,
      "step": 1410
    },
    {
      "epoch": 2.324058919803601,
      "grad_norm": 2.794767379760742,
      "learning_rate": 1.2916919345057613e-05,
      "logits/chosen": -1.354282021522522,
      "logits/rejected": -1.2035588026046753,
      "logps/chosen": -420.01800537109375,
      "logps/rejected": -375.9340515136719,
      "loss": 0.2299,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1438004970550537,
      "rewards/margins": 2.1549389362335205,
      "rewards/rejected": -1.0111383199691772,
      "step": 1420
    },
    {
      "epoch": 2.3404255319148937,
      "grad_norm": 1.970015525817871,
      "learning_rate": 1.2613705275924804e-05,
      "logits/chosen": -1.3893083333969116,
      "logits/rejected": -1.3427255153656006,
      "logps/chosen": -428.79400634765625,
      "logps/rejected": -409.81500244140625,
      "loss": 0.2522,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1792007684707642,
      "rewards/margins": 2.0457987785339355,
      "rewards/rejected": -0.8665979504585266,
      "step": 1430
    },
    {
      "epoch": 2.3567921440261865,
      "grad_norm": 3.0225942134857178,
      "learning_rate": 1.2310491206791997e-05,
      "logits/chosen": -1.5218456983566284,
      "logits/rejected": -1.236580491065979,
      "logps/chosen": -474.56829833984375,
      "logps/rejected": -513.43408203125,
      "loss": 0.2505,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.0795310735702515,
      "rewards/margins": 2.1252081394195557,
      "rewards/rejected": -1.0456770658493042,
      "step": 1440
    },
    {
      "epoch": 2.3731587561374794,
      "grad_norm": 4.234305381774902,
      "learning_rate": 1.2007277137659188e-05,
      "logits/chosen": -1.3646624088287354,
      "logits/rejected": -1.2389724254608154,
      "logps/chosen": -431.1107482910156,
      "logps/rejected": -442.53997802734375,
      "loss": 0.3115,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8404763340950012,
      "rewards/margins": 2.168391466140747,
      "rewards/rejected": -1.3279149532318115,
      "step": 1450
    },
    {
      "epoch": 2.3895253682487727,
      "grad_norm": 9.905987739562988,
      "learning_rate": 1.170406306852638e-05,
      "logits/chosen": -1.4754550457000732,
      "logits/rejected": -1.4108117818832397,
      "logps/chosen": -453.54364013671875,
      "logps/rejected": -432.04693603515625,
      "loss": 0.3368,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.6598278284072876,
      "rewards/margins": 1.6955798864364624,
      "rewards/rejected": -1.0357519388198853,
      "step": 1460
    },
    {
      "epoch": 2.4058919803600656,
      "grad_norm": 4.8183746337890625,
      "learning_rate": 1.1400848999393572e-05,
      "logits/chosen": -1.3298962116241455,
      "logits/rejected": -1.2613223791122437,
      "logps/chosen": -480.92205810546875,
      "logps/rejected": -446.716796875,
      "loss": 0.3334,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.8991876840591431,
      "rewards/margins": 1.7357298135757446,
      "rewards/rejected": -0.8365424275398254,
      "step": 1470
    },
    {
      "epoch": 2.4222585924713584,
      "grad_norm": 6.064842700958252,
      "learning_rate": 1.1097634930260765e-05,
      "logits/chosen": -1.4571373462677002,
      "logits/rejected": -1.1927803754806519,
      "logps/chosen": -447.56787109375,
      "logps/rejected": -440.72064208984375,
      "loss": 0.2964,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0991294384002686,
      "rewards/margins": 2.0829381942749023,
      "rewards/rejected": -0.983808696269989,
      "step": 1480
    },
    {
      "epoch": 2.4386252045826513,
      "grad_norm": 2.8285984992980957,
      "learning_rate": 1.0794420861127958e-05,
      "logits/chosen": -1.4223463535308838,
      "logits/rejected": -1.2678343057632446,
      "logps/chosen": -438.255126953125,
      "logps/rejected": -454.03948974609375,
      "loss": 0.2041,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.2381608486175537,
      "rewards/margins": 2.366389036178589,
      "rewards/rejected": -1.1282285451889038,
      "step": 1490
    },
    {
      "epoch": 2.454991816693944,
      "grad_norm": 3.3027286529541016,
      "learning_rate": 1.0491206791995149e-05,
      "logits/chosen": -1.3270039558410645,
      "logits/rejected": -1.2212953567504883,
      "logps/chosen": -447.6775817871094,
      "logps/rejected": -454.34918212890625,
      "loss": 0.246,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9780976176261902,
      "rewards/margins": 1.9729305505752563,
      "rewards/rejected": -0.9948328137397766,
      "step": 1500
    },
    {
      "epoch": 2.4713584288052375,
      "grad_norm": 5.344712734222412,
      "learning_rate": 1.0187992722862342e-05,
      "logits/chosen": -1.2262275218963623,
      "logits/rejected": -1.0121392011642456,
      "logps/chosen": -464.63897705078125,
      "logps/rejected": -446.21954345703125,
      "loss": 0.2936,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0501153469085693,
      "rewards/margins": 2.157829999923706,
      "rewards/rejected": -1.1077148914337158,
      "step": 1510
    },
    {
      "epoch": 2.4877250409165304,
      "grad_norm": 3.925086259841919,
      "learning_rate": 9.884778653729533e-06,
      "logits/chosen": -1.2609909772872925,
      "logits/rejected": -1.3114054203033447,
      "logps/chosen": -483.9059143066406,
      "logps/rejected": -450.318115234375,
      "loss": 0.3007,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.0718286037445068,
      "rewards/margins": 1.886443853378296,
      "rewards/rejected": -0.8146154284477234,
      "step": 1520
    },
    {
      "epoch": 2.504091653027823,
      "grad_norm": 5.031212329864502,
      "learning_rate": 9.581564584596727e-06,
      "logits/chosen": -1.3080980777740479,
      "logits/rejected": -1.1388887166976929,
      "logps/chosen": -433.4994201660156,
      "logps/rejected": -423.66961669921875,
      "loss": 0.2731,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9743747711181641,
      "rewards/margins": 1.9618078470230103,
      "rewards/rejected": -0.9874328374862671,
      "step": 1530
    },
    {
      "epoch": 2.520458265139116,
      "grad_norm": 4.041630744934082,
      "learning_rate": 9.278350515463918e-06,
      "logits/chosen": -1.4856176376342773,
      "logits/rejected": -1.3291361331939697,
      "logps/chosen": -454.8204040527344,
      "logps/rejected": -457.5218811035156,
      "loss": 0.217,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.9250513911247253,
      "rewards/margins": 2.4876701831817627,
      "rewards/rejected": -1.562618613243103,
      "step": 1540
    },
    {
      "epoch": 2.5368248772504094,
      "grad_norm": 9.800390243530273,
      "learning_rate": 8.97513644633111e-06,
      "logits/chosen": -1.6522566080093384,
      "logits/rejected": -1.5452460050582886,
      "logps/chosen": -444.83001708984375,
      "logps/rejected": -452.7578125,
      "loss": 0.2926,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.9433294534683228,
      "rewards/margins": 1.8863716125488281,
      "rewards/rejected": -0.943041980266571,
      "step": 1550
    },
    {
      "epoch": 2.5531914893617023,
      "grad_norm": 4.188653945922852,
      "learning_rate": 8.671922377198302e-06,
      "logits/chosen": -1.1361147165298462,
      "logits/rejected": -1.007997751235962,
      "logps/chosen": -455.88995361328125,
      "logps/rejected": -436.4647521972656,
      "loss": 0.3054,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.9836293458938599,
      "rewards/margins": 2.064480781555176,
      "rewards/rejected": -1.0808513164520264,
      "step": 1560
    },
    {
      "epoch": 2.569558101472995,
      "grad_norm": 3.1928329467773438,
      "learning_rate": 8.368708308065494e-06,
      "logits/chosen": -1.5517833232879639,
      "logits/rejected": -1.3304557800292969,
      "logps/chosen": -412.3135681152344,
      "logps/rejected": -410.6241149902344,
      "loss": 0.292,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.6604477167129517,
      "rewards/margins": 1.9019458293914795,
      "rewards/rejected": -1.2414982318878174,
      "step": 1570
    },
    {
      "epoch": 2.585924713584288,
      "grad_norm": 4.983669757843018,
      "learning_rate": 8.065494238932688e-06,
      "logits/chosen": -1.5554888248443604,
      "logits/rejected": -1.4289400577545166,
      "logps/chosen": -402.4801025390625,
      "logps/rejected": -390.77130126953125,
      "loss": 0.3494,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9371005296707153,
      "rewards/margins": 1.8578503131866455,
      "rewards/rejected": -0.9207497835159302,
      "step": 1580
    },
    {
      "epoch": 2.602291325695581,
      "grad_norm": 2.0281553268432617,
      "learning_rate": 7.76228016979988e-06,
      "logits/chosen": -1.1428009271621704,
      "logits/rejected": -0.8446558713912964,
      "logps/chosen": -462.48895263671875,
      "logps/rejected": -451.9178161621094,
      "loss": 0.2745,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.9884368777275085,
      "rewards/margins": 2.026540756225586,
      "rewards/rejected": -1.0381040573120117,
      "step": 1590
    },
    {
      "epoch": 2.6186579378068737,
      "grad_norm": 2.423633337020874,
      "learning_rate": 7.459066100667071e-06,
      "logits/chosen": -1.3738397359848022,
      "logits/rejected": -1.1381474733352661,
      "logps/chosen": -450.98870849609375,
      "logps/rejected": -426.8583068847656,
      "loss": 0.2262,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0699903964996338,
      "rewards/margins": 2.1258344650268555,
      "rewards/rejected": -1.0558441877365112,
      "step": 1600
    },
    {
      "epoch": 2.635024549918167,
      "grad_norm": 4.554977893829346,
      "learning_rate": 7.155852031534263e-06,
      "logits/chosen": -1.4370437860488892,
      "logits/rejected": -1.2265866994857788,
      "logps/chosen": -460.3763732910156,
      "logps/rejected": -427.2916564941406,
      "loss": 0.2898,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.8689700961112976,
      "rewards/margins": 2.047645092010498,
      "rewards/rejected": -1.1786749362945557,
      "step": 1610
    },
    {
      "epoch": 2.65139116202946,
      "grad_norm": 5.240372180938721,
      "learning_rate": 6.852637962401456e-06,
      "logits/chosen": -1.2221819162368774,
      "logits/rejected": -0.9715522527694702,
      "logps/chosen": -443.93487548828125,
      "logps/rejected": -417.72918701171875,
      "loss": 0.2262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.062182903289795,
      "rewards/margins": 2.2453255653381348,
      "rewards/rejected": -1.1831427812576294,
      "step": 1620
    },
    {
      "epoch": 2.6677577741407528,
      "grad_norm": 2.34958553314209,
      "learning_rate": 6.549423893268648e-06,
      "logits/chosen": -1.2060081958770752,
      "logits/rejected": -1.2046623229980469,
      "logps/chosen": -474.5213317871094,
      "logps/rejected": -523.4539794921875,
      "loss": 0.2484,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.9350897669792175,
      "rewards/margins": 2.201655149459839,
      "rewards/rejected": -1.2665654420852661,
      "step": 1630
    },
    {
      "epoch": 2.684124386252046,
      "grad_norm": 1.095371961593628,
      "learning_rate": 6.24620982413584e-06,
      "logits/chosen": -1.0618741512298584,
      "logits/rejected": -0.699921727180481,
      "logps/chosen": -463.51556396484375,
      "logps/rejected": -431.7710876464844,
      "loss": 0.2054,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.197824239730835,
      "rewards/margins": 2.400318145751953,
      "rewards/rejected": -1.2024939060211182,
      "step": 1640
    },
    {
      "epoch": 2.700490998363339,
      "grad_norm": 2.7052650451660156,
      "learning_rate": 5.942995755003032e-06,
      "logits/chosen": -1.5151501893997192,
      "logits/rejected": -1.2694567441940308,
      "logps/chosen": -461.31048583984375,
      "logps/rejected": -468.01806640625,
      "loss": 0.2232,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1455790996551514,
      "rewards/margins": 2.2321972846984863,
      "rewards/rejected": -1.086618185043335,
      "step": 1650
    },
    {
      "epoch": 2.716857610474632,
      "grad_norm": 6.372529029846191,
      "learning_rate": 5.639781685870225e-06,
      "logits/chosen": -1.3566817045211792,
      "logits/rejected": -1.074830412864685,
      "logps/chosen": -443.8597717285156,
      "logps/rejected": -400.56842041015625,
      "loss": 0.367,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.7199681997299194,
      "rewards/margins": 1.740003228187561,
      "rewards/rejected": -1.020034909248352,
      "step": 1660
    },
    {
      "epoch": 2.7332242225859247,
      "grad_norm": 8.768135070800781,
      "learning_rate": 5.336567616737417e-06,
      "logits/chosen": -1.5840198993682861,
      "logits/rejected": -1.4000660181045532,
      "logps/chosen": -443.77056884765625,
      "logps/rejected": -431.5873107910156,
      "loss": 0.2731,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.0286372900009155,
      "rewards/margins": 2.149087429046631,
      "rewards/rejected": -1.1204503774642944,
      "step": 1670
    },
    {
      "epoch": 2.7495908346972175,
      "grad_norm": 2.6619386672973633,
      "learning_rate": 5.033353547604609e-06,
      "logits/chosen": -1.4543578624725342,
      "logits/rejected": -1.2476505041122437,
      "logps/chosen": -423.39764404296875,
      "logps/rejected": -421.17462158203125,
      "loss": 0.2493,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9351073503494263,
      "rewards/margins": 2.071972370147705,
      "rewards/rejected": -1.1368649005889893,
      "step": 1680
    },
    {
      "epoch": 2.7659574468085104,
      "grad_norm": 4.837337493896484,
      "learning_rate": 4.730139478471801e-06,
      "logits/chosen": -1.6075165271759033,
      "logits/rejected": NaN,
      "logps/chosen": -437.904052734375,
      "logps/rejected": -409.699951171875,
      "loss": 0.2651,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9201074838638306,
      "rewards/margins": 2.1968226432800293,
      "rewards/rejected": -1.2767151594161987,
      "step": 1690
    },
    {
      "epoch": 2.7823240589198037,
      "grad_norm": 7.250920295715332,
      "learning_rate": 4.426925409338994e-06,
      "logits/chosen": -1.3620741367340088,
      "logits/rejected": -1.1615064144134521,
      "logps/chosen": -470.387451171875,
      "logps/rejected": -418.77716064453125,
      "loss": 0.3151,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.0041719675064087,
      "rewards/margins": 1.8728843927383423,
      "rewards/rejected": -0.868712306022644,
      "step": 1700
    },
    {
      "epoch": 2.7986906710310966,
      "grad_norm": 5.2374982833862305,
      "learning_rate": 4.123711340206186e-06,
      "logits/chosen": -1.193421721458435,
      "logits/rejected": -1.2663942575454712,
      "logps/chosen": -432.0567932128906,
      "logps/rejected": -416.0810546875,
      "loss": 0.3065,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.8853948712348938,
      "rewards/margins": 1.7219581604003906,
      "rewards/rejected": -0.8365634679794312,
      "step": 1710
    },
    {
      "epoch": 2.8150572831423895,
      "grad_norm": 6.9263224601745605,
      "learning_rate": 3.8204972710733785e-06,
      "logits/chosen": -1.42405366897583,
      "logits/rejected": -1.272728681564331,
      "logps/chosen": -438.894287109375,
      "logps/rejected": -394.73284912109375,
      "loss": 0.2415,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0647566318511963,
      "rewards/margins": 2.0775082111358643,
      "rewards/rejected": -1.0127513408660889,
      "step": 1720
    },
    {
      "epoch": 2.8314238952536823,
      "grad_norm": 2.6749045848846436,
      "learning_rate": 3.51728320194057e-06,
      "logits/chosen": -1.2201988697052002,
      "logits/rejected": -0.9426723718643188,
      "logps/chosen": -474.87530517578125,
      "logps/rejected": -410.86798095703125,
      "loss": 0.2664,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1347922086715698,
      "rewards/margins": 2.3549981117248535,
      "rewards/rejected": -1.2202056646347046,
      "step": 1730
    },
    {
      "epoch": 2.8477905073649756,
      "grad_norm": 4.696692943572998,
      "learning_rate": 3.214069132807762e-06,
      "logits/chosen": -1.124267339706421,
      "logits/rejected": -0.9841214418411255,
      "logps/chosen": -530.7108154296875,
      "logps/rejected": -499.8089904785156,
      "loss": 0.2433,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.319337010383606,
      "rewards/margins": 2.231715202331543,
      "rewards/rejected": -0.9123780131340027,
      "step": 1740
    },
    {
      "epoch": 2.8641571194762685,
      "grad_norm": 2.817408323287964,
      "learning_rate": 2.910855063674955e-06,
      "logits/chosen": -1.3371939659118652,
      "logits/rejected": -1.0595276355743408,
      "logps/chosen": -457.59234619140625,
      "logps/rejected": -451.3644104003906,
      "loss": 0.2328,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0093997716903687,
      "rewards/margins": 2.203674554824829,
      "rewards/rejected": -1.194274663925171,
      "step": 1750
    },
    {
      "epoch": 2.8805237315875614,
      "grad_norm": 1.328909993171692,
      "learning_rate": 2.607640994542147e-06,
      "logits/chosen": -1.3779757022857666,
      "logits/rejected": -1.2147364616394043,
      "logps/chosen": -431.8528747558594,
      "logps/rejected": -456.26727294921875,
      "loss": 0.2552,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.9310500025749207,
      "rewards/margins": 2.0603978633880615,
      "rewards/rejected": -1.1293480396270752,
      "step": 1760
    },
    {
      "epoch": 2.8968903436988542,
      "grad_norm": 2.1010754108428955,
      "learning_rate": 2.3044269254093393e-06,
      "logits/chosen": -1.59529447555542,
      "logits/rejected": -1.4413836002349854,
      "logps/chosen": -441.6121520996094,
      "logps/rejected": -462.1275939941406,
      "loss": 0.2184,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.1032123565673828,
      "rewards/margins": 2.270881175994873,
      "rewards/rejected": -1.1676688194274902,
      "step": 1770
    },
    {
      "epoch": 2.913256955810147,
      "grad_norm": 2.0799882411956787,
      "learning_rate": 2.0012128562765313e-06,
      "logits/chosen": -1.1777410507202148,
      "logits/rejected": -1.0771524906158447,
      "logps/chosen": -493.805419921875,
      "logps/rejected": -447.61468505859375,
      "loss": 0.2117,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1594716310501099,
      "rewards/margins": 2.220271587371826,
      "rewards/rejected": -1.0607998371124268,
      "step": 1780
    },
    {
      "epoch": 2.9296235679214404,
      "grad_norm": 5.403112888336182,
      "learning_rate": 1.6979987871437233e-06,
      "logits/chosen": -1.5264140367507935,
      "logits/rejected": -1.4546772241592407,
      "logps/chosen": -421.8680725097656,
      "logps/rejected": -406.50579833984375,
      "loss": 0.2935,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.9573639035224915,
      "rewards/margins": 1.8112157583236694,
      "rewards/rejected": -0.8538519144058228,
      "step": 1790
    },
    {
      "epoch": 2.9459901800327333,
      "grad_norm": 2.95246958732605,
      "learning_rate": 1.394784718010916e-06,
      "logits/chosen": -1.1958624124526978,
      "logits/rejected": -0.8615037798881531,
      "logps/chosen": -402.82305908203125,
      "logps/rejected": -379.9725341796875,
      "loss": 0.2734,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.86333829164505,
      "rewards/margins": 2.0156140327453613,
      "rewards/rejected": -1.152275800704956,
      "step": 1800
    },
    {
      "epoch": 2.962356792144026,
      "grad_norm": 2.8880884647369385,
      "learning_rate": 1.091570648878108e-06,
      "logits/chosen": -1.4118956327438354,
      "logits/rejected": -1.0699918270111084,
      "logps/chosen": -404.4219665527344,
      "logps/rejected": -381.11639404296875,
      "loss": 0.2644,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.8657559156417847,
      "rewards/margins": 2.258451223373413,
      "rewards/rejected": -1.392695665359497,
      "step": 1810
    },
    {
      "epoch": 2.978723404255319,
      "grad_norm": 1.423218011856079,
      "learning_rate": 7.883565797453002e-07,
      "logits/chosen": -1.5209050178527832,
      "logits/rejected": -1.1870254278182983,
      "logps/chosen": -393.4710998535156,
      "logps/rejected": -393.87353515625,
      "loss": 0.2505,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.913676917552948,
      "rewards/margins": 2.0497212409973145,
      "rewards/rejected": -1.1360442638397217,
      "step": 1820
    },
    {
      "epoch": 2.9950900163666123,
      "grad_norm": 9.121944427490234,
      "learning_rate": 4.851425106124925e-07,
      "logits/chosen": -1.419418215751648,
      "logits/rejected": -1.2076327800750732,
      "logps/chosen": -436.1336364746094,
      "logps/rejected": -481.68548583984375,
      "loss": 0.2557,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.0021239519119263,
      "rewards/margins": 2.3086962699890137,
      "rewards/rejected": -1.3065721988677979,
      "step": 1830
    }
  ],
  "logging_steps": 10,
  "max_steps": 1833,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}