{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.6366612111292962,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016366612111292964,
      "grad_norm": NaN,
      "learning_rate": 1.6304347826086957e-06,
      "logits/chosen": -1.2515310049057007,
      "logits/rejected": -0.8573959469795227,
      "logps/chosen": -455.1468811035156,
      "logps/rejected": -387.0008239746094,
      "loss": 0.6923,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0002114105736836791,
      "rewards/margins": 0.00180077797267586,
      "rewards/rejected": -0.0015893673989921808,
      "step": 10
    },
    {
      "epoch": 0.03273322422258593,
      "grad_norm": 10.154256820678711,
      "learning_rate": 3.2608695652173914e-06,
      "logits/chosen": -1.145971417427063,
      "logits/rejected": -1.082183837890625,
      "logps/chosen": -446.249755859375,
      "logps/rejected": -426.87237548828125,
      "loss": 0.6942,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0005732345744036138,
      "rewards/margins": -0.002082281280308962,
      "rewards/rejected": 0.0015090465312823653,
      "step": 20
    },
    {
      "epoch": 0.049099836333878884,
      "grad_norm": 8.393162727355957,
      "learning_rate": 5.4347826086956525e-06,
      "logits/chosen": -0.9614235162734985,
      "logits/rejected": -0.8527517318725586,
      "logps/chosen": -460.367431640625,
      "logps/rejected": -433.61285400390625,
      "loss": 0.6934,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0062791816890239716,
      "rewards/margins": -0.00045429245801642537,
      "rewards/rejected": 0.006733474787324667,
      "step": 30
    },
    {
      "epoch": 0.06546644844517185,
      "grad_norm": 9.398829460144043,
      "learning_rate": 8.15217391304348e-06,
      "logits/chosen": -1.171631097793579,
      "logits/rejected": -1.0616101026535034,
      "logps/chosen": -421.583984375,
      "logps/rejected": -424.787109375,
      "loss": 0.6902,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.020799484103918076,
      "rewards/margins": 0.006221642717719078,
      "rewards/rejected": 0.014577841386198997,
      "step": 40
    },
    {
      "epoch": 0.08183306055646482,
      "grad_norm": 9.43005657196045,
      "learning_rate": 1.0869565217391305e-05,
      "logits/chosen": -1.2450587749481201,
      "logits/rejected": -1.1417832374572754,
      "logps/chosen": -476.83538818359375,
      "logps/rejected": -444.6556091308594,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.042164258658885956,
      "rewards/margins": 0.010725794360041618,
      "rewards/rejected": 0.03143846243619919,
      "step": 50
    },
    {
      "epoch": 0.09819967266775777,
      "grad_norm": 8.825221061706543,
      "learning_rate": 1.331521739130435e-05,
      "logits/chosen": -1.3223823308944702,
      "logits/rejected": NaN,
      "logps/chosen": -383.4811706542969,
      "logps/rejected": -454.43780517578125,
      "loss": 0.6965,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.03668512776494026,
      "rewards/margins": -0.003995636478066444,
      "rewards/rejected": 0.04068076238036156,
      "step": 60
    },
    {
      "epoch": 0.11456628477905073,
      "grad_norm": 9.5093994140625,
      "learning_rate": 1.6032608695652173e-05,
      "logits/chosen": -1.2538468837738037,
      "logits/rejected": -1.2771422863006592,
      "logps/chosen": -468.79400634765625,
      "logps/rejected": -402.565673828125,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04952714219689369,
      "rewards/margins": 0.01824185810983181,
      "rewards/rejected": 0.03128528222441673,
      "step": 70
    },
    {
      "epoch": 0.1309328968903437,
      "grad_norm": 7.945922374725342,
      "learning_rate": 1.8750000000000002e-05,
      "logits/chosen": -1.386731743812561,
      "logits/rejected": -0.8966825604438782,
      "logps/chosen": -446.54736328125,
      "logps/rejected": -430.45025634765625,
      "loss": 0.6719,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.06767664849758148,
      "rewards/margins": 0.048766594380140305,
      "rewards/rejected": 0.01891004480421543,
      "step": 80
    },
    {
      "epoch": 0.14729950900163666,
      "grad_norm": 8.496461868286133,
      "learning_rate": 2.1467391304347828e-05,
      "logits/chosen": -1.3730494976043701,
      "logits/rejected": -1.1826181411743164,
      "logps/chosen": -413.87835693359375,
      "logps/rejected": -385.1138916015625,
      "loss": 0.69,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.07122664153575897,
      "rewards/margins": 0.014941570349037647,
      "rewards/rejected": 0.0562850646674633,
      "step": 90
    },
    {
      "epoch": 0.16366612111292964,
      "grad_norm": 10.744088172912598,
      "learning_rate": 2.4184782608695653e-05,
      "logits/chosen": -1.2254825830459595,
      "logits/rejected": -1.1136434078216553,
      "logps/chosen": -495.34295654296875,
      "logps/rejected": -435.0003967285156,
      "loss": 0.6618,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.1682208627462387,
      "rewards/margins": 0.08198796212673187,
      "rewards/rejected": 0.08623288571834564,
      "step": 100
    },
    {
      "epoch": 0.18003273322422259,
      "grad_norm": 11.016289710998535,
      "learning_rate": 2.6902173913043476e-05,
      "logits/chosen": -1.3138881921768188,
      "logits/rejected": -1.0289314985275269,
      "logps/chosen": -476.26153564453125,
      "logps/rejected": -426.837646484375,
      "loss": 0.6355,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.21744687855243683,
      "rewards/margins": 0.15108919143676758,
      "rewards/rejected": 0.06635768711566925,
      "step": 110
    },
    {
      "epoch": 0.19639934533551553,
      "grad_norm": 11.619248390197754,
      "learning_rate": 2.9619565217391305e-05,
      "logits/chosen": -1.2751747369766235,
      "logits/rejected": -1.1858090162277222,
      "logps/chosen": -444.4823303222656,
      "logps/rejected": -404.58123779296875,
      "loss": 0.6716,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.2203199863433838,
      "rewards/margins": 0.06618332862854004,
      "rewards/rejected": 0.15413665771484375,
      "step": 120
    },
    {
      "epoch": 0.2127659574468085,
      "grad_norm": 8.900009155273438,
      "learning_rate": 3.233695652173913e-05,
      "logits/chosen": -1.2365785837173462,
      "logits/rejected": -1.2031922340393066,
      "logps/chosen": -477.8089294433594,
      "logps/rejected": -456.80120849609375,
      "loss": 0.6794,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.20705166459083557,
      "rewards/margins": 0.060810189694166183,
      "rewards/rejected": 0.1462414562702179,
      "step": 130
    },
    {
      "epoch": 0.22913256955810146,
      "grad_norm": 8.928816795349121,
      "learning_rate": 3.505434782608696e-05,
      "logits/chosen": -1.2421447038650513,
      "logits/rejected": -1.136252522468567,
      "logps/chosen": -547.4909057617188,
      "logps/rejected": -460.38275146484375,
      "loss": 0.6047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.21343636512756348,
      "rewards/margins": 0.24689535796642303,
      "rewards/rejected": -0.033459004014730453,
      "step": 140
    },
    {
      "epoch": 0.24549918166939444,
      "grad_norm": 11.977150917053223,
      "learning_rate": 3.777173913043478e-05,
      "logits/chosen": -1.338332176208496,
      "logits/rejected": -1.1015567779541016,
      "logps/chosen": -432.2693786621094,
      "logps/rejected": -423.0946350097656,
      "loss": 0.6946,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.15119442343711853,
      "rewards/margins": 0.07870660722255707,
      "rewards/rejected": 0.07248781621456146,
      "step": 150
    },
    {
      "epoch": 0.2618657937806874,
      "grad_norm": 13.040871620178223,
      "learning_rate": 4.0489130434782615e-05,
      "logits/chosen": -1.0111101865768433,
      "logits/rejected": NaN,
      "logps/chosen": -472.6917419433594,
      "logps/rejected": -426.6333923339844,
      "loss": 0.6598,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.24765905737876892,
      "rewards/margins": 0.13647916913032532,
      "rewards/rejected": 0.11117993295192719,
      "step": 160
    },
    {
      "epoch": 0.27823240589198034,
      "grad_norm": 8.028325080871582,
      "learning_rate": 4.3206521739130434e-05,
      "logits/chosen": -0.9152072668075562,
      "logits/rejected": -0.8340185284614563,
      "logps/chosen": -438.38555908203125,
      "logps/rejected": -392.93426513671875,
      "loss": 0.6272,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.3285876512527466,
      "rewards/margins": 0.20840699970722198,
      "rewards/rejected": 0.1201806515455246,
      "step": 170
    },
    {
      "epoch": 0.2945990180032733,
      "grad_norm": 9.436712265014648,
      "learning_rate": 4.592391304347826e-05,
      "logits/chosen": -1.0703661441802979,
      "logits/rejected": -1.0127884149551392,
      "logps/chosen": -449.83905029296875,
      "logps/rejected": -437.92822265625,
      "loss": 0.6684,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.34186336398124695,
      "rewards/margins": 0.14258861541748047,
      "rewards/rejected": 0.19927474856376648,
      "step": 180
    },
    {
      "epoch": 0.3109656301145663,
      "grad_norm": 8.914007186889648,
      "learning_rate": 4.8641304347826086e-05,
      "logits/chosen": -0.8762575387954712,
      "logits/rejected": -0.5631248354911804,
      "logps/chosen": -473.806396484375,
      "logps/rejected": -443.6448669433594,
      "loss": 0.6243,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.3772050440311432,
      "rewards/margins": 0.24287982285022736,
      "rewards/rejected": 0.13432522118091583,
      "step": 190
    },
    {
      "epoch": 0.32733224222585927,
      "grad_norm": 14.76502513885498,
      "learning_rate": 4.98483929654336e-05,
      "logits/chosen": NaN,
      "logits/rejected": -0.9821416139602661,
      "logps/chosen": -447.65386962890625,
      "logps/rejected": -400.1768493652344,
      "loss": 0.6805,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.2871859073638916,
      "rewards/margins": 0.17125040292739868,
      "rewards/rejected": 0.11593550443649292,
      "step": 200
    },
    {
      "epoch": 0.3436988543371522,
      "grad_norm": 11.80213451385498,
      "learning_rate": 4.954517889630079e-05,
      "logits/chosen": -1.2535641193389893,
      "logits/rejected": -1.0489479303359985,
      "logps/chosen": -457.010986328125,
      "logps/rejected": -432.8558044433594,
      "loss": 0.6769,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.24471895396709442,
      "rewards/margins": 0.16205154359340668,
      "rewards/rejected": 0.08266742527484894,
      "step": 210
    },
    {
      "epoch": 0.36006546644844517,
      "grad_norm": 7.534430980682373,
      "learning_rate": 4.924196482716799e-05,
      "logits/chosen": -1.5050650835037231,
      "logits/rejected": -1.5133517980575562,
      "logps/chosen": -445.3924865722656,
      "logps/rejected": -420.99114990234375,
      "loss": 0.7003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.18017885088920593,
      "rewards/margins": 0.1387586146593094,
      "rewards/rejected": 0.04142025485634804,
      "step": 220
    },
    {
      "epoch": 0.37643207855973815,
      "grad_norm": 7.207632541656494,
      "learning_rate": 4.893875075803517e-05,
      "logits/chosen": -1.3496973514556885,
      "logits/rejected": -1.2030962705612183,
      "logps/chosen": -436.5039978027344,
      "logps/rejected": -396.91534423828125,
      "loss": 0.6192,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.19479091465473175,
      "rewards/margins": 0.353401243686676,
      "rewards/rejected": -0.15861034393310547,
      "step": 230
    },
    {
      "epoch": 0.39279869067103107,
      "grad_norm": 9.134820938110352,
      "learning_rate": 4.8665858095815645e-05,
      "logits/chosen": -1.2978178262710571,
      "logits/rejected": -1.1864020824432373,
      "logps/chosen": -486.7569885253906,
      "logps/rejected": -457.94586181640625,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.29108190536499023,
      "rewards/margins": 0.15347807109355927,
      "rewards/rejected": 0.13760384917259216,
      "step": 240
    },
    {
      "epoch": 0.40916530278232405,
      "grad_norm": 10.825005531311035,
      "learning_rate": 4.836264402668284e-05,
      "logits/chosen": -1.3271738290786743,
      "logits/rejected": -1.030890703201294,
      "logps/chosen": -421.89349365234375,
      "logps/rejected": -400.1674499511719,
      "loss": 0.6536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.42532652616500854,
      "rewards/margins": 0.34351587295532227,
      "rewards/rejected": 0.08181065320968628,
      "step": 250
    },
    {
      "epoch": 0.425531914893617,
      "grad_norm": 9.450189590454102,
      "learning_rate": 4.8059429957550034e-05,
      "logits/chosen": -1.0212231874465942,
      "logits/rejected": -1.0058345794677734,
      "logps/chosen": -422.1316833496094,
      "logps/rejected": -390.0425109863281,
      "loss": 0.6384,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.490867555141449,
      "rewards/margins": 0.35965341329574585,
      "rewards/rejected": 0.13121409714221954,
      "step": 260
    },
    {
      "epoch": 0.44189852700491,
      "grad_norm": 8.231124877929688,
      "learning_rate": 4.7756215888417225e-05,
      "logits/chosen": -0.9735445976257324,
      "logits/rejected": -0.937743067741394,
      "logps/chosen": -552.4168701171875,
      "logps/rejected": -501.0535583496094,
      "loss": 0.6694,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.4536367952823639,
      "rewards/margins": 0.19916944205760956,
      "rewards/rejected": 0.2544674277305603,
      "step": 270
    },
    {
      "epoch": 0.4582651391162029,
      "grad_norm": 9.207930564880371,
      "learning_rate": 4.7453001819284416e-05,
      "logits/chosen": -0.9774928092956543,
      "logits/rejected": -0.8485246896743774,
      "logps/chosen": -444.54827880859375,
      "logps/rejected": -419.5541076660156,
      "loss": 0.6607,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.3690536618232727,
      "rewards/margins": 0.22144463658332825,
      "rewards/rejected": 0.14760902523994446,
      "step": 280
    },
    {
      "epoch": 0.4746317512274959,
      "grad_norm": 6.876308441162109,
      "learning_rate": 4.714978775015161e-05,
      "logits/chosen": -1.2095867395401,
      "logits/rejected": -0.8773800730705261,
      "logps/chosen": -437.869384765625,
      "logps/rejected": -419.9366149902344,
      "loss": 0.5867,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.5094749331474304,
      "rewards/margins": 0.5250319242477417,
      "rewards/rejected": -0.015556998550891876,
      "step": 290
    },
    {
      "epoch": 0.4909983633387889,
      "grad_norm": 12.918950080871582,
      "learning_rate": 4.68465736810188e-05,
      "logits/chosen": -0.9076827764511108,
      "logits/rejected": -0.7181829214096069,
      "logps/chosen": -469.4864196777344,
      "logps/rejected": -420.38885498046875,
      "loss": 0.6833,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.5444287657737732,
      "rewards/margins": 0.2743019461631775,
      "rewards/rejected": 0.2701267600059509,
      "step": 300
    },
    {
      "epoch": 0.5073649754500819,
      "grad_norm": 4.9246110916137695,
      "learning_rate": 4.6543359611885996e-05,
      "logits/chosen": -1.214247226715088,
      "logits/rejected": -0.9910456538200378,
      "logps/chosen": -385.4250183105469,
      "logps/rejected": -390.26043701171875,
      "loss": 0.6419,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.4199371337890625,
      "rewards/margins": 0.34462255239486694,
      "rewards/rejected": 0.07531457394361496,
      "step": 310
    },
    {
      "epoch": 0.5237315875613748,
      "grad_norm": 8.829249382019043,
      "learning_rate": 4.624014554275319e-05,
      "logits/chosen": -1.0396336317062378,
      "logits/rejected": -0.9439867734909058,
      "logps/chosen": -452.1678161621094,
      "logps/rejected": -379.70745849609375,
      "loss": 0.6396,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.472791850566864,
      "rewards/margins": 0.23509594798088074,
      "rewards/rejected": 0.23769588768482208,
      "step": 320
    },
    {
      "epoch": 0.5400981996726678,
      "grad_norm": 13.249287605285645,
      "learning_rate": 4.593693147362038e-05,
      "logits/chosen": -0.927478015422821,
      "logits/rejected": -0.7090943455696106,
      "logps/chosen": -488.009765625,
      "logps/rejected": -473.47833251953125,
      "loss": 0.6833,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.4911763668060303,
      "rewards/margins": 0.25351986289024353,
      "rewards/rejected": 0.23765654861927032,
      "step": 330
    },
    {
      "epoch": 0.5564648117839607,
      "grad_norm": 9.580172538757324,
      "learning_rate": 4.566403881140085e-05,
      "logits/chosen": -0.9790302515029907,
      "logits/rejected": -0.8282285928726196,
      "logps/chosen": -481.6971740722656,
      "logps/rejected": -441.3619689941406,
      "loss": 0.6922,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.4195849299430847,
      "rewards/margins": 0.210377499461174,
      "rewards/rejected": 0.20920750498771667,
      "step": 340
    },
    {
      "epoch": 0.5728314238952537,
      "grad_norm": 6.872241973876953,
      "learning_rate": 4.536082474226804e-05,
      "logits/chosen": -0.9942235946655273,
      "logits/rejected": -0.824662983417511,
      "logps/chosen": -437.9869079589844,
      "logps/rejected": -417.3473205566406,
      "loss": 0.6071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.49881643056869507,
      "rewards/margins": 0.41778793931007385,
      "rewards/rejected": 0.08102855831384659,
      "step": 350
    },
    {
      "epoch": 0.5891980360065466,
      "grad_norm": 7.748345375061035,
      "learning_rate": 4.5057610673135234e-05,
      "logits/chosen": -1.0704625844955444,
      "logits/rejected": -0.7879603505134583,
      "logps/chosen": -428.53021240234375,
      "logps/rejected": -435.9026794433594,
      "loss": 0.6084,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.4484317898750305,
      "rewards/margins": 0.4125775396823883,
      "rewards/rejected": 0.035854239016771317,
      "step": 360
    },
    {
      "epoch": 0.6055646481178396,
      "grad_norm": 7.428864479064941,
      "learning_rate": 4.475439660400243e-05,
      "logits/chosen": -1.098026990890503,
      "logits/rejected": -0.9232645034790039,
      "logps/chosen": -441.7071228027344,
      "logps/rejected": -402.95770263671875,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.37514036893844604,
      "rewards/margins": 0.2525361478328705,
      "rewards/rejected": 0.12260414659976959,
      "step": 370
    },
    {
      "epoch": 0.6219312602291326,
      "grad_norm": 9.591557502746582,
      "learning_rate": 4.445118253486962e-05,
      "logits/chosen": -1.2409312725067139,
      "logits/rejected": -0.8442489504814148,
      "logps/chosen": -466.13311767578125,
      "logps/rejected": -420.9102478027344,
      "loss": 0.7684,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.2286566197872162,
      "rewards/margins": 0.11457403004169464,
      "rewards/rejected": 0.11408257484436035,
      "step": 380
    },
    {
      "epoch": 0.6382978723404256,
      "grad_norm": 7.025293350219727,
      "learning_rate": 4.414796846573681e-05,
      "logits/chosen": -1.1844158172607422,
      "logits/rejected": -1.1069843769073486,
      "logps/chosen": -428.5450134277344,
      "logps/rejected": -371.85614013671875,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.2637328803539276,
      "rewards/margins": 0.10889965295791626,
      "rewards/rejected": 0.15483322739601135,
      "step": 390
    },
    {
      "epoch": 0.6546644844517185,
      "grad_norm": 19.2535343170166,
      "learning_rate": 4.3844754396604006e-05,
      "logits/chosen": -1.1141570806503296,
      "logits/rejected": -0.8798454999923706,
      "logps/chosen": -493.08197021484375,
      "logps/rejected": -465.0943298339844,
      "loss": 0.6545,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.4174181818962097,
      "rewards/margins": 0.25259822607040405,
      "rewards/rejected": 0.16481998562812805,
      "step": 400
    },
    {
      "epoch": 0.6710310965630114,
      "grad_norm": 8.808697700500488,
      "learning_rate": 4.35415403274712e-05,
      "logits/chosen": -0.9878439903259277,
      "logits/rejected": -0.7299278974533081,
      "logps/chosen": -523.9139404296875,
      "logps/rejected": -482.0597229003906,
      "loss": 0.552,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.4945390820503235,
      "rewards/margins": 0.6175243854522705,
      "rewards/rejected": -0.1229853481054306,
      "step": 410
    },
    {
      "epoch": 0.6873977086743044,
      "grad_norm": 9.99456787109375,
      "learning_rate": 4.323832625833839e-05,
      "logits/chosen": -1.0902565717697144,
      "logits/rejected": -0.9221500158309937,
      "logps/chosen": -443.0076599121094,
      "logps/rejected": -375.8013610839844,
      "loss": 0.5684,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.46177297830581665,
      "rewards/margins": 0.509793221950531,
      "rewards/rejected": -0.048020295798778534,
      "step": 420
    },
    {
      "epoch": 0.7037643207855974,
      "grad_norm": 5.767345428466797,
      "learning_rate": 4.293511218920558e-05,
      "logits/chosen": -1.0450439453125,
      "logits/rejected": -0.7902520298957825,
      "logps/chosen": -458.2330017089844,
      "logps/rejected": -382.6787109375,
      "loss": 0.6126,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.5793684124946594,
      "rewards/margins": 0.35584205389022827,
      "rewards/rejected": 0.22352632880210876,
      "step": 430
    },
    {
      "epoch": 0.7201309328968903,
      "grad_norm": 11.51710033416748,
      "learning_rate": 4.263189812007277e-05,
      "logits/chosen": -0.8710979223251343,
      "logits/rejected": -0.783151388168335,
      "logps/chosen": -451.32122802734375,
      "logps/rejected": -416.0005798339844,
      "loss": 0.6751,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.5284687280654907,
      "rewards/margins": 0.2338237315416336,
      "rewards/rejected": 0.2946449816226959,
      "step": 440
    },
    {
      "epoch": 0.7364975450081833,
      "grad_norm": 8.408599853515625,
      "learning_rate": 4.232868405093997e-05,
      "logits/chosen": -0.9611997604370117,
      "logits/rejected": -0.8005222082138062,
      "logps/chosen": -390.06817626953125,
      "logps/rejected": -376.57183837890625,
      "loss": 0.723,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.35626164078712463,
      "rewards/margins": 0.07882154732942581,
      "rewards/rejected": 0.27744007110595703,
      "step": 450
    },
    {
      "epoch": 0.7528641571194763,
      "grad_norm": 7.085683822631836,
      "learning_rate": 4.202546998180716e-05,
      "logits/chosen": -1.181065559387207,
      "logits/rejected": -0.9188621640205383,
      "logps/chosen": -446.1695861816406,
      "logps/rejected": -461.0537109375,
      "loss": 0.6376,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.6099992990493774,
      "rewards/margins": 0.40477481484413147,
      "rewards/rejected": 0.20522448420524597,
      "step": 460
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 7.778029918670654,
      "learning_rate": 4.172225591267435e-05,
      "logits/chosen": -1.1612136363983154,
      "logits/rejected": -0.7909379601478577,
      "logps/chosen": -415.356201171875,
      "logps/rejected": -405.0163879394531,
      "loss": 0.639,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.5554541349411011,
      "rewards/margins": 0.49387890100479126,
      "rewards/rejected": 0.06157524511218071,
      "step": 470
    },
    {
      "epoch": 0.7855973813420621,
      "grad_norm": 9.144447326660156,
      "learning_rate": 4.141904184354154e-05,
      "logits/chosen": -1.1227318048477173,
      "logits/rejected": -0.8980533480644226,
      "logps/chosen": -470.1067810058594,
      "logps/rejected": -385.7353210449219,
      "loss": 0.6301,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.5011923909187317,
      "rewards/margins": 0.36913150548934937,
      "rewards/rejected": 0.1320609152317047,
      "step": 480
    },
    {
      "epoch": 0.8019639934533551,
      "grad_norm": 9.528294563293457,
      "learning_rate": 4.111582777440873e-05,
      "logits/chosen": -1.4198505878448486,
      "logits/rejected": -1.1528406143188477,
      "logps/chosen": -430.28668212890625,
      "logps/rejected": -385.2845764160156,
      "loss": 0.634,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.6369161605834961,
      "rewards/margins": 0.4485607147216797,
      "rewards/rejected": 0.18835540115833282,
      "step": 490
    },
    {
      "epoch": 0.8183306055646481,
      "grad_norm": 6.964690208435059,
      "learning_rate": 4.0812613705275924e-05,
      "logits/chosen": -1.1329933404922485,
      "logits/rejected": -1.066542625427246,
      "logps/chosen": -423.4967346191406,
      "logps/rejected": -376.92803955078125,
      "loss": 0.5904,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.6795145273208618,
      "rewards/margins": 0.3617789149284363,
      "rewards/rejected": 0.3177356719970703,
      "step": 500
    },
    {
      "epoch": 0.8346972176759411,
      "grad_norm": 11.78022289276123,
      "learning_rate": 4.050939963614312e-05,
      "logits/chosen": -1.1565020084381104,
      "logits/rejected": -1.064300298690796,
      "logps/chosen": -465.0497131347656,
      "logps/rejected": -438.4883728027344,
      "loss": 0.6384,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.6123272180557251,
      "rewards/margins": 0.3706541657447815,
      "rewards/rejected": 0.241673082113266,
      "step": 510
    },
    {
      "epoch": 0.851063829787234,
      "grad_norm": 6.378419876098633,
      "learning_rate": 4.020618556701031e-05,
      "logits/chosen": -1.0936065912246704,
      "logits/rejected": -1.157414436340332,
      "logps/chosen": -431.05657958984375,
      "logps/rejected": -380.1803283691406,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.514751672744751,
      "rewards/margins": 0.2915990948677063,
      "rewards/rejected": 0.22315259277820587,
      "step": 520
    },
    {
      "epoch": 0.867430441898527,
      "grad_norm": 6.811694145202637,
      "learning_rate": 3.9902971497877504e-05,
      "logits/chosen": -1.0564637184143066,
      "logits/rejected": -0.8537761569023132,
      "logps/chosen": -400.8801574707031,
      "logps/rejected": -400.9678039550781,
      "loss": 0.6036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5905786156654358,
      "rewards/margins": 0.49240899085998535,
      "rewards/rejected": 0.09816964715719223,
      "step": 530
    },
    {
      "epoch": 0.88379705400982,
      "grad_norm": 8.516433715820312,
      "learning_rate": 3.9599757428744695e-05,
      "logits/chosen": -1.0425868034362793,
      "logits/rejected": -1.0106391906738281,
      "logps/chosen": -494.23681640625,
      "logps/rejected": -451.3269958496094,
      "loss": 0.7295,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.480445921421051,
      "rewards/margins": 0.12438355386257172,
      "rewards/rejected": 0.3560623824596405,
      "step": 540
    },
    {
      "epoch": 0.900163666121113,
      "grad_norm": 9.584715843200684,
      "learning_rate": 3.9296543359611886e-05,
      "logits/chosen": -1.1160447597503662,
      "logits/rejected": -0.898239016532898,
      "logps/chosen": -471.17718505859375,
      "logps/rejected": -404.9838562011719,
      "loss": 0.5973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4906229078769684,
      "rewards/margins": 0.3674588203430176,
      "rewards/rejected": 0.1231641173362732,
      "step": 550
    },
    {
      "epoch": 0.9165302782324058,
      "grad_norm": 9.050358772277832,
      "learning_rate": 3.899332929047908e-05,
      "logits/chosen": -1.0188591480255127,
      "logits/rejected": NaN,
      "logps/chosen": -505.81146240234375,
      "logps/rejected": -455.074462890625,
      "loss": 0.672,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.5001901388168335,
      "rewards/margins": 0.3212779462337494,
      "rewards/rejected": 0.1789121776819229,
      "step": 560
    },
    {
      "epoch": 0.9328968903436988,
      "grad_norm": 10.636481285095215,
      "learning_rate": 3.8690115221346276e-05,
      "logits/chosen": -1.0234591960906982,
      "logits/rejected": -0.9390643835067749,
      "logps/chosen": -423.03594970703125,
      "logps/rejected": -383.97369384765625,
      "loss": 0.6515,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.40482932329177856,
      "rewards/margins": 0.20538429915905,
      "rewards/rejected": 0.19944503903388977,
      "step": 570
    },
    {
      "epoch": 0.9492635024549918,
      "grad_norm": 12.001391410827637,
      "learning_rate": 3.838690115221347e-05,
      "logits/chosen": -1.254690170288086,
      "logits/rejected": -1.098749041557312,
      "logps/chosen": -400.72021484375,
      "logps/rejected": -348.39678955078125,
      "loss": 0.6332,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.5642321705818176,
      "rewards/margins": 0.34682947397232056,
      "rewards/rejected": 0.21740269660949707,
      "step": 580
    },
    {
      "epoch": 0.9656301145662848,
      "grad_norm": 9.584880828857422,
      "learning_rate": 3.808368708308066e-05,
      "logits/chosen": -0.9567238092422485,
      "logits/rejected": -0.8445194959640503,
      "logps/chosen": -475.3368225097656,
      "logps/rejected": -418.3975524902344,
      "loss": 0.609,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.7818068861961365,
      "rewards/margins": 0.4438362717628479,
      "rewards/rejected": 0.3379705548286438,
      "step": 590
    },
    {
      "epoch": 0.9819967266775778,
      "grad_norm": 8.225208282470703,
      "learning_rate": 3.778047301394785e-05,
      "logits/chosen": -0.8194648623466492,
      "logits/rejected": -0.6516283750534058,
      "logps/chosen": -464.95648193359375,
      "logps/rejected": -454.74896240234375,
      "loss": 0.6792,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.673040509223938,
      "rewards/margins": 0.3073888421058655,
      "rewards/rejected": 0.3656516671180725,
      "step": 600
    },
    {
      "epoch": 0.9983633387888707,
      "grad_norm": 7.938261032104492,
      "learning_rate": 3.747725894481504e-05,
      "logits/chosen": -0.9632278680801392,
      "logits/rejected": -0.7192381620407104,
      "logps/chosen": -428.2535095214844,
      "logps/rejected": -426.05780029296875,
      "loss": 0.5784,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.778376042842865,
      "rewards/margins": 0.44757041335105896,
      "rewards/rejected": 0.33080559968948364,
      "step": 610
    },
    {
      "epoch": 1.0147299509001637,
      "grad_norm": 6.6163482666015625,
      "learning_rate": 3.717404487568223e-05,
      "logits/chosen": -0.8798831105232239,
      "logits/rejected": -0.716325581073761,
      "logps/chosen": -401.0724182128906,
      "logps/rejected": -393.04925537109375,
      "loss": 0.5397,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.6812083721160889,
      "rewards/margins": 0.6590946912765503,
      "rewards/rejected": 0.022113658487796783,
      "step": 620
    },
    {
      "epoch": 1.0310965630114566,
      "grad_norm": 4.5942792892456055,
      "learning_rate": 3.687083080654943e-05,
      "logits/chosen": -0.595669150352478,
      "logits/rejected": -0.6483780741691589,
      "logps/chosen": -519.7013549804688,
      "logps/rejected": -459.6736755371094,
      "loss": 0.3798,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9690455198287964,
      "rewards/margins": 1.2637821435928345,
      "rewards/rejected": -0.2947366237640381,
      "step": 630
    },
    {
      "epoch": 1.0474631751227497,
      "grad_norm": 5.822299003601074,
      "learning_rate": 3.6567616737416614e-05,
      "logits/chosen": -1.1080423593521118,
      "logits/rejected": -0.8690654635429382,
      "logps/chosen": -430.8399963378906,
      "logps/rejected": -382.16131591796875,
      "loss": 0.3725,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.926641583442688,
      "rewards/margins": 1.1296237707138062,
      "rewards/rejected": -0.20298215746879578,
      "step": 640
    },
    {
      "epoch": 1.0638297872340425,
      "grad_norm": 5.2786078453063965,
      "learning_rate": 3.626440266828381e-05,
      "logits/chosen": -0.986005961894989,
      "logits/rejected": -0.8763575553894043,
      "logps/chosen": -407.85260009765625,
      "logps/rejected": -399.2723693847656,
      "loss": 0.4381,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.8758468627929688,
      "rewards/margins": 0.9750221371650696,
      "rewards/rejected": -0.09917531907558441,
      "step": 650
    },
    {
      "epoch": 1.0801963993453354,
      "grad_norm": 11.273237228393555,
      "learning_rate": 3.5961188599151e-05,
      "logits/chosen": -1.3430869579315186,
      "logits/rejected": -1.056379795074463,
      "logps/chosen": -410.79791259765625,
      "logps/rejected": -434.85845947265625,
      "loss": 0.4005,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9130380749702454,
      "rewards/margins": 0.9950882196426392,
      "rewards/rejected": -0.08205002546310425,
      "step": 660
    },
    {
      "epoch": 1.0965630114566285,
      "grad_norm": 7.220339775085449,
      "learning_rate": 3.5657974530018194e-05,
      "logits/chosen": -1.0506983995437622,
      "logits/rejected": -1.0361990928649902,
      "logps/chosen": -408.18035888671875,
      "logps/rejected": -391.70562744140625,
      "loss": 0.4471,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7731181979179382,
      "rewards/margins": 0.9366775751113892,
      "rewards/rejected": -0.16355939209461212,
      "step": 670
    },
    {
      "epoch": 1.1129296235679214,
      "grad_norm": 5.74124813079834,
      "learning_rate": 3.535476046088539e-05,
      "logits/chosen": -1.206479787826538,
      "logits/rejected": -1.1451863050460815,
      "logps/chosen": -444.80438232421875,
      "logps/rejected": -439.0628356933594,
      "loss": 0.4043,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8405340313911438,
      "rewards/margins": 1.1449936628341675,
      "rewards/rejected": -0.30445969104766846,
      "step": 680
    },
    {
      "epoch": 1.1292962356792144,
      "grad_norm": 7.32386589050293,
      "learning_rate": 3.5051546391752576e-05,
      "logits/chosen": -1.0227863788604736,
      "logits/rejected": -0.9067742228507996,
      "logps/chosen": -476.1012268066406,
      "logps/rejected": -466.14971923828125,
      "loss": 0.4261,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8802071809768677,
      "rewards/margins": 1.1421091556549072,
      "rewards/rejected": -0.2619021236896515,
      "step": 690
    },
    {
      "epoch": 1.1456628477905073,
      "grad_norm": 4.43737268447876,
      "learning_rate": 3.474833232261977e-05,
      "logits/chosen": -1.135181188583374,
      "logits/rejected": -1.0297483205795288,
      "logps/chosen": -506.5615234375,
      "logps/rejected": -483.1039123535156,
      "loss": 0.3875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.5547564029693604,
      "rewards/margins": 1.281551480293274,
      "rewards/rejected": -0.7267951369285583,
      "step": 700
    },
    {
      "epoch": 1.1620294599018004,
      "grad_norm": 2.298978328704834,
      "learning_rate": 3.4445118253486965e-05,
      "logits/chosen": -1.426138162612915,
      "logits/rejected": -1.091489553451538,
      "logps/chosen": -400.107177734375,
      "logps/rejected": -401.62109375,
      "loss": 0.4072,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6946335434913635,
      "rewards/margins": 1.441784143447876,
      "rewards/rejected": -0.7471505999565125,
      "step": 710
    },
    {
      "epoch": 1.1783960720130933,
      "grad_norm": 4.7108154296875,
      "learning_rate": 3.4141904184354156e-05,
      "logits/chosen": -1.1755582094192505,
      "logits/rejected": -0.901213526725769,
      "logps/chosen": -425.28759765625,
      "logps/rejected": -402.042236328125,
      "loss": 0.4615,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.6500431299209595,
      "rewards/margins": 0.9347268342971802,
      "rewards/rejected": -0.2846837639808655,
      "step": 720
    },
    {
      "epoch": 1.1947626841243864,
      "grad_norm": 12.316296577453613,
      "learning_rate": 3.383869011522135e-05,
      "logits/chosen": -1.284589409828186,
      "logits/rejected": -1.0795433521270752,
      "logps/chosen": -413.8037109375,
      "logps/rejected": -491.56146240234375,
      "loss": 0.4385,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.6515102386474609,
      "rewards/margins": 1.2281266450881958,
      "rewards/rejected": -0.5766164660453796,
      "step": 730
    },
    {
      "epoch": 1.2111292962356792,
      "grad_norm": 9.386224746704102,
      "learning_rate": 3.353547604608854e-05,
      "logits/chosen": -1.257037878036499,
      "logits/rejected": -1.0705680847167969,
      "logps/chosen": -505.6416931152344,
      "logps/rejected": -459.56805419921875,
      "loss": 0.3726,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7937537431716919,
      "rewards/margins": 1.4233770370483398,
      "rewards/rejected": -0.629623293876648,
      "step": 740
    },
    {
      "epoch": 1.227495908346972,
      "grad_norm": 6.265888214111328,
      "learning_rate": 3.323226197695573e-05,
      "logits/chosen": -1.332099199295044,
      "logits/rejected": -1.0086009502410889,
      "logps/chosen": -457.0177307128906,
      "logps/rejected": -449.7435607910156,
      "loss": 0.4034,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.790605366230011,
      "rewards/margins": 1.2190719842910767,
      "rewards/rejected": -0.4284667372703552,
      "step": 750
    },
    {
      "epoch": 1.2438625204582652,
      "grad_norm": 6.070497035980225,
      "learning_rate": 3.292904790782292e-05,
      "logits/chosen": -1.2865736484527588,
      "logits/rejected": -1.072819471359253,
      "logps/chosen": -432.710693359375,
      "logps/rejected": -467.6375427246094,
      "loss": 0.3773,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.7443059682846069,
      "rewards/margins": 1.3002400398254395,
      "rewards/rejected": -0.5559341907501221,
      "step": 760
    },
    {
      "epoch": 1.260229132569558,
      "grad_norm": 5.7988128662109375,
      "learning_rate": 3.262583383869012e-05,
      "logits/chosen": -1.0803556442260742,
      "logits/rejected": -0.8561310768127441,
      "logps/chosen": -514.7871704101562,
      "logps/rejected": -503.19500732421875,
      "loss": 0.3216,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7713959217071533,
      "rewards/margins": 1.6244882345199585,
      "rewards/rejected": -0.8530920743942261,
      "step": 770
    },
    {
      "epoch": 1.2765957446808511,
      "grad_norm": 4.323042869567871,
      "learning_rate": 3.232261976955731e-05,
      "logits/chosen": -1.408276915550232,
      "logits/rejected": -0.9835014343261719,
      "logps/chosen": -456.21142578125,
      "logps/rejected": -429.10467529296875,
      "loss": 0.3832,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.5816126465797424,
      "rewards/margins": 1.3987071514129639,
      "rewards/rejected": -0.8170945048332214,
      "step": 780
    },
    {
      "epoch": 1.292962356792144,
      "grad_norm": 7.249789237976074,
      "learning_rate": 3.20194057004245e-05,
      "logits/chosen": -1.1671850681304932,
      "logits/rejected": -0.653769314289093,
      "logps/chosen": -481.74859619140625,
      "logps/rejected": -441.5457458496094,
      "loss": 0.3293,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5319666862487793,
      "rewards/margins": 1.7422034740447998,
      "rewards/rejected": -1.21023690700531,
      "step": 790
    },
    {
      "epoch": 1.3093289689034369,
      "grad_norm": 9.989459037780762,
      "learning_rate": 3.171619163129169e-05,
      "logits/chosen": -1.2330873012542725,
      "logits/rejected": -1.1575677394866943,
      "logps/chosen": -462.0821228027344,
      "logps/rejected": -419.34149169921875,
      "loss": 0.4737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7070258855819702,
      "rewards/margins": 1.2338263988494873,
      "rewards/rejected": -0.5268004536628723,
      "step": 800
    },
    {
      "epoch": 1.32569558101473,
      "grad_norm": 5.914211750030518,
      "learning_rate": 3.141297756215888e-05,
      "logits/chosen": -1.257925271987915,
      "logits/rejected": -1.1667311191558838,
      "logps/chosen": -492.0726623535156,
      "logps/rejected": -463.2273864746094,
      "loss": 0.3597,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7776979207992554,
      "rewards/margins": 1.3328384160995483,
      "rewards/rejected": -0.555140495300293,
      "step": 810
    },
    {
      "epoch": 1.342062193126023,
      "grad_norm": 3.587632179260254,
      "learning_rate": 3.110976349302608e-05,
      "logits/chosen": -1.2870782613754272,
      "logits/rejected": -0.9442241787910461,
      "logps/chosen": -493.3888244628906,
      "logps/rejected": -464.4835510253906,
      "loss": 0.3151,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.9334325790405273,
      "rewards/margins": 1.9022817611694336,
      "rewards/rejected": -0.9688493013381958,
      "step": 820
    },
    {
      "epoch": 1.358428805237316,
      "grad_norm": 6.462862968444824,
      "learning_rate": 3.080654942389327e-05,
      "logits/chosen": -1.4893635511398315,
      "logits/rejected": -1.0626758337020874,
      "logps/chosen": -373.52288818359375,
      "logps/rejected": -411.831298828125,
      "loss": 0.438,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.5565814971923828,
      "rewards/margins": 1.1154134273529053,
      "rewards/rejected": -0.5588318705558777,
      "step": 830
    },
    {
      "epoch": 1.3747954173486088,
      "grad_norm": 6.482163429260254,
      "learning_rate": 3.0503335354760464e-05,
      "logits/chosen": -1.2426273822784424,
      "logits/rejected": -1.0962947607040405,
      "logps/chosen": -418.48577880859375,
      "logps/rejected": -414.7897033691406,
      "loss": 0.4516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7719891667366028,
      "rewards/margins": 1.0017683506011963,
      "rewards/rejected": -0.22977924346923828,
      "step": 840
    },
    {
      "epoch": 1.3911620294599019,
      "grad_norm": 8.918207168579102,
      "learning_rate": 3.020012128562765e-05,
      "logits/chosen": -1.3620984554290771,
      "logits/rejected": -1.1984447240829468,
      "logps/chosen": -465.46270751953125,
      "logps/rejected": -446.5343322753906,
      "loss": 0.4053,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6932386159896851,
      "rewards/margins": 1.3631744384765625,
      "rewards/rejected": -0.6699355840682983,
      "step": 850
    },
    {
      "epoch": 1.4075286415711947,
      "grad_norm": 19.940839767456055,
      "learning_rate": 2.9896907216494846e-05,
      "logits/chosen": -1.2039759159088135,
      "logits/rejected": -0.9377444386482239,
      "logps/chosen": -574.588623046875,
      "logps/rejected": -502.51214599609375,
      "loss": 0.4461,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8642657399177551,
      "rewards/margins": 1.3982216119766235,
      "rewards/rejected": -0.5339558124542236,
      "step": 860
    },
    {
      "epoch": 1.4238952536824878,
      "grad_norm": 6.069481372833252,
      "learning_rate": 2.959369314736204e-05,
      "logits/chosen": -1.3913848400115967,
      "logits/rejected": -1.2518432140350342,
      "logps/chosen": -458.3941345214844,
      "logps/rejected": -452.67352294921875,
      "loss": 0.3604,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.043166160583496,
      "rewards/margins": 1.7257499694824219,
      "rewards/rejected": -0.6825838685035706,
      "step": 870
    },
    {
      "epoch": 1.4402618657937807,
      "grad_norm": 3.334609031677246,
      "learning_rate": 2.929047907822923e-05,
      "logits/chosen": -1.3935604095458984,
      "logits/rejected": NaN,
      "logps/chosen": -490.9991149902344,
      "logps/rejected": -452.0387268066406,
      "loss": 0.3173,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0780329704284668,
      "rewards/margins": 1.919669508934021,
      "rewards/rejected": -0.8416364789009094,
      "step": 880
    },
    {
      "epoch": 1.4566284779050735,
      "grad_norm": 5.730930805206299,
      "learning_rate": 2.8987265009096426e-05,
      "logits/chosen": -1.3444249629974365,
      "logits/rejected": -1.124793291091919,
      "logps/chosen": -409.4859619140625,
      "logps/rejected": -429.79150390625,
      "loss": 0.4307,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.8416284322738647,
      "rewards/margins": 1.227608561515808,
      "rewards/rejected": -0.3859800696372986,
      "step": 890
    },
    {
      "epoch": 1.4729950900163666,
      "grad_norm": 4.7208333015441895,
      "learning_rate": 2.8684050939963614e-05,
      "logits/chosen": -1.2483845949172974,
      "logits/rejected": -1.162945032119751,
      "logps/chosen": -438.02978515625,
      "logps/rejected": -483.64776611328125,
      "loss": 0.4359,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6855508089065552,
      "rewards/margins": 1.3711297512054443,
      "rewards/rejected": -0.6855790019035339,
      "step": 900
    },
    {
      "epoch": 1.4893617021276595,
      "grad_norm": 4.603919982910156,
      "learning_rate": 2.838083687083081e-05,
      "logits/chosen": -1.3359647989273071,
      "logits/rejected": -1.2829469442367554,
      "logps/chosen": -478.17022705078125,
      "logps/rejected": -408.0987243652344,
      "loss": 0.431,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.9056881666183472,
      "rewards/margins": 1.124969244003296,
      "rewards/rejected": -0.21928110718727112,
      "step": 910
    },
    {
      "epoch": 1.5057283142389526,
      "grad_norm": 7.60992431640625,
      "learning_rate": 2.8077622801698e-05,
      "logits/chosen": -1.4223133325576782,
      "logits/rejected": -1.2066584825515747,
      "logps/chosen": -428.1543884277344,
      "logps/rejected": -370.4458923339844,
      "loss": 0.4003,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.8528293371200562,
      "rewards/margins": 1.363084077835083,
      "rewards/rejected": -0.5102548599243164,
      "step": 920
    },
    {
      "epoch": 1.5220949263502455,
      "grad_norm": 4.768836498260498,
      "learning_rate": 2.7774408732565194e-05,
      "logits/chosen": -1.2342239618301392,
      "logits/rejected": -1.1602811813354492,
      "logps/chosen": -440.5987854003906,
      "logps/rejected": -405.0908508300781,
      "loss": 0.3879,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.9851611256599426,
      "rewards/margins": 1.1822245121002197,
      "rewards/rejected": -0.19706343114376068,
      "step": 930
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 8.149114608764648,
      "learning_rate": 2.7471194663432385e-05,
      "logits/chosen": -1.1173126697540283,
      "logits/rejected": -1.014575481414795,
      "logps/chosen": -402.3595275878906,
      "logps/rejected": -410.60003662109375,
      "loss": 0.4529,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.7943775653839111,
      "rewards/margins": 1.2599968910217285,
      "rewards/rejected": -0.4656193256378174,
      "step": 940
    },
    {
      "epoch": 1.5548281505728314,
      "grad_norm": 7.4743852615356445,
      "learning_rate": 2.7167980594299573e-05,
      "logits/chosen": NaN,
      "logits/rejected": -1.143424391746521,
      "logps/chosen": -446.26629638671875,
      "logps/rejected": -469.7017517089844,
      "loss": 0.491,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.797395646572113,
      "rewards/margins": 1.1872708797454834,
      "rewards/rejected": -0.38987529277801514,
      "step": 950
    },
    {
      "epoch": 1.5711947626841245,
      "grad_norm": 8.159062385559082,
      "learning_rate": 2.6864766525166768e-05,
      "logits/chosen": -1.14289128780365,
      "logits/rejected": NaN,
      "logps/chosen": -450.77117919921875,
      "logps/rejected": -438.8096618652344,
      "loss": 0.4282,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.7298113107681274,
      "rewards/margins": 1.092790126800537,
      "rewards/rejected": -0.3629787862300873,
      "step": 960
    },
    {
      "epoch": 1.5875613747954174,
      "grad_norm": 9.274547576904297,
      "learning_rate": 2.6561552456033962e-05,
      "logits/chosen": -0.9688614010810852,
      "logits/rejected": -0.897729754447937,
      "logps/chosen": -515.4688720703125,
      "logps/rejected": -436.51275634765625,
      "loss": 0.4169,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.7370742559432983,
      "rewards/margins": 1.1543539762496948,
      "rewards/rejected": -0.4172796607017517,
      "step": 970
    },
    {
      "epoch": 1.6039279869067102,
      "grad_norm": 5.051360130310059,
      "learning_rate": 2.6258338386901153e-05,
      "logits/chosen": -1.0737905502319336,
      "logits/rejected": -0.9251714944839478,
      "logps/chosen": -471.31146240234375,
      "logps/rejected": -432.572509765625,
      "loss": 0.3841,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.8669958114624023,
      "rewards/margins": 1.4004991054534912,
      "rewards/rejected": -0.5335032939910889,
      "step": 980
    },
    {
      "epoch": 1.6202945990180033,
      "grad_norm": 6.768540859222412,
      "learning_rate": 2.5955124317768348e-05,
      "logits/chosen": -1.1147942543029785,
      "logits/rejected": -1.1377754211425781,
      "logps/chosen": -462.5465393066406,
      "logps/rejected": -390.2607727050781,
      "loss": 0.3919,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8462246060371399,
      "rewards/margins": 1.2740905284881592,
      "rewards/rejected": -0.4278659224510193,
      "step": 990
    },
    {
      "epoch": 1.6366612111292962,
      "grad_norm": 8.313865661621094,
      "learning_rate": 2.5651910248635535e-05,
      "logits/chosen": -1.0041131973266602,
      "logits/rejected": -0.8046473264694214,
      "logps/chosen": -461.7525939941406,
      "logps/rejected": -443.0941467285156,
      "loss": 0.4222,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8126258850097656,
      "rewards/margins": 1.1657203435897827,
      "rewards/rejected": -0.3530944585800171,
      "step": 1000
    }
  ],
  "logging_steps": 10,
  "max_steps": 1833,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}