{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 50,
  "global_step": 3884,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0012873326467559218,
      "grad_norm": 206.0,
      "learning_rate": 4.99356333676622e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -255.60000610351562,
      "logps/rejected": -215.60000610351562,
      "loss": 0.6883,
      "rewards/accuracies": 0.18916667997837067,
      "rewards/chosen": -0.0028564452659338713,
      "rewards/margins": 0.009753418155014515,
      "rewards/rejected": -0.012631225399672985,
      "step": 5
    },
    {
      "epoch": 0.0025746652935118436,
      "grad_norm": 188.0,
      "learning_rate": 4.987126673532441e-07,
      "logits/chosen": -2.8062500953674316,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -274.20001220703125,
      "logps/rejected": -200.8000030517578,
      "loss": 0.6609,
      "rewards/accuracies": 0.32230159640312195,
      "rewards/chosen": 0.13947753608226776,
      "rewards/margins": 0.14555664360523224,
      "rewards/rejected": -0.006365966983139515,
      "step": 10
    },
    {
      "epoch": 0.0038619979402677654,
      "grad_norm": 223.0,
      "learning_rate": 4.980690010298661e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -298.0,
      "logps/rejected": -281.79998779296875,
      "loss": 0.6844,
      "rewards/accuracies": 0.3194047808647156,
      "rewards/chosen": 0.02211303636431694,
      "rewards/margins": 0.01730957068502903,
      "rewards/rejected": 0.004803466610610485,
      "step": 15
    },
    {
      "epoch": 0.005149330587023687,
      "grad_norm": 199.0,
      "learning_rate": 4.974253347064881e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -322.3999938964844,
      "logps/rejected": -262.6000061035156,
      "loss": 0.6859,
      "rewards/accuracies": 0.29500001668930054,
      "rewards/chosen": 0.01585693284869194,
      "rewards/margins": 0.015551757998764515,
      "rewards/rejected": 0.000274658203125,
      "step": 20
    },
    {
      "epoch": 0.006436663233779609,
      "grad_norm": 212.0,
      "learning_rate": 4.967816683831102e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.21875,
      "logps/chosen": -303.20001220703125,
      "logps/rejected": -259.3999938964844,
      "loss": 0.6813,
      "rewards/accuracies": 0.3466666638851166,
      "rewards/chosen": 0.01540527306497097,
      "rewards/margins": 0.02518920972943306,
      "rewards/rejected": -0.009765625,
      "step": 25
    },
    {
      "epoch": 0.007723995880535531,
      "grad_norm": 215.0,
      "learning_rate": 4.961380020597322e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.296875,
      "logps/chosen": -271.3999938964844,
      "logps/rejected": -239.39999389648438,
      "loss": 0.6898,
      "rewards/accuracies": 0.28565019369125366,
      "rewards/chosen": 0.00954589806497097,
      "rewards/margins": 0.005908203311264515,
      "rewards/rejected": 0.0036163330078125,
      "step": 30
    },
    {
      "epoch": 0.009011328527291453,
      "grad_norm": 380.0,
      "learning_rate": 4.954943357363543e-07,
      "logits/chosen": -3.0718750953674316,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -229.89999389648438,
      "logps/rejected": -213.60000610351562,
      "loss": 0.6719,
      "rewards/accuracies": 0.39215368032455444,
      "rewards/chosen": 0.07059326022863388,
      "rewards/margins": 0.05167236179113388,
      "rewards/rejected": 0.01886596716940403,
      "step": 35
    },
    {
      "epoch": 0.010298661174047374,
      "grad_norm": 312.0,
      "learning_rate": 4.948506694129763e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -292.6000061035156,
      "logps/rejected": -255.0,
      "loss": 0.6789,
      "rewards/accuracies": 0.24272894859313965,
      "rewards/chosen": 0.03836669772863388,
      "rewards/margins": 0.03143310546875,
      "rewards/rejected": 0.0068908692337572575,
      "step": 40
    },
    {
      "epoch": 0.011585993820803296,
      "grad_norm": 248.0,
      "learning_rate": 4.942070030895984e-07,
      "logits/chosen": -3.293750047683716,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -318.79998779296875,
      "logps/rejected": -286.6000061035156,
      "loss": 0.6766,
      "rewards/accuracies": 0.3248809576034546,
      "rewards/chosen": 0.04169616848230362,
      "rewards/margins": 0.0389404296875,
      "rewards/rejected": 0.0027709961868822575,
      "step": 45
    },
    {
      "epoch": 0.012873326467559218,
      "grad_norm": 247.0,
      "learning_rate": 4.935633367662204e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -275.0,
      "logps/rejected": -237.8000030517578,
      "loss": 0.6586,
      "rewards/accuracies": 0.534166693687439,
      "rewards/chosen": 0.08574219048023224,
      "rewards/margins": 0.08336181938648224,
      "rewards/rejected": 0.0025146484840661287,
      "step": 50
    },
    {
      "epoch": 0.01416065911431514,
      "grad_norm": 616.0,
      "learning_rate": 4.929196704428423e-07,
      "logits/chosen": -3.0875000953674316,
      "logits/rejected": -2.8187499046325684,
      "logps/chosen": -246.0,
      "logps/rejected": -223.60000610351562,
      "loss": 0.6898,
      "rewards/accuracies": 0.4071428179740906,
      "rewards/chosen": 0.05268554762005806,
      "rewards/margins": 0.02159576490521431,
      "rewards/rejected": 0.03104248084127903,
      "step": 55
    },
    {
      "epoch": 0.015447991761071062,
      "grad_norm": 208.0,
      "learning_rate": 4.922760041194645e-07,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.106250047683716,
      "logps/chosen": -206.8000030517578,
      "logps/rejected": -197.39999389648438,
      "loss": 0.6937,
      "rewards/accuracies": 0.41111111640930176,
      "rewards/chosen": 0.0355224609375,
      "rewards/margins": -0.0008789062267169356,
      "rewards/rejected": 0.036376953125,
      "step": 60
    },
    {
      "epoch": 0.016735324407826983,
      "grad_norm": 274.0,
      "learning_rate": 4.916323377960865e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.203125,
      "logps/chosen": -285.6000061035156,
      "logps/rejected": -249.60000610351562,
      "loss": 0.6906,
      "rewards/accuracies": 0.3185897469520569,
      "rewards/chosen": 0.0501556396484375,
      "rewards/margins": 0.015270233154296875,
      "rewards/rejected": 0.034848786890506744,
      "step": 65
    },
    {
      "epoch": 0.018022657054582905,
      "grad_norm": 342.0,
      "learning_rate": 4.909886714727085e-07,
      "logits/chosen": -2.987499952316284,
      "logits/rejected": -2.9390625953674316,
      "logps/chosen": -314.6000061035156,
      "logps/rejected": -268.8999938964844,
      "loss": 0.668,
      "rewards/accuracies": 0.4643254280090332,
      "rewards/chosen": 0.08603515475988388,
      "rewards/margins": 0.05771942064166069,
      "rewards/rejected": 0.02829589881002903,
      "step": 70
    },
    {
      "epoch": 0.019309989701338827,
      "grad_norm": 231.0,
      "learning_rate": 4.903450051493306e-07,
      "logits/chosen": -3.0218749046325684,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -320.79998779296875,
      "logps/rejected": -233.10000610351562,
      "loss": 0.6516,
      "rewards/accuracies": 0.44914668798446655,
      "rewards/chosen": 0.18281249701976776,
      "rewards/margins": 0.16166992485523224,
      "rewards/rejected": 0.0213623046875,
      "step": 75
    },
    {
      "epoch": 0.02059732234809475,
      "grad_norm": 256.0,
      "learning_rate": 4.897013388259526e-07,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.096874952316284,
      "logps/chosen": -378.3999938964844,
      "logps/rejected": -348.0,
      "loss": 0.6438,
      "rewards/accuracies": 0.4992307722568512,
      "rewards/chosen": 0.14267578721046448,
      "rewards/margins": 0.10410156100988388,
      "rewards/rejected": 0.0386962890625,
      "step": 80
    },
    {
      "epoch": 0.02188465499485067,
      "grad_norm": 218.0,
      "learning_rate": 4.890576725025746e-07,
      "logits/chosen": -3.3343749046325684,
      "logits/rejected": -3.325000047683716,
      "logps/chosen": -291.20001220703125,
      "logps/rejected": -302.79998779296875,
      "loss": 0.6672,
      "rewards/accuracies": 0.5241667032241821,
      "rewards/chosen": 0.084014892578125,
      "rewards/margins": 0.05776367336511612,
      "rewards/rejected": 0.026214599609375,
      "step": 85
    },
    {
      "epoch": 0.023171987641606592,
      "grad_norm": 266.0,
      "learning_rate": 4.884140061791967e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.171875,
      "logps/chosen": -290.6000061035156,
      "logps/rejected": -253.0,
      "loss": 0.6781,
      "rewards/accuracies": 0.48999086022377014,
      "rewards/chosen": 0.07816161960363388,
      "rewards/margins": 0.03767089918255806,
      "rewards/rejected": 0.04047851637005806,
      "step": 90
    },
    {
      "epoch": 0.024459320288362514,
      "grad_norm": 217.0,
      "learning_rate": 4.877703398558187e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -304.20001220703125,
      "logps/rejected": -240.39999389648438,
      "loss": 0.6664,
      "rewards/accuracies": 0.43472224473953247,
      "rewards/chosen": 0.07504882663488388,
      "rewards/margins": 0.05483398586511612,
      "rewards/rejected": 0.02030639722943306,
      "step": 95
    },
    {
      "epoch": 0.025746652935118436,
      "grad_norm": 204.0,
      "learning_rate": 4.871266735324407e-07,
      "logits/chosen": -3.2874999046325684,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -329.3999938964844,
      "logps/rejected": -278.79998779296875,
      "loss": 0.6727,
      "rewards/accuracies": 0.5108516812324524,
      "rewards/chosen": 0.09018554538488388,
      "rewards/margins": 0.07102050632238388,
      "rewards/rejected": 0.01927490159869194,
      "step": 100
    },
    {
      "epoch": 0.027033985581874358,
      "grad_norm": 290.0,
      "learning_rate": 4.864830072090629e-07,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -3.078125,
      "logps/chosen": -301.6000061035156,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6758,
      "rewards/accuracies": 0.4570237994194031,
      "rewards/chosen": 0.12485351413488388,
      "rewards/margins": 0.04887695237994194,
      "rewards/rejected": 0.075927734375,
      "step": 105
    },
    {
      "epoch": 0.02832131822863028,
      "grad_norm": 228.0,
      "learning_rate": 4.858393408856848e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.265625,
      "logps/chosen": -294.6000061035156,
      "logps/rejected": -269.0,
      "loss": 0.6703,
      "rewards/accuracies": 0.45154762268066406,
      "rewards/chosen": 0.091796875,
      "rewards/margins": 0.058611202985048294,
      "rewards/rejected": 0.03325195237994194,
      "step": 110
    },
    {
      "epoch": 0.0296086508753862,
      "grad_norm": 248.0,
      "learning_rate": 4.851956745623069e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -311.79998779296875,
      "logps/rejected": -278.0,
      "loss": 0.6687,
      "rewards/accuracies": 0.5027777552604675,
      "rewards/chosen": 0.15566405653953552,
      "rewards/margins": 0.08112792670726776,
      "rewards/rejected": 0.07456054538488388,
      "step": 115
    },
    {
      "epoch": 0.030895983522142123,
      "grad_norm": 324.0,
      "learning_rate": 4.845520082389289e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -294.0,
      "logps/rejected": -248.60000610351562,
      "loss": 0.6867,
      "rewards/accuracies": 0.4208333492279053,
      "rewards/chosen": 0.09580077975988388,
      "rewards/margins": 0.02167968824505806,
      "rewards/rejected": 0.0740966796875,
      "step": 120
    },
    {
      "epoch": 0.032183316168898045,
      "grad_norm": 536.0,
      "learning_rate": 4.839083419155509e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -309.3999938964844,
      "logps/rejected": -240.60000610351562,
      "loss": 0.6977,
      "rewards/accuracies": 0.396785706281662,
      "rewards/chosen": 0.15751953423023224,
      "rewards/margins": 0.009936523623764515,
      "rewards/rejected": 0.14775696396827698,
      "step": 125
    },
    {
      "epoch": 0.03347064881565397,
      "grad_norm": 284.0,
      "learning_rate": 4.83264675592173e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -264.79998779296875,
      "logps/rejected": -245.8000030517578,
      "loss": 0.6805,
      "rewards/accuracies": 0.4714627265930176,
      "rewards/chosen": 0.08936462551355362,
      "rewards/margins": 0.04328613355755806,
      "rewards/rejected": 0.04611816257238388,
      "step": 130
    },
    {
      "epoch": 0.03475798146240989,
      "grad_norm": 231.0,
      "learning_rate": 4.82621009268795e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -293.20001220703125,
      "logps/rejected": -212.1999969482422,
      "loss": 0.675,
      "rewards/accuracies": 0.4858333468437195,
      "rewards/chosen": 0.11679687350988388,
      "rewards/margins": 0.04426269605755806,
      "rewards/rejected": 0.07266845554113388,
      "step": 135
    },
    {
      "epoch": 0.03604531410916581,
      "grad_norm": 260.0,
      "learning_rate": 4.819773429454171e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -212.0,
      "loss": 0.668,
      "rewards/accuracies": 0.5297435522079468,
      "rewards/chosen": 0.10280761867761612,
      "rewards/margins": 0.06215820461511612,
      "rewards/rejected": 0.04050292819738388,
      "step": 140
    },
    {
      "epoch": 0.03733264675592173,
      "grad_norm": 436.0,
      "learning_rate": 4.813336766220391e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -268.3999938964844,
      "logps/rejected": -202.0,
      "loss": 0.6617,
      "rewards/accuracies": 0.5116666555404663,
      "rewards/chosen": 0.115234375,
      "rewards/margins": 0.06391601264476776,
      "rewards/rejected": 0.05126953125,
      "step": 145
    },
    {
      "epoch": 0.038619979402677654,
      "grad_norm": 225.0,
      "learning_rate": 4.806900102986612e-07,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -3.203125,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -240.1999969482422,
      "loss": 0.675,
      "rewards/accuracies": 0.5071429014205933,
      "rewards/chosen": 0.13076171278953552,
      "rewards/margins": 0.05214843899011612,
      "rewards/rejected": 0.078857421875,
      "step": 150
    },
    {
      "epoch": 0.039907312049433576,
      "grad_norm": 278.0,
      "learning_rate": 4.800463439752832e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -165.1999969482422,
      "logps/rejected": -154.8000030517578,
      "loss": 0.7039,
      "rewards/accuracies": 0.4074999690055847,
      "rewards/chosen": 0.075927734375,
      "rewards/margins": -0.01259765587747097,
      "rewards/rejected": 0.08870849758386612,
      "step": 155
    },
    {
      "epoch": 0.0411946446961895,
      "grad_norm": 508.0,
      "learning_rate": 4.794026776519052e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -192.39999389648438,
      "logps/rejected": -168.14999389648438,
      "loss": 0.6898,
      "rewards/accuracies": 0.5305556058883667,
      "rewards/chosen": 0.11484374850988388,
      "rewards/margins": 0.03442993015050888,
      "rewards/rejected": 0.08056640625,
      "step": 160
    },
    {
      "epoch": 0.04248197734294542,
      "grad_norm": 266.0,
      "learning_rate": 4.787590113285273e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -251.39999389648438,
      "logps/rejected": -254.8000030517578,
      "loss": 0.6375,
      "rewards/accuracies": 0.6444047689437866,
      "rewards/chosen": 0.13828125596046448,
      "rewards/margins": 0.12412109225988388,
      "rewards/rejected": 0.013964843936264515,
      "step": 165
    },
    {
      "epoch": 0.04376930998970134,
      "grad_norm": 392.0,
      "learning_rate": 4.781153450051493e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -276.79998779296875,
      "logps/rejected": -266.20001220703125,
      "loss": 0.6781,
      "rewards/accuracies": 0.49253249168395996,
      "rewards/chosen": 0.13662108778953552,
      "rewards/margins": 0.05128173902630806,
      "rewards/rejected": 0.08544921875,
      "step": 170
    },
    {
      "epoch": 0.04505664263645726,
      "grad_norm": 218.0,
      "learning_rate": 4.774716786817714e-07,
      "logits/chosen": -2.987499952316284,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -278.3999938964844,
      "logps/rejected": -268.79998779296875,
      "loss": 0.6461,
      "rewards/accuracies": 0.5777472853660583,
      "rewards/chosen": 0.1689453125,
      "rewards/margins": 0.11292724311351776,
      "rewards/rejected": 0.05599059909582138,
      "step": 175
    },
    {
      "epoch": 0.046343975283213185,
      "grad_norm": 200.0,
      "learning_rate": 4.7682801235839336e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.0531249046325684,
      "logps/chosen": -243.1999969482422,
      "logps/rejected": -238.8000030517578,
      "loss": 0.6742,
      "rewards/accuracies": 0.47343435883522034,
      "rewards/chosen": 0.12441406399011612,
      "rewards/margins": 0.03370361402630806,
      "rewards/rejected": 0.09064941108226776,
      "step": 180
    },
    {
      "epoch": 0.047631307929969106,
      "grad_norm": 222.0,
      "learning_rate": 4.7618434603501545e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -303.6000061035156,
      "logps/rejected": -271.3999938964844,
      "loss": 0.6656,
      "rewards/accuracies": 0.46525639295578003,
      "rewards/chosen": 0.12602539360523224,
      "rewards/margins": 0.07691650092601776,
      "rewards/rejected": 0.04915161058306694,
      "step": 185
    },
    {
      "epoch": 0.04891864057672503,
      "grad_norm": 234.0,
      "learning_rate": 4.755406797116375e-07,
      "logits/chosen": -3.3218750953674316,
      "logits/rejected": -3.0218749046325684,
      "logps/chosen": -256.3999938964844,
      "logps/rejected": -291.0,
      "loss": 0.6641,
      "rewards/accuracies": 0.49954214692115784,
      "rewards/chosen": 0.093994140625,
      "rewards/margins": 0.06173095852136612,
      "rewards/rejected": 0.0323486328125,
      "step": 190
    },
    {
      "epoch": 0.05020597322348095,
      "grad_norm": 227.0,
      "learning_rate": 4.748970133882595e-07,
      "logits/chosen": -3.2718749046325684,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -255.8000030517578,
      "logps/rejected": -223.0,
      "loss": 0.6633,
      "rewards/accuracies": 0.5204700827598572,
      "rewards/chosen": 0.13730469346046448,
      "rewards/margins": 0.06943359225988388,
      "rewards/rejected": 0.06776122748851776,
      "step": 195
    },
    {
      "epoch": 0.05149330587023687,
      "grad_norm": 406.0,
      "learning_rate": 4.742533470648816e-07,
      "logits/chosen": -2.9156250953674316,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -217.0,
      "logps/rejected": -214.8000030517578,
      "loss": 0.6703,
      "rewards/accuracies": 0.5442099571228027,
      "rewards/chosen": 0.18320313096046448,
      "rewards/margins": 0.06836547702550888,
      "rewards/rejected": 0.11455078423023224,
      "step": 200
    },
    {
      "epoch": 0.052780638516992794,
      "grad_norm": 201.0,
      "learning_rate": 4.7360968074150357e-07,
      "logits/chosen": -3.3218750953674316,
      "logits/rejected": -3.3687500953674316,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -272.6000061035156,
      "loss": 0.6484,
      "rewards/accuracies": 0.561089813709259,
      "rewards/chosen": 0.11601562798023224,
      "rewards/margins": 0.10400390625,
      "rewards/rejected": 0.01210937462747097,
      "step": 205
    },
    {
      "epoch": 0.054067971163748715,
      "grad_norm": 266.0,
      "learning_rate": 4.729660144181256e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -314.0,
      "logps/rejected": -276.20001220703125,
      "loss": 0.6813,
      "rewards/accuracies": 0.42166668176651,
      "rewards/chosen": 0.11264648288488388,
      "rewards/margins": 0.03688964992761612,
      "rewards/rejected": 0.07563476264476776,
      "step": 210
    },
    {
      "epoch": 0.05535530381050464,
      "grad_norm": 3680.0,
      "learning_rate": 4.7232234809474765e-07,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -2.9156250953674316,
      "logps/chosen": -266.20001220703125,
      "logps/rejected": -241.0,
      "loss": 0.6977,
      "rewards/accuracies": 0.46667584776878357,
      "rewards/chosen": 0.12089844048023224,
      "rewards/margins": 0.0077148438431322575,
      "rewards/rejected": 0.11301269382238388,
      "step": 215
    },
    {
      "epoch": 0.05664263645726056,
      "grad_norm": 254.0,
      "learning_rate": 4.716786817713697e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.2718749046325684,
      "logps/chosen": -314.1000061035156,
      "logps/rejected": -245.3000030517578,
      "loss": 0.6391,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.12807616591453552,
      "rewards/margins": 0.13349609076976776,
      "rewards/rejected": -0.0053344727493822575,
      "step": 220
    },
    {
      "epoch": 0.05792996910401648,
      "grad_norm": 266.0,
      "learning_rate": 4.7103501544799174e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -303.6000061035156,
      "logps/rejected": -278.20001220703125,
      "loss": 0.6656,
      "rewards/accuracies": 0.4482051432132721,
      "rewards/chosen": 0.14140625298023224,
      "rewards/margins": 0.07213135063648224,
      "rewards/rejected": 0.06917724758386612,
      "step": 225
    },
    {
      "epoch": 0.0592173017507724,
      "grad_norm": 292.0,
      "learning_rate": 4.703913491246138e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -311.0,
      "logps/rejected": -268.0,
      "loss": 0.6656,
      "rewards/accuracies": 0.520616888999939,
      "rewards/chosen": 0.15878906846046448,
      "rewards/margins": 0.0697021484375,
      "rewards/rejected": 0.08918456733226776,
      "step": 230
    },
    {
      "epoch": 0.060504634397528324,
      "grad_norm": 300.0,
      "learning_rate": 4.697476828012358e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.078125,
      "logps/chosen": -330.3999938964844,
      "logps/rejected": -292.6000061035156,
      "loss": 0.6562,
      "rewards/accuracies": 0.4980769157409668,
      "rewards/chosen": 0.14179687201976776,
      "rewards/margins": 0.09501953423023224,
      "rewards/rejected": 0.04721679538488388,
      "step": 235
    },
    {
      "epoch": 0.061791967044284246,
      "grad_norm": 264.0,
      "learning_rate": 4.6910401647785787e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -239.39999389648438,
      "logps/rejected": -218.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.5517638325691223,
      "rewards/chosen": 0.16318359971046448,
      "rewards/margins": 0.08962402492761612,
      "rewards/rejected": 0.07353515923023224,
      "step": 240
    },
    {
      "epoch": 0.06307929969104016,
      "grad_norm": 278.0,
      "learning_rate": 4.6846035015447986e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.046875,
      "logps/chosen": -228.1999969482422,
      "logps/rejected": -224.1999969482422,
      "loss": 0.6477,
      "rewards/accuracies": 0.5161616206169128,
      "rewards/chosen": 0.15742187201976776,
      "rewards/margins": 0.11433105170726776,
      "rewards/rejected": 0.04304199293255806,
      "step": 245
    },
    {
      "epoch": 0.06436663233779609,
      "grad_norm": 179.0,
      "learning_rate": 4.6781668383110195e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.234375,
      "logps/chosen": -256.5,
      "logps/rejected": -290.79998779296875,
      "loss": 0.6508,
      "rewards/accuracies": 0.5385605692863464,
      "rewards/chosen": 0.07485351711511612,
      "rewards/margins": 0.09757690131664276,
      "rewards/rejected": -0.02265625074505806,
      "step": 250
    },
    {
      "epoch": 0.065653964984552,
      "grad_norm": 468.0,
      "learning_rate": 4.67173017507724e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.090625047683716,
      "logps/chosen": -294.79998779296875,
      "logps/rejected": -277.0,
      "loss": 0.6586,
      "rewards/accuracies": 0.533141016960144,
      "rewards/chosen": 0.09638671576976776,
      "rewards/margins": 0.09467773139476776,
      "rewards/rejected": 0.001953125,
      "step": 255
    },
    {
      "epoch": 0.06694129763130793,
      "grad_norm": 249.0,
      "learning_rate": 4.66529351184346e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -313.3999938964844,
      "logps/rejected": -288.20001220703125,
      "loss": 0.6531,
      "rewards/accuracies": 0.5691666603088379,
      "rewards/chosen": 0.10101318359375,
      "rewards/margins": 0.09951172024011612,
      "rewards/rejected": 0.0018066406482830644,
      "step": 260
    },
    {
      "epoch": 0.06822863027806385,
      "grad_norm": 302.0,
      "learning_rate": 4.658856848609681e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -301.79998779296875,
      "logps/rejected": -232.60000610351562,
      "loss": 0.6305,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.10062255710363388,
      "rewards/margins": 0.14775390923023224,
      "rewards/rejected": -0.04689941555261612,
      "step": 265
    },
    {
      "epoch": 0.06951596292481978,
      "grad_norm": 262.0,
      "learning_rate": 4.652420185375901e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -357.79998779296875,
      "logps/rejected": -302.3999938964844,
      "loss": 0.6422,
      "rewards/accuracies": 0.6066666841506958,
      "rewards/chosen": 0.13505859673023224,
      "rewards/margins": 0.12954100966453552,
      "rewards/rejected": 0.0056701661087572575,
      "step": 270
    },
    {
      "epoch": 0.07080329557157569,
      "grad_norm": 270.0,
      "learning_rate": 4.645983522142121e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.2718749046325684,
      "logps/chosen": -277.6000061035156,
      "logps/rejected": -211.89999389648438,
      "loss": 0.6469,
      "rewards/accuracies": 0.5826923251152039,
      "rewards/chosen": 0.13876953721046448,
      "rewards/margins": 0.14497070014476776,
      "rewards/rejected": -0.006134033203125,
      "step": 275
    },
    {
      "epoch": 0.07209062821833162,
      "grad_norm": 234.0,
      "learning_rate": 4.639546858908342e-07,
      "logits/chosen": -2.987499952316284,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -349.20001220703125,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6141,
      "rewards/accuracies": 0.6309523582458496,
      "rewards/chosen": 0.21152344346046448,
      "rewards/margins": 0.22622069716453552,
      "rewards/rejected": -0.01431884802877903,
      "step": 280
    },
    {
      "epoch": 0.07337796086508754,
      "grad_norm": 294.0,
      "learning_rate": 4.633110195674562e-07,
      "logits/chosen": -3.0250000953674316,
      "logits/rejected": -3.012500047683716,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -251.0,
      "loss": 0.6492,
      "rewards/accuracies": 0.5100433230400085,
      "rewards/chosen": 0.18310546875,
      "rewards/margins": 0.14501953125,
      "rewards/rejected": 0.03842773288488388,
      "step": 285
    },
    {
      "epoch": 0.07466529351184346,
      "grad_norm": 255.0,
      "learning_rate": 4.6266735324407824e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.234375,
      "logps/chosen": -263.79998779296875,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6742,
      "rewards/accuracies": 0.5057691931724548,
      "rewards/chosen": 0.09726562350988388,
      "rewards/margins": 0.07226181030273438,
      "rewards/rejected": 0.02507324144244194,
      "step": 290
    },
    {
      "epoch": 0.07595262615859938,
      "grad_norm": 262.0,
      "learning_rate": 4.620236869207003e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -332.20001220703125,
      "logps/rejected": -308.3999938964844,
      "loss": 0.6453,
      "rewards/accuracies": 0.5708791017532349,
      "rewards/chosen": 0.16005858778953552,
      "rewards/margins": 0.12479858100414276,
      "rewards/rejected": 0.03510742262005806,
      "step": 295
    },
    {
      "epoch": 0.07723995880535531,
      "grad_norm": 241.0,
      "learning_rate": 4.613800205973223e-07,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.081249952316284,
      "logps/chosen": -268.0,
      "logps/rejected": -237.89999389648438,
      "loss": 0.6641,
      "rewards/accuracies": 0.4896644949913025,
      "rewards/chosen": 0.12031249701976776,
      "rewards/margins": 0.09135742485523224,
      "rewards/rejected": 0.02875976637005806,
      "step": 300
    },
    {
      "epoch": 0.07852729145211122,
      "grad_norm": 232.0,
      "learning_rate": 4.6073635427394437e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -313.79998779296875,
      "logps/rejected": -268.20001220703125,
      "loss": 0.6562,
      "rewards/accuracies": 0.5767857432365417,
      "rewards/chosen": 0.07949218899011612,
      "rewards/margins": 0.11306152492761612,
      "rewards/rejected": -0.03364257887005806,
      "step": 305
    },
    {
      "epoch": 0.07981462409886715,
      "grad_norm": 296.0,
      "learning_rate": 4.6009268795056636e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -301.6000061035156,
      "logps/rejected": -255.8000030517578,
      "loss": 0.6539,
      "rewards/accuracies": 0.643333375453949,
      "rewards/chosen": 0.14833983778953552,
      "rewards/margins": 0.11479492485523224,
      "rewards/rejected": 0.03383789211511612,
      "step": 310
    },
    {
      "epoch": 0.08110195674562307,
      "grad_norm": 210.0,
      "learning_rate": 4.5944902162718845e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -243.39999389648438,
      "logps/rejected": -267.79998779296875,
      "loss": 0.6734,
      "rewards/accuracies": 0.5199267864227295,
      "rewards/chosen": 0.13408203423023224,
      "rewards/margins": 0.0811767578125,
      "rewards/rejected": 0.0528564453125,
      "step": 315
    },
    {
      "epoch": 0.082389289392379,
      "grad_norm": 268.0,
      "learning_rate": 4.588053553038105e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -238.39999389648438,
      "logps/rejected": -196.89999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.6315018534660339,
      "rewards/chosen": 0.12851563096046448,
      "rewards/margins": 0.1219482421875,
      "rewards/rejected": 0.0067382813431322575,
      "step": 320
    },
    {
      "epoch": 0.08367662203913491,
      "grad_norm": 230.0,
      "learning_rate": 4.581616889804325e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.15625,
      "logps/chosen": -285.79998779296875,
      "logps/rejected": -260.79998779296875,
      "loss": 0.668,
      "rewards/accuracies": 0.5408333539962769,
      "rewards/chosen": 0.13496093451976776,
      "rewards/margins": 0.07763671875,
      "rewards/rejected": 0.057373046875,
      "step": 325
    },
    {
      "epoch": 0.08496395468589084,
      "grad_norm": 214.0,
      "learning_rate": 4.575180226570546e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -261.3999938964844,
      "logps/rejected": -247.0,
      "loss": 0.643,
      "rewards/accuracies": 0.5932575464248657,
      "rewards/chosen": 0.08861084282398224,
      "rewards/margins": 0.13564452528953552,
      "rewards/rejected": -0.04672851413488388,
      "step": 330
    },
    {
      "epoch": 0.08625128733264675,
      "grad_norm": 548.0,
      "learning_rate": 4.568743563336766e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -248.60000610351562,
      "logps/rejected": -188.6999969482422,
      "loss": 0.7188,
      "rewards/accuracies": 0.5567857027053833,
      "rewards/chosen": 0.11767578125,
      "rewards/margins": 0.018798828125,
      "rewards/rejected": 0.09892578423023224,
      "step": 335
    },
    {
      "epoch": 0.08753861997940268,
      "grad_norm": 298.0,
      "learning_rate": 4.562306900102986e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.21875,
      "logps/chosen": -206.8000030517578,
      "logps/rejected": -196.8000030517578,
      "loss": 0.6859,
      "rewards/accuracies": 0.44714289903640747,
      "rewards/chosen": 0.09848632663488388,
      "rewards/margins": 0.04213256761431694,
      "rewards/rejected": 0.05634765699505806,
      "step": 340
    },
    {
      "epoch": 0.0888259526261586,
      "grad_norm": 458.0,
      "learning_rate": 4.555870236869207e-07,
      "logits/chosen": -3.328125,
      "logits/rejected": -2.895312547683716,
      "logps/chosen": -138.35000610351562,
      "logps/rejected": -167.8000030517578,
      "loss": 0.6336,
      "rewards/accuracies": 0.6295238733291626,
      "rewards/chosen": 0.11826171725988388,
      "rewards/margins": 0.14248046278953552,
      "rewards/rejected": -0.024169921875,
      "step": 345
    },
    {
      "epoch": 0.09011328527291453,
      "grad_norm": 204.0,
      "learning_rate": 4.549433573635427e-07,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -235.0,
      "loss": 0.6328,
      "rewards/accuracies": 0.6198626756668091,
      "rewards/chosen": 0.19970703125,
      "rewards/margins": 0.1605224609375,
      "rewards/rejected": 0.039306640625,
      "step": 350
    },
    {
      "epoch": 0.09140061791967044,
      "grad_norm": 346.0,
      "learning_rate": 4.5429969104016474e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.0374999046325684,
      "logps/chosen": -264.6000061035156,
      "logps/rejected": -214.60000610351562,
      "loss": 0.6555,
      "rewards/accuracies": 0.5372436046600342,
      "rewards/chosen": 0.13188476860523224,
      "rewards/margins": 0.1103515625,
      "rewards/rejected": 0.02174072340130806,
      "step": 355
    },
    {
      "epoch": 0.09268795056642637,
      "grad_norm": 225.0,
      "learning_rate": 4.5365602471678684e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -267.0,
      "loss": 0.6477,
      "rewards/accuracies": 0.6477381587028503,
      "rewards/chosen": 0.15468749403953552,
      "rewards/margins": 0.12026367336511612,
      "rewards/rejected": 0.03466796875,
      "step": 360
    },
    {
      "epoch": 0.09397528321318228,
      "grad_norm": 258.0,
      "learning_rate": 4.5301235839340883e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -329.6000061035156,
      "logps/rejected": -243.39999389648438,
      "loss": 0.6422,
      "rewards/accuracies": 0.5434964895248413,
      "rewards/chosen": 0.13984374701976776,
      "rewards/margins": 0.1405717432498932,
      "rewards/rejected": -0.0003417968691792339,
      "step": 365
    },
    {
      "epoch": 0.09526261585993821,
      "grad_norm": 243.0,
      "learning_rate": 4.5236869207003087e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -275.79998779296875,
      "logps/rejected": -245.0,
      "loss": 0.6664,
      "rewards/accuracies": 0.5366666913032532,
      "rewards/chosen": 0.106201171875,
      "rewards/margins": 0.09165038913488388,
      "rewards/rejected": 0.01436767540872097,
      "step": 370
    },
    {
      "epoch": 0.09654994850669413,
      "grad_norm": 260.0,
      "learning_rate": 4.517250257466529e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -328.20001220703125,
      "logps/rejected": -286.79998779296875,
      "loss": 0.5961,
      "rewards/accuracies": 0.6678571701049805,
      "rewards/chosen": 0.23203125596046448,
      "rewards/margins": 0.24375000596046448,
      "rewards/rejected": -0.01169433631002903,
      "step": 375
    },
    {
      "epoch": 0.09783728115345006,
      "grad_norm": 214.0,
      "learning_rate": 4.5108135942327496e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -210.6999969482422,
      "loss": 0.6039,
      "rewards/accuracies": 0.6592033505439758,
      "rewards/chosen": 0.20136718451976776,
      "rewards/margins": 0.23183593153953552,
      "rewards/rejected": -0.0302276611328125,
      "step": 380
    },
    {
      "epoch": 0.09912461380020597,
      "grad_norm": 196.0,
      "learning_rate": 4.50437693099897e-07,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -297.79998779296875,
      "logps/rejected": -214.58749389648438,
      "loss": 0.625,
      "rewards/accuracies": 0.6364102363586426,
      "rewards/chosen": 0.18359375,
      "rewards/margins": 0.1669921875,
      "rewards/rejected": 0.01673584058880806,
      "step": 385
    },
    {
      "epoch": 0.1004119464469619,
      "grad_norm": 245.0,
      "learning_rate": 4.49794026776519e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.21875,
      "logps/chosen": -323.3999938964844,
      "logps/rejected": -264.3999938964844,
      "loss": 0.6367,
      "rewards/accuracies": 0.566428542137146,
      "rewards/chosen": 0.16875000298023224,
      "rewards/margins": 0.12900391221046448,
      "rewards/rejected": 0.039794921875,
      "step": 390
    },
    {
      "epoch": 0.10169927909371781,
      "grad_norm": 414.0,
      "learning_rate": 4.491503604531411e-07,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -2.981250047683716,
      "logps/chosen": -265.0,
      "logps/rejected": -183.60000610351562,
      "loss": 0.6336,
      "rewards/accuracies": 0.5235714316368103,
      "rewards/chosen": 0.17167969048023224,
      "rewards/margins": 0.17727050185203552,
      "rewards/rejected": -0.005963134579360485,
      "step": 395
    },
    {
      "epoch": 0.10298661174047374,
      "grad_norm": 256.0,
      "learning_rate": 4.4850669412976313e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -3.0218749046325684,
      "logps/chosen": -251.39999389648438,
      "logps/rejected": -237.8000030517578,
      "loss": 0.6555,
      "rewards/accuracies": 0.611334502696991,
      "rewards/chosen": 0.15288086235523224,
      "rewards/margins": 0.11357422173023224,
      "rewards/rejected": 0.03916015475988388,
      "step": 400
    },
    {
      "epoch": 0.10427394438722966,
      "grad_norm": 240.0,
      "learning_rate": 4.478630278063851e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -250.1999969482422,
      "loss": 0.6461,
      "rewards/accuracies": 0.5133333206176758,
      "rewards/chosen": 0.20488281548023224,
      "rewards/margins": 0.13090820610523224,
      "rewards/rejected": 0.07392577826976776,
      "step": 405
    },
    {
      "epoch": 0.10556127703398559,
      "grad_norm": 238.0,
      "learning_rate": 4.472193614830072e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -282.20001220703125,
      "logps/rejected": -236.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.1527099609375,
      "rewards/margins": 0.1650390625,
      "rewards/rejected": -0.01252136193215847,
      "step": 410
    },
    {
      "epoch": 0.1068486096807415,
      "grad_norm": 286.0,
      "learning_rate": 4.4657569515962926e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0625,
      "logps/chosen": -228.39999389648438,
      "logps/rejected": -204.39999389648438,
      "loss": 0.6406,
      "rewards/accuracies": 0.5440934300422668,
      "rewards/chosen": 0.23945312201976776,
      "rewards/margins": 0.14858397841453552,
      "rewards/rejected": 0.09108886867761612,
      "step": 415
    },
    {
      "epoch": 0.10813594232749743,
      "grad_norm": 213.0,
      "learning_rate": 4.4593202883625124e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.3031249046325684,
      "logps/chosen": -287.3999938964844,
      "logps/rejected": -257.0,
      "loss": 0.6453,
      "rewards/accuracies": 0.6150000095367432,
      "rewards/chosen": 0.131591796875,
      "rewards/margins": 0.14150390028953552,
      "rewards/rejected": -0.009936523623764515,
      "step": 420
    },
    {
      "epoch": 0.10942327497425335,
      "grad_norm": 255.0,
      "learning_rate": 4.4528836251287334e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.140625,
      "logps/chosen": -265.6000061035156,
      "logps/rejected": -255.60000610351562,
      "loss": 0.6383,
      "rewards/accuracies": 0.6238095164299011,
      "rewards/chosen": 0.20429687201976776,
      "rewards/margins": 0.14882811903953552,
      "rewards/rejected": 0.05559081956744194,
      "step": 425
    },
    {
      "epoch": 0.11071060762100927,
      "grad_norm": 312.0,
      "learning_rate": 4.4464469618949533e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.15625,
      "logps/chosen": -240.60000610351562,
      "logps/rejected": -245.39999389648438,
      "loss": 0.6492,
      "rewards/accuracies": 0.5344047546386719,
      "rewards/chosen": 0.1466064453125,
      "rewards/margins": 0.12653808295726776,
      "rewards/rejected": 0.01989135704934597,
      "step": 430
    },
    {
      "epoch": 0.11199794026776519,
      "grad_norm": 262.0,
      "learning_rate": 4.4400102986611737e-07,
      "logits/chosen": -3.2437500953674316,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -330.3999938964844,
      "logps/rejected": -300.79998779296875,
      "loss": 0.6633,
      "rewards/accuracies": 0.4880952835083008,
      "rewards/chosen": 0.14853516221046448,
      "rewards/margins": 0.083740234375,
      "rewards/rejected": 0.06491699069738388,
      "step": 435
    },
    {
      "epoch": 0.11328527291452112,
      "grad_norm": 246.0,
      "learning_rate": 4.4335736354273947e-07,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -314.6000061035156,
      "logps/rejected": -277.20001220703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.549725353717804,
      "rewards/chosen": 0.20180663466453552,
      "rewards/margins": 0.10898437350988388,
      "rewards/rejected": 0.0927734375,
      "step": 440
    },
    {
      "epoch": 0.11457260556127703,
      "grad_norm": 184.0,
      "learning_rate": 4.4271369721936146e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -257.0,
      "logps/rejected": -234.8000030517578,
      "loss": 0.6188,
      "rewards/accuracies": 0.583614706993103,
      "rewards/chosen": 0.13186034560203552,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": -0.04471435397863388,
      "step": 445
    },
    {
      "epoch": 0.11585993820803296,
      "grad_norm": 270.0,
      "learning_rate": 4.420700308959835e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.059375047683716,
      "logps/chosen": -292.79998779296875,
      "logps/rejected": -303.79998779296875,
      "loss": 0.6555,
      "rewards/accuracies": 0.5223809480667114,
      "rewards/chosen": 0.13591308891773224,
      "rewards/margins": 0.11638183891773224,
      "rewards/rejected": 0.01955566368997097,
      "step": 450
    },
    {
      "epoch": 0.11714727085478888,
      "grad_norm": 221.0,
      "learning_rate": 4.414263645726055e-07,
      "logits/chosen": -3.28125,
      "logits/rejected": -3.315624952316284,
      "logps/chosen": -283.3999938964844,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6281,
      "rewards/accuracies": 0.6641666889190674,
      "rewards/chosen": 0.04499511793255806,
      "rewards/margins": 0.17336425185203552,
      "rewards/rejected": -0.12858887016773224,
      "step": 455
    },
    {
      "epoch": 0.1184346035015448,
      "grad_norm": 300.0,
      "learning_rate": 4.407826982492276e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -254.1999969482422,
      "logps/rejected": -243.8000030517578,
      "loss": 0.6594,
      "rewards/accuracies": 0.48076924681663513,
      "rewards/chosen": 0.07998047024011612,
      "rewards/margins": 0.11005859076976776,
      "rewards/rejected": -0.03038330003619194,
      "step": 460
    },
    {
      "epoch": 0.11972193614830072,
      "grad_norm": 253.0,
      "learning_rate": 4.4013903192584963e-07,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.034374952316284,
      "logps/chosen": -234.8000030517578,
      "logps/rejected": -254.85000610351562,
      "loss": 0.6398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.24765625596046448,
      "rewards/margins": 0.13007812201976776,
      "rewards/rejected": 0.11748047173023224,
      "step": 465
    },
    {
      "epoch": 0.12100926879505665,
      "grad_norm": 274.0,
      "learning_rate": 4.394953656024716e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -245.8000030517578,
      "loss": 0.6312,
      "rewards/accuracies": 0.5233333706855774,
      "rewards/chosen": 0.22421875596046448,
      "rewards/margins": 0.15517577528953552,
      "rewards/rejected": 0.06918945163488388,
      "step": 470
    },
    {
      "epoch": 0.12229660144181256,
      "grad_norm": 174.0,
      "learning_rate": 4.388516992790937e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -257.20001220703125,
      "logps/rejected": -276.0,
      "loss": 0.6328,
      "rewards/accuracies": 0.5576923489570618,
      "rewards/chosen": 0.07561035454273224,
      "rewards/margins": 0.16567382216453552,
      "rewards/rejected": -0.08984375,
      "step": 475
    },
    {
      "epoch": 0.12358393408856849,
      "grad_norm": 434.0,
      "learning_rate": 4.3820803295571576e-07,
      "logits/chosen": -3.028125047683716,
      "logits/rejected": -3.09375,
      "logps/chosen": -299.3999938964844,
      "logps/rejected": -247.875,
      "loss": 0.6,
      "rewards/accuracies": 0.6367856860160828,
      "rewards/chosen": 0.24990233778953552,
      "rewards/margins": 0.22617188096046448,
      "rewards/rejected": 0.023681640625,
      "step": 480
    },
    {
      "epoch": 0.12487126673532441,
      "grad_norm": 237.0,
      "learning_rate": 4.3756436663233775e-07,
      "logits/chosen": NaN,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -267.79998779296875,
      "logps/rejected": -255.74374389648438,
      "loss": 0.6586,
      "rewards/accuracies": 0.6613888740539551,
      "rewards/chosen": 0.14687499403953552,
      "rewards/margins": 0.11721191555261612,
      "rewards/rejected": 0.02955322340130806,
      "step": 485
    },
    {
      "epoch": 0.12615859938208032,
      "grad_norm": 238.0,
      "learning_rate": 4.3692070030895984e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.1875,
      "logps/chosen": -239.1999969482422,
      "logps/rejected": -301.3999938964844,
      "loss": 0.6836,
      "rewards/accuracies": 0.5560714602470398,
      "rewards/chosen": 0.03804931789636612,
      "rewards/margins": 0.07314453274011612,
      "rewards/rejected": -0.03505859524011612,
      "step": 490
    },
    {
      "epoch": 0.12744593202883625,
      "grad_norm": 225.0,
      "learning_rate": 4.3627703398558183e-07,
      "logits/chosen": -3.0062499046325684,
      "logits/rejected": -2.9749999046325684,
      "logps/chosen": -251.1999969482422,
      "logps/rejected": -293.3999938964844,
      "loss": 0.6414,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.24091796576976776,
      "rewards/margins": 0.14521484076976776,
      "rewards/rejected": 0.09550781548023224,
      "step": 495
    },
    {
      "epoch": 0.12873326467559218,
      "grad_norm": 201.0,
      "learning_rate": 4.356333676622039e-07,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -2.875,
      "logps/chosen": -249.60000610351562,
      "logps/rejected": -220.1999969482422,
      "loss": 0.6102,
      "rewards/accuracies": 0.6730769276618958,
      "rewards/chosen": 0.1941215544939041,
      "rewards/margins": 0.21054688096046448,
      "rewards/rejected": -0.01669921912252903,
      "step": 500
    },
    {
      "epoch": 0.1300205973223481,
      "grad_norm": 205.0,
      "learning_rate": 4.3498970133882597e-07,
      "logits/chosen": NaN,
      "logits/rejected": -3.043750047683716,
      "logps/chosen": -222.0,
      "logps/rejected": -179.83438110351562,
      "loss": 0.6242,
      "rewards/accuracies": 0.7066667079925537,
      "rewards/chosen": 0.15703125298023224,
      "rewards/margins": 0.18059082329273224,
      "rewards/rejected": -0.02338867262005806,
      "step": 505
    },
    {
      "epoch": 0.131307929969104,
      "grad_norm": 180.0,
      "learning_rate": 4.3434603501544796e-07,
      "logits/chosen": -3.315624952316284,
      "logits/rejected": -3.3531250953674316,
      "logps/chosen": -284.6000061035156,
      "logps/rejected": -251.0,
      "loss": 0.6258,
      "rewards/accuracies": 0.6460714936256409,
      "rewards/chosen": 0.12263183295726776,
      "rewards/margins": 0.18414306640625,
      "rewards/rejected": -0.06181640550494194,
      "step": 510
    },
    {
      "epoch": 0.13259526261585994,
      "grad_norm": 268.0,
      "learning_rate": 4.3370236869207e-07,
      "logits/chosen": -3.0250000953674316,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -233.60000610351562,
      "logps/rejected": -213.8000030517578,
      "loss": 0.6398,
      "rewards/accuracies": 0.5933441519737244,
      "rewards/chosen": 0.24109497666358948,
      "rewards/margins": 0.15725097060203552,
      "rewards/rejected": 0.08370361477136612,
      "step": 515
    },
    {
      "epoch": 0.13388259526261587,
      "grad_norm": 189.0,
      "learning_rate": 4.330587023686921e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -282.79998779296875,
      "loss": 0.6195,
      "rewards/accuracies": 0.6032575368881226,
      "rewards/chosen": 0.14301757514476776,
      "rewards/margins": 0.1884765625,
      "rewards/rejected": -0.04595947265625,
      "step": 520
    },
    {
      "epoch": 0.1351699279093718,
      "grad_norm": 230.0,
      "learning_rate": 4.324150360453141e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -307.20001220703125,
      "logps/rejected": -220.60000610351562,
      "loss": 0.6234,
      "rewards/accuracies": 0.6148592829704285,
      "rewards/chosen": 0.2601562440395355,
      "rewards/margins": 0.17949219048023224,
      "rewards/rejected": 0.08090820163488388,
      "step": 525
    },
    {
      "epoch": 0.1364572605561277,
      "grad_norm": 496.0,
      "learning_rate": 4.3177136972193613e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -271.79998779296875,
      "logps/rejected": -249.60000610351562,
      "loss": 0.6461,
      "rewards/accuracies": 0.5240908861160278,
      "rewards/chosen": 0.07033691555261612,
      "rewards/margins": 0.14453125,
      "rewards/rejected": -0.07421875,
      "step": 530
    },
    {
      "epoch": 0.13774459320288363,
      "grad_norm": 268.0,
      "learning_rate": 4.311277033985581e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -252.39999389648438,
      "logps/rejected": -278.20001220703125,
      "loss": 0.6797,
      "rewards/accuracies": 0.5016841292381287,
      "rewards/chosen": 0.16220703721046448,
      "rewards/margins": 0.064208984375,
      "rewards/rejected": 0.097900390625,
      "step": 535
    },
    {
      "epoch": 0.13903192584963955,
      "grad_norm": 243.0,
      "learning_rate": 4.304840370751802e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -2.780468702316284,
      "logps/chosen": -268.5,
      "logps/rejected": -280.0,
      "loss": 0.7602,
      "rewards/accuracies": 0.4908333420753479,
      "rewards/chosen": 0.19272461533546448,
      "rewards/margins": -0.02329101599752903,
      "rewards/rejected": 0.21562500298023224,
      "step": 540
    },
    {
      "epoch": 0.14031925849639545,
      "grad_norm": 228.0,
      "learning_rate": 4.2984037075180226e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -252.39999389648438,
      "logps/rejected": -230.60000610351562,
      "loss": 0.607,
      "rewards/accuracies": 0.5980011820793152,
      "rewards/chosen": 0.15168151259422302,
      "rewards/margins": 0.22832031548023224,
      "rewards/rejected": -0.07638225704431534,
      "step": 545
    },
    {
      "epoch": 0.14160659114315138,
      "grad_norm": 189.0,
      "learning_rate": 4.2919670442842425e-07,
      "logits/chosen": -3.296875,
      "logits/rejected": -3.387500047683716,
      "logps/chosen": -306.6000061035156,
      "logps/rejected": -241.1999969482422,
      "loss": 0.6438,
      "rewards/accuracies": 0.552063524723053,
      "rewards/chosen": 0.16010741889476776,
      "rewards/margins": 0.15078124403953552,
      "rewards/rejected": 0.00931396521627903,
      "step": 550
    },
    {
      "epoch": 0.1428939237899073,
      "grad_norm": 438.0,
      "learning_rate": 4.2855303810504634e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -236.8000030517578,
      "logps/rejected": -216.60000610351562,
      "loss": 0.6508,
      "rewards/accuracies": 0.5377289652824402,
      "rewards/chosen": 0.16602173447608948,
      "rewards/margins": 0.123046875,
      "rewards/rejected": 0.043180275708436966,
      "step": 555
    },
    {
      "epoch": 0.14418125643666324,
      "grad_norm": 262.0,
      "learning_rate": 4.2790937178166833e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -243.1999969482422,
      "logps/rejected": -215.1999969482422,
      "loss": 0.6617,
      "rewards/accuracies": 0.4709523618221283,
      "rewards/chosen": 0.15131835639476776,
      "rewards/margins": 0.10859374701976776,
      "rewards/rejected": 0.04287109524011612,
      "step": 560
    },
    {
      "epoch": 0.14546858908341914,
      "grad_norm": 219.0,
      "learning_rate": 4.272657054582904e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -2.9749999046325684,
      "logps/chosen": -275.20001220703125,
      "logps/rejected": -243.3000030517578,
      "loss": 0.6172,
      "rewards/accuracies": 0.6569444537162781,
      "rewards/chosen": 0.17167969048023224,
      "rewards/margins": 0.21048584580421448,
      "rewards/rejected": -0.03852539137005806,
      "step": 565
    },
    {
      "epoch": 0.14675592173017507,
      "grad_norm": 268.0,
      "learning_rate": 4.2662203913491247e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -2.8499999046325684,
      "logps/chosen": -263.3999938964844,
      "logps/rejected": -236.1999969482422,
      "loss": 0.6273,
      "rewards/accuracies": 0.5721403956413269,
      "rewards/chosen": 0.14580078423023224,
      "rewards/margins": 0.20371094346046448,
      "rewards/rejected": -0.05732421949505806,
      "step": 570
    },
    {
      "epoch": 0.148043254376931,
      "grad_norm": 249.0,
      "learning_rate": 4.2597837281153446e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -226.1999969482422,
      "logps/rejected": -289.6000061035156,
      "loss": 0.6633,
      "rewards/accuracies": 0.6109523773193359,
      "rewards/chosen": 0.13151855766773224,
      "rewards/margins": 0.14345093071460724,
      "rewards/rejected": -0.01242675818502903,
      "step": 575
    },
    {
      "epoch": 0.14933058702368693,
      "grad_norm": 191.0,
      "learning_rate": 4.253347064881565e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -2.9625000953674316,
      "logps/chosen": -270.79998779296875,
      "logps/rejected": -262.0,
      "loss": 0.6227,
      "rewards/accuracies": 0.6691269874572754,
      "rewards/chosen": 0.165771484375,
      "rewards/margins": 0.18939819931983948,
      "rewards/rejected": -0.02363281324505806,
      "step": 580
    },
    {
      "epoch": 0.15061791967044283,
      "grad_norm": 204.0,
      "learning_rate": 4.246910401647786e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -298.3999938964844,
      "logps/rejected": -263.79998779296875,
      "loss": 0.5992,
      "rewards/accuracies": 0.6393939852714539,
      "rewards/chosen": 0.25654298067092896,
      "rewards/margins": 0.23232421278953552,
      "rewards/rejected": 0.02470703050494194,
      "step": 585
    },
    {
      "epoch": 0.15190525231719876,
      "grad_norm": 302.0,
      "learning_rate": 4.240473738414006e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.09375,
      "logps/chosen": -266.20001220703125,
      "logps/rejected": -220.6999969482422,
      "loss": 0.6742,
      "rewards/accuracies": 0.47340911626815796,
      "rewards/chosen": 0.26704102754592896,
      "rewards/margins": 0.10516051948070526,
      "rewards/rejected": 0.16157226264476776,
      "step": 590
    },
    {
      "epoch": 0.1531925849639547,
      "grad_norm": 240.0,
      "learning_rate": 4.2340370751802263e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.25,
      "logps/chosen": -323.20001220703125,
      "logps/rejected": -256.6000061035156,
      "loss": 0.5766,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": 0.17265625298023224,
      "rewards/margins": 0.29521483182907104,
      "rewards/rejected": -0.12239990383386612,
      "step": 595
    },
    {
      "epoch": 0.15447991761071062,
      "grad_norm": 176.0,
      "learning_rate": 4.227600411946447e-07,
      "logits/chosen": -3.246875047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -279.3999938964844,
      "logps/rejected": -204.89999389648438,
      "loss": 0.6156,
      "rewards/accuracies": 0.6200000047683716,
      "rewards/chosen": 0.16535644233226776,
      "rewards/margins": 0.21303710341453552,
      "rewards/rejected": -0.04716796800494194,
      "step": 600
    },
    {
      "epoch": 0.15576725025746652,
      "grad_norm": 251.0,
      "learning_rate": 4.221163748712667e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -353.6000061035156,
      "logps/rejected": -286.0,
      "loss": 0.6422,
      "rewards/accuracies": 0.6336538791656494,
      "rewards/chosen": 0.19008788466453552,
      "rewards/margins": 0.1552734375,
      "rewards/rejected": 0.0341796875,
      "step": 605
    },
    {
      "epoch": 0.15705458290422245,
      "grad_norm": 191.0,
      "learning_rate": 4.2147270854788876e-07,
      "logits/chosen": -3.2718749046325684,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -233.6999969482422,
      "logps/rejected": -212.3000030517578,
      "loss": 0.6336,
      "rewards/accuracies": 0.5709524154663086,
      "rewards/chosen": 0.12265624850988388,
      "rewards/margins": 0.175537109375,
      "rewards/rejected": -0.05327148362994194,
      "step": 610
    },
    {
      "epoch": 0.15834191555097837,
      "grad_norm": 201.0,
      "learning_rate": 4.208290422245108e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -275.8999938964844,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6508,
      "rewards/accuracies": 0.5587338209152222,
      "rewards/chosen": 0.14438477158546448,
      "rewards/margins": 0.13203124701976776,
      "rewards/rejected": 0.012255859561264515,
      "step": 615
    },
    {
      "epoch": 0.1596292481977343,
      "grad_norm": 216.0,
      "learning_rate": 4.2018537590113285e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -252.0,
      "logps/rejected": -307.20001220703125,
      "loss": 0.6656,
      "rewards/accuracies": 0.6060714721679688,
      "rewards/chosen": 0.04769287258386612,
      "rewards/margins": 0.111328125,
      "rewards/rejected": -0.06374511867761612,
      "step": 620
    },
    {
      "epoch": 0.1609165808444902,
      "grad_norm": 196.0,
      "learning_rate": 4.195417095777549e-07,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -242.8000030517578,
      "logps/rejected": -213.1999969482422,
      "loss": 0.5875,
      "rewards/accuracies": 0.7047619819641113,
      "rewards/chosen": 0.21389159560203552,
      "rewards/margins": 0.27070313692092896,
      "rewards/rejected": -0.0565185546875,
      "step": 625
    },
    {
      "epoch": 0.16220391349124613,
      "grad_norm": 231.0,
      "learning_rate": 4.188980432543769e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.0531249046325684,
      "logps/chosen": -241.89999389648438,
      "logps/rejected": -196.5,
      "loss": 0.6273,
      "rewards/accuracies": 0.6974242925643921,
      "rewards/chosen": 0.107421875,
      "rewards/margins": 0.20395508408546448,
      "rewards/rejected": -0.09663085639476776,
      "step": 630
    },
    {
      "epoch": 0.16349124613800206,
      "grad_norm": 246.0,
      "learning_rate": 4.18254376930999e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -299.0,
      "logps/rejected": -277.79998779296875,
      "loss": 0.607,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": 0.186798095703125,
      "rewards/margins": 0.22499999403953552,
      "rewards/rejected": -0.03779296949505806,
      "step": 635
    },
    {
      "epoch": 0.164778578784758,
      "grad_norm": 262.0,
      "learning_rate": 4.1761071060762096e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.171875,
      "logps/chosen": -272.20001220703125,
      "logps/rejected": -258.0,
      "loss": 0.6164,
      "rewards/accuracies": 0.6571428775787354,
      "rewards/chosen": 0.17044372856616974,
      "rewards/margins": 0.21835938096046448,
      "rewards/rejected": -0.04746093600988388,
      "step": 640
    },
    {
      "epoch": 0.1660659114315139,
      "grad_norm": 390.0,
      "learning_rate": 4.16967044284243e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.21875,
      "logps/chosen": -300.0,
      "logps/rejected": -268.20001220703125,
      "loss": 0.6445,
      "rewards/accuracies": 0.6309524178504944,
      "rewards/chosen": 0.18796387314796448,
      "rewards/margins": 0.15449218451976776,
      "rewards/rejected": 0.03386840969324112,
      "step": 645
    },
    {
      "epoch": 0.16735324407826982,
      "grad_norm": 358.0,
      "learning_rate": 4.163233779608651e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.0218749046325684,
      "logps/chosen": -270.6000061035156,
      "logps/rejected": -232.0,
      "loss": 0.6094,
      "rewards/accuracies": 0.5803418755531311,
      "rewards/chosen": 0.2953124940395355,
      "rewards/margins": 0.23223876953125,
      "rewards/rejected": 0.06291504204273224,
      "step": 650
    },
    {
      "epoch": 0.16864057672502575,
      "grad_norm": 214.0,
      "learning_rate": 4.156797116374871e-07,
      "logits/chosen": -2.903125047683716,
      "logits/rejected": -2.840625047683716,
      "logps/chosen": -170.75,
      "logps/rejected": -184.3000030517578,
      "loss": 0.6234,
      "rewards/accuracies": 0.6217948794364929,
      "rewards/chosen": 0.17540283501148224,
      "rewards/margins": 0.17917481064796448,
      "rewards/rejected": -0.0037597655318677425,
      "step": 655
    },
    {
      "epoch": 0.16992790937178168,
      "grad_norm": 206.0,
      "learning_rate": 4.1503604531410913e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -343.20001220703125,
      "logps/rejected": -275.20001220703125,
      "loss": 0.5852,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.25214844942092896,
      "rewards/margins": 0.3199218809604645,
      "rewards/rejected": -0.06767578423023224,
      "step": 660
    },
    {
      "epoch": 0.17121524201853758,
      "grad_norm": 193.0,
      "learning_rate": 4.1439237899073123e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.140625,
      "logps/chosen": -270.6000061035156,
      "logps/rejected": -298.6000061035156,
      "loss": 0.657,
      "rewards/accuracies": 0.519505500793457,
      "rewards/chosen": 0.08811035007238388,
      "rewards/margins": 0.12363891303539276,
      "rewards/rejected": -0.0352783203125,
      "step": 665
    },
    {
      "epoch": 0.1725025746652935,
      "grad_norm": 185.0,
      "learning_rate": 4.137487126673532e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -2.9781250953674316,
      "logps/chosen": -223.39999389648438,
      "logps/rejected": -207.60000610351562,
      "loss": 0.5938,
      "rewards/accuracies": 0.7043687105178833,
      "rewards/chosen": 0.22177734971046448,
      "rewards/margins": 0.2412109375,
      "rewards/rejected": -0.02001953125,
      "step": 670
    },
    {
      "epoch": 0.17378990731204944,
      "grad_norm": 302.0,
      "learning_rate": 4.1310504634397526e-07,
      "logits/chosen": -3.0875000953674316,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -347.20001220703125,
      "logps/rejected": -248.1999969482422,
      "loss": 0.6234,
      "rewards/accuracies": 0.6422619223594666,
      "rewards/chosen": 0.24619141221046448,
      "rewards/margins": 0.20126953721046448,
      "rewards/rejected": 0.0447998046875,
      "step": 675
    },
    {
      "epoch": 0.17507723995880536,
      "grad_norm": 201.0,
      "learning_rate": 4.124613800205973e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.09375,
      "logps/chosen": -326.6000061035156,
      "logps/rejected": -340.0,
      "loss": 0.6258,
      "rewards/accuracies": 0.5663461685180664,
      "rewards/chosen": 0.16782227158546448,
      "rewards/margins": 0.19741210341453552,
      "rewards/rejected": -0.02956542931497097,
      "step": 680
    },
    {
      "epoch": 0.17636457260556127,
      "grad_norm": 217.0,
      "learning_rate": 4.1181771369721935e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -327.6000061035156,
      "logps/rejected": -251.60000610351562,
      "loss": 0.6195,
      "rewards/accuracies": 0.6480768918991089,
      "rewards/chosen": 0.27167969942092896,
      "rewards/margins": 0.21914061903953552,
      "rewards/rejected": 0.05280761793255806,
      "step": 685
    },
    {
      "epoch": 0.1776519052523172,
      "grad_norm": 243.0,
      "learning_rate": 4.111740473738414e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -326.3999938964844,
      "loss": 0.668,
      "rewards/accuracies": 0.6235897541046143,
      "rewards/chosen": 0.1357421875,
      "rewards/margins": 0.11796875298023224,
      "rewards/rejected": 0.01754150353372097,
      "step": 690
    },
    {
      "epoch": 0.17893923789907312,
      "grad_norm": 282.0,
      "learning_rate": 4.1053038105046343e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.296875,
      "logps/chosen": -340.3999938964844,
      "logps/rejected": -301.20001220703125,
      "loss": 0.6008,
      "rewards/accuracies": 0.6414102911949158,
      "rewards/chosen": 0.19970703125,
      "rewards/margins": 0.2802734375,
      "rewards/rejected": -0.080078125,
      "step": 695
    },
    {
      "epoch": 0.18022657054582905,
      "grad_norm": 442.0,
      "learning_rate": 4.098867147270855e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.2906250953674316,
      "logps/chosen": -289.0,
      "logps/rejected": -241.60000610351562,
      "loss": 0.632,
      "rewards/accuracies": 0.5610256791114807,
      "rewards/chosen": 0.23662109673023224,
      "rewards/margins": 0.208984375,
      "rewards/rejected": 0.02768554724752903,
      "step": 700
    },
    {
      "epoch": 0.18151390319258495,
      "grad_norm": 210.0,
      "learning_rate": 4.0924304840370747e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.28125,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -291.6000061035156,
      "loss": 0.6094,
      "rewards/accuracies": 0.6103571653366089,
      "rewards/chosen": 0.11777343600988388,
      "rewards/margins": 0.24140624701976776,
      "rewards/rejected": -0.12333984673023224,
      "step": 705
    },
    {
      "epoch": 0.18280123583934088,
      "grad_norm": 237.0,
      "learning_rate": 4.085993820803295e-07,
      "logits/chosen": -3.2437500953674316,
      "logits/rejected": -3.21875,
      "logps/chosen": -324.79998779296875,
      "logps/rejected": -301.3999938964844,
      "loss": 0.5977,
      "rewards/accuracies": 0.6791666746139526,
      "rewards/chosen": 0.26738280057907104,
      "rewards/margins": 0.25117188692092896,
      "rewards/rejected": 0.01658935472369194,
      "step": 710
    },
    {
      "epoch": 0.1840885684860968,
      "grad_norm": 218.0,
      "learning_rate": 4.079557157569516e-07,
      "logits/chosen": -2.871875047683716,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -237.02499389648438,
      "logps/rejected": -221.0,
      "loss": 0.6215,
      "rewards/accuracies": 0.5633333921432495,
      "rewards/chosen": 0.4273437559604645,
      "rewards/margins": 0.31855469942092896,
      "rewards/rejected": 0.10792236030101776,
      "step": 715
    },
    {
      "epoch": 0.18537590113285274,
      "grad_norm": 202.0,
      "learning_rate": 4.073120494335736e-07,
      "logits/chosen": -3.3531250953674316,
      "logits/rejected": -3.3343749046325684,
      "logps/chosen": -289.20001220703125,
      "logps/rejected": -235.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.6198809742927551,
      "rewards/chosen": 0.13872070610523224,
      "rewards/margins": 0.20166015625,
      "rewards/rejected": -0.06298828125,
      "step": 720
    },
    {
      "epoch": 0.18666323377960864,
      "grad_norm": 208.0,
      "learning_rate": 4.0666838311019564e-07,
      "logits/chosen": -3.0062499046325684,
      "logits/rejected": -2.971874952316284,
      "logps/chosen": -289.0,
      "logps/rejected": -233.60000610351562,
      "loss": 0.5961,
      "rewards/accuracies": 0.6740584373474121,
      "rewards/chosen": 0.2061767578125,
      "rewards/margins": 0.2694335877895355,
      "rewards/rejected": -0.06339111179113388,
      "step": 725
    },
    {
      "epoch": 0.18795056642636457,
      "grad_norm": 358.0,
      "learning_rate": 4.0602471678681773e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.015625,
      "logps/chosen": -275.79998779296875,
      "logps/rejected": -275.6000061035156,
      "loss": 0.6813,
      "rewards/accuracies": 0.5667948722839355,
      "rewards/chosen": 0.09931640326976776,
      "rewards/margins": 0.0859375,
      "rewards/rejected": 0.01352539099752903,
      "step": 730
    },
    {
      "epoch": 0.1892378990731205,
      "grad_norm": 227.0,
      "learning_rate": 4.053810504634397e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -321.3999938964844,
      "logps/rejected": -320.79998779296875,
      "loss": 0.6273,
      "rewards/accuracies": 0.5446678400039673,
      "rewards/chosen": 0.237548828125,
      "rewards/margins": 0.19931641221046448,
      "rewards/rejected": 0.03852539137005806,
      "step": 735
    },
    {
      "epoch": 0.19052523171987643,
      "grad_norm": 255.0,
      "learning_rate": 4.0473738414006176e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -238.1999969482422,
      "logps/rejected": -232.60000610351562,
      "loss": 0.6367,
      "rewards/accuracies": 0.6466667056083679,
      "rewards/chosen": 0.18713530898094177,
      "rewards/margins": 0.15830078721046448,
      "rewards/rejected": 0.0286865234375,
      "step": 740
    },
    {
      "epoch": 0.19181256436663233,
      "grad_norm": 216.0,
      "learning_rate": 4.040937178166838e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.299999952316284,
      "logps/chosen": -314.6000061035156,
      "logps/rejected": -236.8000030517578,
      "loss": 0.5742,
      "rewards/accuracies": 0.754978358745575,
      "rewards/chosen": 0.27873533964157104,
      "rewards/margins": 0.3228515684604645,
      "rewards/rejected": -0.04372558742761612,
      "step": 745
    },
    {
      "epoch": 0.19309989701338826,
      "grad_norm": 272.0,
      "learning_rate": 4.0345005149330585e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -283.6000061035156,
      "logps/rejected": -249.60000610351562,
      "loss": 0.6648,
      "rewards/accuracies": 0.6291667222976685,
      "rewards/chosen": 0.19106444716453552,
      "rewards/margins": 0.14719238877296448,
      "rewards/rejected": 0.04384765774011612,
      "step": 750
    },
    {
      "epoch": 0.19438722966014418,
      "grad_norm": 262.0,
      "learning_rate": 4.028063851699279e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -3.1875,
      "logps/chosen": -314.0,
      "logps/rejected": -282.0,
      "loss": 0.6086,
      "rewards/accuracies": 0.6943939328193665,
      "rewards/chosen": 0.2572265565395355,
      "rewards/margins": 0.25371092557907104,
      "rewards/rejected": 0.0033447265159338713,
      "step": 755
    },
    {
      "epoch": 0.1956745623069001,
      "grad_norm": 228.0,
      "learning_rate": 4.0216271884654994e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.25,
      "logps/chosen": -265.20001220703125,
      "logps/rejected": -270.0,
      "loss": 0.6531,
      "rewards/accuracies": 0.5880953073501587,
      "rewards/chosen": 0.08477783203125,
      "rewards/margins": 0.14360351860523224,
      "rewards/rejected": -0.05848388746380806,
      "step": 760
    },
    {
      "epoch": 0.196961894953656,
      "grad_norm": 221.0,
      "learning_rate": 4.01519052523172e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -247.0,
      "logps/rejected": -193.8000030517578,
      "loss": 0.6234,
      "rewards/accuracies": 0.6492033004760742,
      "rewards/chosen": 0.21699218451976776,
      "rewards/margins": 0.18706054985523224,
      "rewards/rejected": 0.02985839918255806,
      "step": 765
    },
    {
      "epoch": 0.19824922760041194,
      "grad_norm": 211.0,
      "learning_rate": 4.00875386199794e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.268749952316284,
      "logps/chosen": -280.6000061035156,
      "logps/rejected": -238.1999969482422,
      "loss": 0.5977,
      "rewards/accuracies": 0.6554545760154724,
      "rewards/chosen": 0.24062499403953552,
      "rewards/margins": 0.2793945372104645,
      "rewards/rejected": -0.03886718675494194,
      "step": 770
    },
    {
      "epoch": 0.19953656024716787,
      "grad_norm": 177.0,
      "learning_rate": 4.0023171987641606e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -253.60000610351562,
      "loss": 0.6141,
      "rewards/accuracies": 0.6825000047683716,
      "rewards/chosen": 0.14196166396141052,
      "rewards/margins": 0.24453124403953552,
      "rewards/rejected": -0.10202636569738388,
      "step": 775
    },
    {
      "epoch": 0.2008238928939238,
      "grad_norm": 181.0,
      "learning_rate": 3.995880535530381e-07,
      "logits/chosen": -3.0218749046325684,
      "logits/rejected": -3.340625047683716,
      "logps/chosen": -260.1000061035156,
      "logps/rejected": -247.6999969482422,
      "loss": 0.5875,
      "rewards/accuracies": 0.6575000286102295,
      "rewards/chosen": 0.21064452826976776,
      "rewards/margins": 0.27324217557907104,
      "rewards/rejected": -0.06279297173023224,
      "step": 780
    },
    {
      "epoch": 0.2021112255406797,
      "grad_norm": 205.0,
      "learning_rate": 3.989443872296601e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -243.8000030517578,
      "logps/rejected": -214.60000610351562,
      "loss": 0.6344,
      "rewards/accuracies": 0.5816666483879089,
      "rewards/chosen": 0.11201171576976776,
      "rewards/margins": 0.19550780951976776,
      "rewards/rejected": -0.08339843899011612,
      "step": 785
    },
    {
      "epoch": 0.20339855818743563,
      "grad_norm": 228.0,
      "learning_rate": 3.9830072090628214e-07,
      "logits/chosen": -2.934375047683716,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -277.3999938964844,
      "logps/rejected": -225.1999969482422,
      "loss": 0.6148,
      "rewards/accuracies": 0.6529004573822021,
      "rewards/chosen": 0.14278388023376465,
      "rewards/margins": 0.2520507872104645,
      "rewards/rejected": -0.10922851413488388,
      "step": 790
    },
    {
      "epoch": 0.20468589083419156,
      "grad_norm": 223.0,
      "learning_rate": 3.9765705458290423e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.25,
      "logps/chosen": -290.6000061035156,
      "logps/rejected": -257.0,
      "loss": 0.6297,
      "rewards/accuracies": 0.5804654359817505,
      "rewards/chosen": 0.30449217557907104,
      "rewards/margins": 0.18339844048023224,
      "rewards/rejected": 0.12095947563648224,
      "step": 795
    },
    {
      "epoch": 0.2059732234809475,
      "grad_norm": 174.0,
      "learning_rate": 3.970133882595262e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -296.79998779296875,
      "logps/rejected": -274.3999938964844,
      "loss": 0.6422,
      "rewards/accuracies": 0.6208333373069763,
      "rewards/chosen": 0.07084961235523224,
      "rewards/margins": 0.16098633408546448,
      "rewards/rejected": -0.09042968600988388,
      "step": 800
    },
    {
      "epoch": 0.2072605561277034,
      "grad_norm": 258.0,
      "learning_rate": 3.9636972193614827e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -262.3999938964844,
      "loss": 0.6328,
      "rewards/accuracies": 0.6117857694625854,
      "rewards/chosen": 0.22553710639476776,
      "rewards/margins": 0.20661620795726776,
      "rewards/rejected": 0.0189208984375,
      "step": 805
    },
    {
      "epoch": 0.20854788877445932,
      "grad_norm": 230.0,
      "learning_rate": 3.9572605561277036e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.0718750953674316,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -271.0,
      "loss": 0.6281,
      "rewards/accuracies": 0.591785728931427,
      "rewards/chosen": 0.167236328125,
      "rewards/margins": 0.18808594346046448,
      "rewards/rejected": -0.02089843712747097,
      "step": 810
    },
    {
      "epoch": 0.20983522142121525,
      "grad_norm": 192.0,
      "learning_rate": 3.9508238928939235e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.003124952316284,
      "logps/chosen": -254.1999969482422,
      "logps/rejected": -239.60000610351562,
      "loss": 0.6156,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.2745117247104645,
      "rewards/margins": 0.22275391221046448,
      "rewards/rejected": 0.05183105543255806,
      "step": 815
    },
    {
      "epoch": 0.21112255406797117,
      "grad_norm": 189.0,
      "learning_rate": 3.944387229660144e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -237.39999389648438,
      "logps/rejected": -181.5,
      "loss": 0.5898,
      "rewards/accuracies": 0.6586111187934875,
      "rewards/chosen": 0.23924560844898224,
      "rewards/margins": 0.31074219942092896,
      "rewards/rejected": -0.07222290337085724,
      "step": 820
    },
    {
      "epoch": 0.21240988671472708,
      "grad_norm": 266.0,
      "learning_rate": 3.9379505664263644e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.3343749046325684,
      "logps/chosen": -333.20001220703125,
      "logps/rejected": -295.20001220703125,
      "loss": 0.6445,
      "rewards/accuracies": 0.5610256791114807,
      "rewards/chosen": 0.19245605170726776,
      "rewards/margins": 0.17177733778953552,
      "rewards/rejected": 0.02036132849752903,
      "step": 825
    },
    {
      "epoch": 0.213697219361483,
      "grad_norm": 284.0,
      "learning_rate": 3.931513903192585e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -297.6000061035156,
      "logps/rejected": -258.6000061035156,
      "loss": 0.6039,
      "rewards/accuracies": 0.6251165270805359,
      "rewards/chosen": 0.300537109375,
      "rewards/margins": 0.24394531548023224,
      "rewards/rejected": 0.05673827975988388,
      "step": 830
    },
    {
      "epoch": 0.21498455200823893,
      "grad_norm": 318.0,
      "learning_rate": 3.925077239958805e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -268.0,
      "logps/rejected": -235.60000610351562,
      "loss": 0.5965,
      "rewards/accuracies": 0.6885029077529907,
      "rewards/chosen": 0.306640625,
      "rewards/margins": 0.29973143339157104,
      "rewards/rejected": 0.006640625186264515,
      "step": 835
    },
    {
      "epoch": 0.21627188465499486,
      "grad_norm": 524.0,
      "learning_rate": 3.9186405767250257e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.003124952316284,
      "logps/chosen": -290.3999938964844,
      "logps/rejected": -214.39999389648438,
      "loss": 0.6594,
      "rewards/accuracies": 0.5119048357009888,
      "rewards/chosen": 0.15068359673023224,
      "rewards/margins": 0.13559570908546448,
      "rewards/rejected": 0.01513671875,
      "step": 840
    },
    {
      "epoch": 0.21755921730175076,
      "grad_norm": 211.0,
      "learning_rate": 3.912203913491246e-07,
      "logits/chosen": -3.268749952316284,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -250.8000030517578,
      "logps/rejected": -232.0,
      "loss": 0.6039,
      "rewards/accuracies": 0.693928599357605,
      "rewards/chosen": -0.027740478515625,
      "rewards/margins": 0.2392578125,
      "rewards/rejected": -0.2671875059604645,
      "step": 845
    },
    {
      "epoch": 0.2188465499485067,
      "grad_norm": 193.0,
      "learning_rate": 3.905767250257466e-07,
      "logits/chosen": -3.012500047683716,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -257.79998779296875,
      "logps/rejected": -228.0,
      "loss": 0.6203,
      "rewards/accuracies": 0.5807143449783325,
      "rewards/chosen": 0.25273436307907104,
      "rewards/margins": 0.19359131157398224,
      "rewards/rejected": 0.05905761569738388,
      "step": 850
    },
    {
      "epoch": 0.22013388259526262,
      "grad_norm": 241.0,
      "learning_rate": 3.899330587023687e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -293.0,
      "logps/rejected": -241.0,
      "loss": 0.6,
      "rewards/accuracies": 0.6499542593955994,
      "rewards/chosen": 0.34199219942092896,
      "rewards/margins": 0.2542968690395355,
      "rewards/rejected": 0.08784179389476776,
      "step": 855
    },
    {
      "epoch": 0.22142121524201855,
      "grad_norm": 244.0,
      "learning_rate": 3.8928939237899074e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.234375,
      "logps/chosen": -315.3999938964844,
      "logps/rejected": -248.60000610351562,
      "loss": 0.6195,
      "rewards/accuracies": 0.627500057220459,
      "rewards/chosen": 0.24147948622703552,
      "rewards/margins": 0.2347412109375,
      "rewards/rejected": 0.005963134579360485,
      "step": 860
    },
    {
      "epoch": 0.22270854788877445,
      "grad_norm": 244.0,
      "learning_rate": 3.886457260556127e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -234.60000610351562,
      "logps/rejected": -194.8000030517578,
      "loss": 0.6164,
      "rewards/accuracies": 0.7075091600418091,
      "rewards/chosen": 0.19398193061351776,
      "rewards/margins": 0.21674804389476776,
      "rewards/rejected": -0.02275390550494194,
      "step": 865
    },
    {
      "epoch": 0.22399588053553038,
      "grad_norm": 230.0,
      "learning_rate": 3.8800205973223477e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -259.0,
      "logps/rejected": -315.0,
      "loss": 0.6609,
      "rewards/accuracies": 0.59333336353302,
      "rewards/chosen": 0.17882080376148224,
      "rewards/margins": 0.12639160454273224,
      "rewards/rejected": 0.052734375,
      "step": 870
    },
    {
      "epoch": 0.2252832131822863,
      "grad_norm": 198.0,
      "learning_rate": 3.8735839340885686e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -268.0,
      "logps/rejected": -274.3999938964844,
      "loss": 0.5766,
      "rewards/accuracies": 0.6727380752563477,
      "rewards/chosen": 0.10729370266199112,
      "rewards/margins": 0.3290039002895355,
      "rewards/rejected": -0.22148437798023224,
      "step": 875
    },
    {
      "epoch": 0.22657054582904224,
      "grad_norm": 181.0,
      "learning_rate": 3.8671472708547885e-07,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -263.8999938964844,
      "logps/rejected": -197.39999389648438,
      "loss": 0.6281,
      "rewards/accuracies": 0.6116605997085571,
      "rewards/chosen": 0.22440186142921448,
      "rewards/margins": 0.20332030951976776,
      "rewards/rejected": 0.02143554762005806,
      "step": 880
    },
    {
      "epoch": 0.22785787847579814,
      "grad_norm": 262.0,
      "learning_rate": 3.860710607621009e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.0718750953674316,
      "logps/chosen": -317.20001220703125,
      "logps/rejected": -291.3999938964844,
      "loss": 0.6461,
      "rewards/accuracies": 0.6652381420135498,
      "rewards/chosen": 0.16865234076976776,
      "rewards/margins": 0.21967773139476776,
      "rewards/rejected": -0.05103149265050888,
      "step": 885
    },
    {
      "epoch": 0.22914521112255407,
      "grad_norm": 346.0,
      "learning_rate": 3.8542739443872294e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -237.3000030517578,
      "loss": 0.6234,
      "rewards/accuracies": 0.648888885974884,
      "rewards/chosen": 0.11140136420726776,
      "rewards/margins": 0.2177734375,
      "rewards/rejected": -0.10659179836511612,
      "step": 890
    },
    {
      "epoch": 0.23043254376931,
      "grad_norm": 214.0,
      "learning_rate": 3.84783728115345e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -236.60000610351562,
      "logps/rejected": -223.39999389648438,
      "loss": 0.6188,
      "rewards/accuracies": 0.6306593418121338,
      "rewards/chosen": 0.12004394829273224,
      "rewards/margins": 0.20560303330421448,
      "rewards/rejected": -0.08583984524011612,
      "step": 895
    },
    {
      "epoch": 0.23171987641606592,
      "grad_norm": 184.0,
      "learning_rate": 3.84140061791967e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -308.79998779296875,
      "logps/rejected": -234.60000610351562,
      "loss": 0.5906,
      "rewards/accuracies": 0.6491667032241821,
      "rewards/chosen": 0.14517822861671448,
      "rewards/margins": 0.2939453125,
      "rewards/rejected": -0.14863280951976776,
      "step": 900
    },
    {
      "epoch": 0.23300720906282182,
      "grad_norm": 340.0,
      "learning_rate": 3.8349639546858907e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.3062500953674316,
      "logps/chosen": -295.79998779296875,
      "logps/rejected": -219.6999969482422,
      "loss": 0.6375,
      "rewards/accuracies": 0.5182143449783325,
      "rewards/chosen": 0.04722900316119194,
      "rewards/margins": 0.20322266221046448,
      "rewards/rejected": -0.15668945014476776,
      "step": 905
    },
    {
      "epoch": 0.23429454170957775,
      "grad_norm": 358.0,
      "learning_rate": 3.828527291452111e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -309.3999938964844,
      "logps/rejected": -304.3999938964844,
      "loss": 0.6297,
      "rewards/accuracies": 0.6021367311477661,
      "rewards/chosen": 0.3626953065395355,
      "rewards/margins": 0.22988280653953552,
      "rewards/rejected": 0.132568359375,
      "step": 910
    },
    {
      "epoch": 0.23558187435633368,
      "grad_norm": 222.0,
      "learning_rate": 3.8220906282183315e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.890625,
      "logps/chosen": -217.3000030517578,
      "logps/rejected": -261.79998779296875,
      "loss": 0.6133,
      "rewards/accuracies": 0.595180094242096,
      "rewards/chosen": 0.17900390923023224,
      "rewards/margins": 0.18828125298023224,
      "rewards/rejected": -0.009540271945297718,
      "step": 915
    },
    {
      "epoch": 0.2368692070030896,
      "grad_norm": 227.0,
      "learning_rate": 3.815653964984552e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -200.1999969482422,
      "logps/rejected": -157.10000610351562,
      "loss": 0.6867,
      "rewards/accuracies": 0.5120996236801147,
      "rewards/chosen": 0.093505859375,
      "rewards/margins": 0.05888671800494194,
      "rewards/rejected": 0.03427734225988388,
      "step": 920
    },
    {
      "epoch": 0.2381565396498455,
      "grad_norm": 198.0,
      "learning_rate": 3.8092173017507724e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -239.8000030517578,
      "loss": 0.6492,
      "rewards/accuracies": 0.5144230723381042,
      "rewards/chosen": 0.20595702528953552,
      "rewards/margins": 0.16835936903953552,
      "rewards/rejected": 0.03740234300494194,
      "step": 925
    },
    {
      "epoch": 0.23944387229660144,
      "grad_norm": 237.0,
      "learning_rate": 3.8027806385169923e-07,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.3531250953674316,
      "logps/chosen": -246.39999389648438,
      "logps/rejected": -226.3000030517578,
      "loss": 0.6234,
      "rewards/accuracies": 0.5935714244842529,
      "rewards/chosen": 0.11787109076976776,
      "rewards/margins": 0.21562500298023224,
      "rewards/rejected": -0.09775390475988388,
      "step": 930
    },
    {
      "epoch": 0.24073120494335737,
      "grad_norm": 330.0,
      "learning_rate": 3.796343975283213e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -252.1999969482422,
      "logps/rejected": -246.60000610351562,
      "loss": 0.6445,
      "rewards/accuracies": 0.5973449349403381,
      "rewards/chosen": 0.15224608778953552,
      "rewards/margins": 0.189453125,
      "rewards/rejected": -0.03701172024011612,
      "step": 935
    },
    {
      "epoch": 0.2420185375901133,
      "grad_norm": 548.0,
      "learning_rate": 3.7899073120494337e-07,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -3.0062499046325684,
      "logps/chosen": -234.1999969482422,
      "logps/rejected": -233.8000030517578,
      "loss": 0.6438,
      "rewards/accuracies": 0.6329060196876526,
      "rewards/chosen": 0.22153320908546448,
      "rewards/margins": 0.18613281846046448,
      "rewards/rejected": 0.03557128831744194,
      "step": 940
    },
    {
      "epoch": 0.2433058702368692,
      "grad_norm": 224.0,
      "learning_rate": 3.7834706488156536e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -265.20001220703125,
      "logps/rejected": -219.5,
      "loss": 0.6211,
      "rewards/accuracies": 0.6052814722061157,
      "rewards/chosen": 0.32246094942092896,
      "rewards/margins": 0.21044921875,
      "rewards/rejected": 0.11220703274011612,
      "step": 945
    },
    {
      "epoch": 0.24459320288362513,
      "grad_norm": 248.0,
      "learning_rate": 3.777033985581874e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -230.8000030517578,
      "logps/rejected": -232.39999389648438,
      "loss": 0.6516,
      "rewards/accuracies": 0.5884848833084106,
      "rewards/chosen": 0.119384765625,
      "rewards/margins": 0.124267578125,
      "rewards/rejected": -0.0049804686568677425,
      "step": 950
    },
    {
      "epoch": 0.24588053553038106,
      "grad_norm": 199.0,
      "learning_rate": 3.770597322348095e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.0687499046325684,
      "logps/chosen": -258.20001220703125,
      "logps/rejected": -241.39999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.596666693687439,
      "rewards/chosen": 0.07829590141773224,
      "rewards/margins": 0.16054686903953552,
      "rewards/rejected": -0.08261718600988388,
      "step": 955
    },
    {
      "epoch": 0.24716786817713698,
      "grad_norm": 256.0,
      "learning_rate": 3.764160659114315e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -356.0,
      "logps/rejected": -296.20001220703125,
      "loss": 0.5672,
      "rewards/accuracies": 0.6533333659172058,
      "rewards/chosen": 0.24033203721046448,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -0.12714843451976776,
      "step": 960
    },
    {
      "epoch": 0.24845520082389289,
      "grad_norm": 177.0,
      "learning_rate": 3.7577239958805353e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.081249952316284,
      "logps/chosen": -250.39999389648438,
      "logps/rejected": -202.89999389648438,
      "loss": 0.5875,
      "rewards/accuracies": 0.6928571462631226,
      "rewards/chosen": 0.27250975370407104,
      "rewards/margins": 0.31328123807907104,
      "rewards/rejected": -0.04104004055261612,
      "step": 965
    },
    {
      "epoch": 0.24974253347064881,
      "grad_norm": 167.0,
      "learning_rate": 3.7512873326467557e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.375,
      "logps/chosen": -326.3999938964844,
      "logps/rejected": -287.0,
      "loss": 0.6086,
      "rewards/accuracies": 0.5985714197158813,
      "rewards/chosen": 0.17080077528953552,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.10649414360523224,
      "step": 970
    },
    {
      "epoch": 0.25102986611740474,
      "grad_norm": 232.0,
      "learning_rate": 3.744850669412976e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.268749952316284,
      "logps/chosen": -317.20001220703125,
      "logps/rejected": -243.1999969482422,
      "loss": 0.5961,
      "rewards/accuracies": 0.6094871759414673,
      "rewards/chosen": 0.24345703423023224,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.014892578125,
      "step": 975
    },
    {
      "epoch": 0.25231719876416064,
      "grad_norm": 272.0,
      "learning_rate": 3.7384140061791965e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -307.3999938964844,
      "logps/rejected": -255.39999389648438,
      "loss": 0.6227,
      "rewards/accuracies": 0.6233333349227905,
      "rewards/chosen": 0.2549804747104645,
      "rewards/margins": 0.25146484375,
      "rewards/rejected": 0.003662109375,
      "step": 980
    },
    {
      "epoch": 0.2536045314109166,
      "grad_norm": 260.0,
      "learning_rate": 3.731977342945417e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.2750000953674316,
      "logps/chosen": -307.6000061035156,
      "logps/rejected": -279.6000061035156,
      "loss": 0.6531,
      "rewards/accuracies": 0.5655769109725952,
      "rewards/chosen": 0.091552734375,
      "rewards/margins": 0.16499023139476776,
      "rewards/rejected": -0.0732421875,
      "step": 985
    },
    {
      "epoch": 0.2548918640576725,
      "grad_norm": 203.0,
      "learning_rate": 3.7255406797116374e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -308.20001220703125,
      "logps/rejected": -271.3999938964844,
      "loss": 0.5563,
      "rewards/accuracies": 0.6696428656578064,
      "rewards/chosen": 0.17412109673023224,
      "rewards/margins": 0.4009765684604645,
      "rewards/rejected": -0.22636719048023224,
      "step": 990
    },
    {
      "epoch": 0.2561791967044284,
      "grad_norm": 226.0,
      "learning_rate": 3.7191040164778573e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.265625,
      "logps/chosen": -314.20001220703125,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6344,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.05976562574505806,
      "rewards/margins": 0.16796875,
      "rewards/rejected": -0.10806427150964737,
      "step": 995
    },
    {
      "epoch": 0.25746652935118436,
      "grad_norm": 184.0,
      "learning_rate": 3.712667353244078e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -260.6000061035156,
      "logps/rejected": -252.8000030517578,
      "loss": 0.6203,
      "rewards/accuracies": 0.5801648497581482,
      "rewards/chosen": 0.13925781846046448,
      "rewards/margins": 0.23007813096046448,
      "rewards/rejected": -0.09072265774011612,
      "step": 1000
    },
    {
      "epoch": 0.25875386199794026,
      "grad_norm": 211.0,
      "learning_rate": 3.7062306900102987e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -277.6000061035156,
      "logps/rejected": -234.6999969482422,
      "loss": 0.6578,
      "rewards/accuracies": 0.5856410264968872,
      "rewards/chosen": 0.15117187798023224,
      "rewards/margins": 0.14946289360523224,
      "rewards/rejected": 0.0019348144996911287,
      "step": 1005
    },
    {
      "epoch": 0.2600411946446962,
      "grad_norm": 194.0,
      "learning_rate": 3.6997940267765186e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -343.20001220703125,
      "logps/rejected": -272.20001220703125,
      "loss": 0.5406,
      "rewards/accuracies": 0.7164286375045776,
      "rewards/chosen": 0.3980468809604645,
      "rewards/margins": 0.4273437559604645,
      "rewards/rejected": -0.02869873121380806,
      "step": 1010
    },
    {
      "epoch": 0.2613285272914521,
      "grad_norm": 208.0,
      "learning_rate": 3.6933573635427395e-07,
      "logits/chosen": -3.0875000953674316,
      "logits/rejected": -3.106250047683716,
      "logps/chosen": -330.0,
      "logps/rejected": -262.20001220703125,
      "loss": 0.5938,
      "rewards/accuracies": 0.6427813768386841,
      "rewards/chosen": 0.2945312559604645,
      "rewards/margins": 0.25190430879592896,
      "rewards/rejected": 0.04250488430261612,
      "step": 1015
    },
    {
      "epoch": 0.262615859938208,
      "grad_norm": 280.0,
      "learning_rate": 3.68692070030896e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -301.0,
      "loss": 0.6312,
      "rewards/accuracies": 0.5983333587646484,
      "rewards/chosen": 0.16586914658546448,
      "rewards/margins": 0.23906250298023224,
      "rewards/rejected": -0.07288207858800888,
      "step": 1020
    },
    {
      "epoch": 0.263903192584964,
      "grad_norm": 428.0,
      "learning_rate": 3.68048403707518e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -249.0,
      "logps/rejected": -192.60000610351562,
      "loss": 0.6031,
      "rewards/accuracies": 0.6492856740951538,
      "rewards/chosen": 0.21933594346046448,
      "rewards/margins": 0.23300781846046448,
      "rewards/rejected": -0.01376953162252903,
      "step": 1025
    },
    {
      "epoch": 0.2651905252317199,
      "grad_norm": 235.0,
      "learning_rate": 3.6740473738414003e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -284.1000061035156,
      "logps/rejected": -241.60000610351562,
      "loss": 0.6234,
      "rewards/accuracies": 0.6730555295944214,
      "rewards/chosen": 0.23066405951976776,
      "rewards/margins": 0.19960936903953552,
      "rewards/rejected": 0.03056640550494194,
      "step": 1030
    },
    {
      "epoch": 0.2664778578784758,
      "grad_norm": 260.0,
      "learning_rate": 3.6676107106076207e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -2.996875047683716,
      "logps/chosen": -216.0,
      "logps/rejected": -207.60000610351562,
      "loss": 0.6281,
      "rewards/accuracies": 0.5800000429153442,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 0.20087890326976776,
      "rewards/rejected": 0.04633789137005806,
      "step": 1035
    },
    {
      "epoch": 0.26776519052523173,
      "grad_norm": 368.0,
      "learning_rate": 3.661174047373841e-07,
      "logits/chosen": -3.268749952316284,
      "logits/rejected": -3.21875,
      "logps/chosen": -285.6000061035156,
      "logps/rejected": -271.6000061035156,
      "loss": 0.6172,
      "rewards/accuracies": 0.5994047522544861,
      "rewards/chosen": 0.21123047173023224,
      "rewards/margins": 0.261962890625,
      "rewards/rejected": -0.05120849609375,
      "step": 1040
    },
    {
      "epoch": 0.26905252317198763,
      "grad_norm": 322.0,
      "learning_rate": 3.6547373841400616e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.109375,
      "logps/chosen": -344.6000061035156,
      "logps/rejected": -270.20001220703125,
      "loss": 0.6336,
      "rewards/accuracies": 0.6226282119750977,
      "rewards/chosen": 0.20301513373851776,
      "rewards/margins": 0.17929688096046448,
      "rewards/rejected": 0.02351074293255806,
      "step": 1045
    },
    {
      "epoch": 0.2703398558187436,
      "grad_norm": 198.0,
      "learning_rate": 3.648300720906282e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -243.1999969482422,
      "logps/rejected": -225.60000610351562,
      "loss": 0.6152,
      "rewards/accuracies": 0.619359016418457,
      "rewards/chosen": 0.19003906846046448,
      "rewards/margins": 0.2518554627895355,
      "rewards/rejected": -0.061767578125,
      "step": 1050
    },
    {
      "epoch": 0.2716271884654995,
      "grad_norm": 239.0,
      "learning_rate": 3.6418640576725024e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.234375,
      "logps/chosen": -263.0,
      "logps/rejected": -284.3999938964844,
      "loss": 0.6547,
      "rewards/accuracies": 0.540238082408905,
      "rewards/chosen": 0.19169922173023224,
      "rewards/margins": 0.13984374701976776,
      "rewards/rejected": 0.0521240234375,
      "step": 1055
    },
    {
      "epoch": 0.2729145211122554,
      "grad_norm": 346.0,
      "learning_rate": 3.635427394438723e-07,
      "logits/chosen": -3.315624952316284,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -241.39999389648438,
      "logps/rejected": -230.89999389648438,
      "loss": 0.6484,
      "rewards/accuracies": 0.5316072702407837,
      "rewards/chosen": 0.18808594346046448,
      "rewards/margins": 0.16899414360523224,
      "rewards/rejected": 0.01894531212747097,
      "step": 1060
    },
    {
      "epoch": 0.27420185375901135,
      "grad_norm": 270.0,
      "learning_rate": 3.6289907312049433e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -298.0,
      "logps/rejected": -281.20001220703125,
      "loss": 0.6516,
      "rewards/accuracies": 0.5714285969734192,
      "rewards/chosen": 0.170654296875,
      "rewards/margins": 0.15087890625,
      "rewards/rejected": 0.01952514611184597,
      "step": 1065
    },
    {
      "epoch": 0.27548918640576725,
      "grad_norm": 225.0,
      "learning_rate": 3.6225540679711637e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.03125,
      "logps/chosen": -197.10000610351562,
      "logps/rejected": -184.5,
      "loss": 0.6238,
      "rewards/accuracies": 0.71833336353302,
      "rewards/chosen": 0.14929810166358948,
      "rewards/margins": 0.21770019829273224,
      "rewards/rejected": -0.06884765625,
      "step": 1070
    },
    {
      "epoch": 0.27677651905252315,
      "grad_norm": 258.0,
      "learning_rate": 3.6161174047373836e-07,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -331.20001220703125,
      "logps/rejected": -293.79998779296875,
      "loss": 0.643,
      "rewards/accuracies": 0.5926281809806824,
      "rewards/chosen": 0.08506317436695099,
      "rewards/margins": 0.15507812798023224,
      "rewards/rejected": -0.07016601413488388,
      "step": 1075
    },
    {
      "epoch": 0.2780638516992791,
      "grad_norm": 201.0,
      "learning_rate": 3.6096807415036046e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -257.3999938964844,
      "logps/rejected": -234.8000030517578,
      "loss": 0.5805,
      "rewards/accuracies": 0.6484615206718445,
      "rewards/chosen": 0.22382812201976776,
      "rewards/margins": 0.30097657442092896,
      "rewards/rejected": -0.077239990234375,
      "step": 1080
    },
    {
      "epoch": 0.279351184346035,
      "grad_norm": 194.0,
      "learning_rate": 3.603244078269825e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.299999952316284,
      "logps/chosen": -364.3999938964844,
      "logps/rejected": -258.20001220703125,
      "loss": 0.5898,
      "rewards/accuracies": 0.6723076701164246,
      "rewards/chosen": 0.23774413764476776,
      "rewards/margins": 0.2939453125,
      "rewards/rejected": -0.05644531175494194,
      "step": 1085
    },
    {
      "epoch": 0.2806385169927909,
      "grad_norm": 192.0,
      "learning_rate": 3.596807415036045e-07,
      "logits/chosen": -3.0062499046325684,
      "logits/rejected": -3.046875,
      "logps/chosen": -239.89999389648438,
      "logps/rejected": -215.1999969482422,
      "loss": 0.6172,
      "rewards/accuracies": 0.626352846622467,
      "rewards/chosen": 0.3003906309604645,
      "rewards/margins": 0.22601929306983948,
      "rewards/rejected": 0.07412109524011612,
      "step": 1090
    },
    {
      "epoch": 0.28192584963954687,
      "grad_norm": 784.0,
      "learning_rate": 3.590370751802266e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -230.3000030517578,
      "loss": 0.6813,
      "rewards/accuracies": 0.5636471509933472,
      "rewards/chosen": 0.16307373344898224,
      "rewards/margins": 0.07685546576976776,
      "rewards/rejected": 0.08634205162525177,
      "step": 1095
    },
    {
      "epoch": 0.28321318228630277,
      "grad_norm": 217.0,
      "learning_rate": 3.583934088568486e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -311.3999938964844,
      "logps/rejected": -260.20001220703125,
      "loss": 0.5953,
      "rewards/accuracies": 0.6100000143051147,
      "rewards/chosen": 0.19929809868335724,
      "rewards/margins": 0.2587890625,
      "rewards/rejected": -0.05930175632238388,
      "step": 1100
    },
    {
      "epoch": 0.2845005149330587,
      "grad_norm": 206.0,
      "learning_rate": 3.577497425334706e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.09375,
      "logps/chosen": -248.60000610351562,
      "logps/rejected": -228.1999969482422,
      "loss": 0.6484,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.14794310927391052,
      "rewards/margins": 0.15324707329273224,
      "rewards/rejected": -0.0054687499068677425,
      "step": 1105
    },
    {
      "epoch": 0.2857878475798146,
      "grad_norm": 253.0,
      "learning_rate": 3.571060762100927e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.3187499046325684,
      "logps/chosen": -278.6000061035156,
      "logps/rejected": -208.39999389648438,
      "loss": 0.6523,
      "rewards/accuracies": 0.6406060457229614,
      "rewards/chosen": 0.09619140625,
      "rewards/margins": 0.158203125,
      "rewards/rejected": -0.06196289137005806,
      "step": 1110
    },
    {
      "epoch": 0.2870751802265705,
      "grad_norm": 1120.0,
      "learning_rate": 3.564624098867147e-07,
      "logits/chosen": -2.8968749046325684,
      "logits/rejected": -2.934375047683716,
      "logps/chosen": -247.0,
      "logps/rejected": -231.60000610351562,
      "loss": 0.6344,
      "rewards/accuracies": 0.5846079587936401,
      "rewards/chosen": 0.15800781548023224,
      "rewards/margins": 0.17441406846046448,
      "rewards/rejected": -0.01616210862994194,
      "step": 1115
    },
    {
      "epoch": 0.2883625128733265,
      "grad_norm": 330.0,
      "learning_rate": 3.5581874356333674e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -239.8000030517578,
      "logps/rejected": -231.60000610351562,
      "loss": 0.5938,
      "rewards/accuracies": 0.6858333945274353,
      "rewards/chosen": 0.21481934189796448,
      "rewards/margins": 0.26904296875,
      "rewards/rejected": -0.05469970777630806,
      "step": 1120
    },
    {
      "epoch": 0.2896498455200824,
      "grad_norm": 232.0,
      "learning_rate": 3.551750772399588e-07,
      "logits/chosen": -3.112499952316284,
      "logits/rejected": -3.171875,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -235.3000030517578,
      "loss": 0.6109,
      "rewards/accuracies": 0.6938950419425964,
      "rewards/chosen": 0.23749999701976776,
      "rewards/margins": 0.30424803495407104,
      "rewards/rejected": -0.06669922173023224,
      "step": 1125
    },
    {
      "epoch": 0.2909371781668383,
      "grad_norm": 235.0,
      "learning_rate": 3.5453141091658083e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -259.0,
      "logps/rejected": -207.35000610351562,
      "loss": 0.668,
      "rewards/accuracies": 0.566111147403717,
      "rewards/chosen": 0.02353515662252903,
      "rewards/margins": 0.11766357719898224,
      "rewards/rejected": -0.09384765475988388,
      "step": 1130
    },
    {
      "epoch": 0.29222451081359424,
      "grad_norm": 190.0,
      "learning_rate": 3.5388774459320287e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.0718750953674316,
      "logps/chosen": -263.6000061035156,
      "logps/rejected": -274.79998779296875,
      "loss": 0.6133,
      "rewards/accuracies": 0.6322436332702637,
      "rewards/chosen": 0.23886719346046448,
      "rewards/margins": 0.24082031846046448,
      "rewards/rejected": -0.001953125,
      "step": 1135
    },
    {
      "epoch": 0.29351184346035014,
      "grad_norm": 272.0,
      "learning_rate": 3.5324407826982486e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -267.3999938964844,
      "logps/rejected": -211.8000030517578,
      "loss": 0.5945,
      "rewards/accuracies": 0.6463203430175781,
      "rewards/chosen": 0.24741211533546448,
      "rewards/margins": 0.3089141845703125,
      "rewards/rejected": -0.061767578125,
      "step": 1140
    },
    {
      "epoch": 0.2947991761071061,
      "grad_norm": 296.0,
      "learning_rate": 3.5260041194644696e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -325.0,
      "logps/rejected": -296.6000061035156,
      "loss": 0.6227,
      "rewards/accuracies": 0.6266666650772095,
      "rewards/chosen": 0.24160155653953552,
      "rewards/margins": 0.24211426079273224,
      "rewards/rejected": -0.0003173828008584678,
      "step": 1145
    },
    {
      "epoch": 0.296086508753862,
      "grad_norm": 185.0,
      "learning_rate": 3.51956745623069e-07,
      "logits/chosen": -2.9906249046325684,
      "logits/rejected": -2.971874952316284,
      "logps/chosen": -365.0,
      "logps/rejected": -308.79998779296875,
      "loss": 0.5977,
      "rewards/accuracies": 0.6222726702690125,
      "rewards/chosen": 0.16899414360523224,
      "rewards/margins": 0.32402342557907104,
      "rewards/rejected": -0.15522460639476776,
      "step": 1150
    },
    {
      "epoch": 0.2973738414006179,
      "grad_norm": 189.0,
      "learning_rate": 3.51313079299691e-07,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -293.79998779296875,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6062,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": 0.05903320387005806,
      "rewards/margins": 0.25664061307907104,
      "rewards/rejected": -0.1976318359375,
      "step": 1155
    },
    {
      "epoch": 0.29866117404737386,
      "grad_norm": 176.0,
      "learning_rate": 3.506694129763131e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -296.3999938964844,
      "logps/rejected": -233.89999389648438,
      "loss": 0.5844,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.10469360649585724,
      "rewards/margins": 0.3199218809604645,
      "rewards/rejected": -0.21596679091453552,
      "step": 1160
    },
    {
      "epoch": 0.29994850669412976,
      "grad_norm": 211.0,
      "learning_rate": 3.5002574665293513e-07,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -333.6000061035156,
      "logps/rejected": -250.39999389648438,
      "loss": 0.6711,
      "rewards/accuracies": 0.5957576036453247,
      "rewards/chosen": 0.21572265028953552,
      "rewards/margins": 0.16298827528953552,
      "rewards/rejected": 0.05263672024011612,
      "step": 1165
    },
    {
      "epoch": 0.30123583934088566,
      "grad_norm": 284.0,
      "learning_rate": 3.493820803295571e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -333.6000061035156,
      "logps/rejected": -330.79998779296875,
      "loss": 0.668,
      "rewards/accuracies": 0.5225000381469727,
      "rewards/chosen": 0.22402343153953552,
      "rewards/margins": 0.14995117485523224,
      "rewards/rejected": 0.07412109524011612,
      "step": 1170
    },
    {
      "epoch": 0.3025231719876416,
      "grad_norm": 264.0,
      "learning_rate": 3.487384140061792e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -290.79998779296875,
      "logps/rejected": -269.0,
      "loss": 0.6633,
      "rewards/accuracies": 0.6363636255264282,
      "rewards/chosen": 0.22736816108226776,
      "rewards/margins": 0.15999755263328552,
      "rewards/rejected": 0.06748046725988388,
      "step": 1175
    },
    {
      "epoch": 0.3038105046343975,
      "grad_norm": 255.0,
      "learning_rate": 3.480947476828012e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.3687500953674316,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -225.1999969482422,
      "loss": 0.6195,
      "rewards/accuracies": 0.6143590211868286,
      "rewards/chosen": 0.08510742336511612,
      "rewards/margins": 0.24924317002296448,
      "rewards/rejected": -0.16464844346046448,
      "step": 1180
    },
    {
      "epoch": 0.30509783728115347,
      "grad_norm": 207.0,
      "learning_rate": 3.4745108135942325e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -348.3999938964844,
      "logps/rejected": -262.6000061035156,
      "loss": 0.5961,
      "rewards/accuracies": 0.7238461375236511,
      "rewards/chosen": 0.28925782442092896,
      "rewards/margins": 0.27812498807907104,
      "rewards/rejected": 0.011523437686264515,
      "step": 1185
    },
    {
      "epoch": 0.3063851699279094,
      "grad_norm": 256.0,
      "learning_rate": 3.4680741503604534e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -320.79998779296875,
      "logps/rejected": -305.6000061035156,
      "loss": 0.625,
      "rewards/accuracies": 0.552124559879303,
      "rewards/chosen": 0.15312500298023224,
      "rewards/margins": 0.21389159560203552,
      "rewards/rejected": -0.060302734375,
      "step": 1190
    },
    {
      "epoch": 0.3076725025746653,
      "grad_norm": 324.0,
      "learning_rate": 3.4616374871266733e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.768749952316284,
      "logps/chosen": -241.60000610351562,
      "logps/rejected": -222.10000610351562,
      "loss": 0.6156,
      "rewards/accuracies": 0.6727814674377441,
      "rewards/chosen": 0.28691405057907104,
      "rewards/margins": 0.26658934354782104,
      "rewards/rejected": 0.01994628831744194,
      "step": 1195
    },
    {
      "epoch": 0.30895983522142123,
      "grad_norm": 237.0,
      "learning_rate": 3.455200823892894e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.081249952316284,
      "logps/chosen": -311.0,
      "logps/rejected": -239.1999969482422,
      "loss": 0.618,
      "rewards/accuracies": 0.6605555415153503,
      "rewards/chosen": 0.19462890923023224,
      "rewards/margins": 0.263671875,
      "rewards/rejected": -0.06843261420726776,
      "step": 1200
    },
    {
      "epoch": 0.31024716786817713,
      "grad_norm": 237.0,
      "learning_rate": 3.448764160659114e-07,
      "logits/chosen": -3.3375000953674316,
      "logits/rejected": -3.3343749046325684,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -299.20001220703125,
      "loss": 0.6141,
      "rewards/accuracies": 0.6073626279830933,
      "rewards/chosen": 0.12497558444738388,
      "rewards/margins": 0.23886719346046448,
      "rewards/rejected": -0.11328125,
      "step": 1205
    },
    {
      "epoch": 0.31153450051493303,
      "grad_norm": 198.0,
      "learning_rate": 3.4423274974253346e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.3343749046325684,
      "logps/chosen": -339.20001220703125,
      "logps/rejected": -268.3999938964844,
      "loss": 0.5672,
      "rewards/accuracies": 0.6886364221572876,
      "rewards/chosen": 0.3511718809604645,
      "rewards/margins": 0.3814453184604645,
      "rewards/rejected": -0.03056640550494194,
      "step": 1210
    },
    {
      "epoch": 0.312821833161689,
      "grad_norm": 211.0,
      "learning_rate": 3.435890834191555e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.25,
      "logps/chosen": -282.3999938964844,
      "logps/rejected": -234.0,
      "loss": 0.6125,
      "rewards/accuracies": 0.6584523916244507,
      "rewards/chosen": 0.11430664360523224,
      "rewards/margins": 0.23056641221046448,
      "rewards/rejected": -0.11635742336511612,
      "step": 1215
    },
    {
      "epoch": 0.3141091658084449,
      "grad_norm": 218.0,
      "learning_rate": 3.429454170957775e-07,
      "logits/chosen": -3.284374952316284,
      "logits/rejected": -3.34375,
      "logps/chosen": -311.0,
      "logps/rejected": -289.79998779296875,
      "loss": 0.5898,
      "rewards/accuracies": 0.6785714030265808,
      "rewards/chosen": 0.03131103515625,
      "rewards/margins": 0.2906250059604645,
      "rewards/rejected": -0.2591796815395355,
      "step": 1220
    },
    {
      "epoch": 0.31539649845520085,
      "grad_norm": 237.0,
      "learning_rate": 3.423017507723996e-07,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.296875,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -259.6000061035156,
      "loss": 0.5875,
      "rewards/accuracies": 0.6691666841506958,
      "rewards/chosen": 0.20517578721046448,
      "rewards/margins": 0.3031249940395355,
      "rewards/rejected": -0.09869384765625,
      "step": 1225
    },
    {
      "epoch": 0.31668383110195675,
      "grad_norm": 205.0,
      "learning_rate": 3.4165808444902163e-07,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -301.79998779296875,
      "logps/rejected": -273.0,
      "loss": 0.5695,
      "rewards/accuracies": 0.6292856931686401,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 0.36054688692092896,
      "rewards/rejected": -0.13901367783546448,
      "step": 1230
    },
    {
      "epoch": 0.31797116374871265,
      "grad_norm": 149.0,
      "learning_rate": 3.410144181256436e-07,
      "logits/chosen": -2.9312500953674316,
      "logits/rejected": -2.875,
      "logps/chosen": -293.3999938964844,
      "logps/rejected": -230.39999389648438,
      "loss": 0.6742,
      "rewards/accuracies": 0.5569804906845093,
      "rewards/chosen": 0.12607422471046448,
      "rewards/margins": 0.14244385063648224,
      "rewards/rejected": -0.01659545861184597,
      "step": 1235
    },
    {
      "epoch": 0.3192584963954686,
      "grad_norm": 322.0,
      "learning_rate": 3.403707518022657e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.3187499046325684,
      "logps/chosen": -267.79998779296875,
      "logps/rejected": -229.89999389648438,
      "loss": 0.6391,
      "rewards/accuracies": 0.6189102530479431,
      "rewards/chosen": 0.08658142387866974,
      "rewards/margins": 0.20356445014476776,
      "rewards/rejected": -0.11689452826976776,
      "step": 1240
    },
    {
      "epoch": 0.3205458290422245,
      "grad_norm": 458.0,
      "learning_rate": 3.3972708547888776e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -257.6000061035156,
      "logps/rejected": -205.10000610351562,
      "loss": 0.5914,
      "rewards/accuracies": 0.6139286160469055,
      "rewards/chosen": 0.20961913466453552,
      "rewards/margins": 0.28496092557907104,
      "rewards/rejected": -0.07561035454273224,
      "step": 1245
    },
    {
      "epoch": 0.3218331616889804,
      "grad_norm": 268.0,
      "learning_rate": 3.3908341915550975e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -272.0,
      "loss": 0.6523,
      "rewards/accuracies": 0.5894047617912292,
      "rewards/chosen": 0.15791015326976776,
      "rewards/margins": 0.17863769829273224,
      "rewards/rejected": -0.02080078050494194,
      "step": 1250
    },
    {
      "epoch": 0.32312049433573636,
      "grad_norm": 304.0,
      "learning_rate": 3.3843975283213184e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -338.0,
      "loss": 0.5891,
      "rewards/accuracies": 0.7686080932617188,
      "rewards/chosen": 0.21640625596046448,
      "rewards/margins": 0.38554686307907104,
      "rewards/rejected": -0.16914062201976776,
      "step": 1255
    },
    {
      "epoch": 0.32440782698249226,
      "grad_norm": 268.0,
      "learning_rate": 3.3779608650875383e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -264.0,
      "logps/rejected": -207.8000030517578,
      "loss": 0.6539,
      "rewards/accuracies": 0.6118589639663696,
      "rewards/chosen": 0.19853515923023224,
      "rewards/margins": 0.2347412109375,
      "rewards/rejected": -0.03691406175494194,
      "step": 1260
    },
    {
      "epoch": 0.3256951596292482,
      "grad_norm": 212.0,
      "learning_rate": 3.371524201853759e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -222.39999389648438,
      "logps/rejected": -189.8000030517578,
      "loss": 0.5687,
      "rewards/accuracies": 0.7595979571342468,
      "rewards/chosen": 0.26005858182907104,
      "rewards/margins": 0.3451171815395355,
      "rewards/rejected": -0.08432617038488388,
      "step": 1265
    },
    {
      "epoch": 0.3269824922760041,
      "grad_norm": 286.0,
      "learning_rate": 3.3650875386199797e-07,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.265625,
      "logps/chosen": -271.20001220703125,
      "logps/rejected": -245.1999969482422,
      "loss": 0.5961,
      "rewards/accuracies": 0.7177839875221252,
      "rewards/chosen": 0.178466796875,
      "rewards/margins": 0.2751708924770355,
      "rewards/rejected": -0.09672851860523224,
      "step": 1270
    },
    {
      "epoch": 0.32826982492276,
      "grad_norm": 245.0,
      "learning_rate": 3.3586508753861996e-07,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -339.3999938964844,
      "logps/rejected": -287.20001220703125,
      "loss": 0.657,
      "rewards/accuracies": 0.5662912130355835,
      "rewards/chosen": 0.22929687798023224,
      "rewards/margins": 0.1497802734375,
      "rewards/rejected": 0.07905273139476776,
      "step": 1275
    },
    {
      "epoch": 0.329557157569516,
      "grad_norm": 188.0,
      "learning_rate": 3.35221421215242e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -323.79998779296875,
      "logps/rejected": -311.79998779296875,
      "loss": 0.6469,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.142578125,
      "rewards/margins": 0.19263915717601776,
      "rewards/rejected": -0.05043945461511612,
      "step": 1280
    },
    {
      "epoch": 0.3308444902162719,
      "grad_norm": 262.0,
      "learning_rate": 3.34577754891864e-07,
      "logits/chosen": -2.8671875,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -293.79998779296875,
      "logps/rejected": -197.1999969482422,
      "loss": 0.5125,
      "rewards/accuracies": 0.686904788017273,
      "rewards/chosen": 0.3667968809604645,
      "rewards/margins": 0.518359363079071,
      "rewards/rejected": -0.15200194716453552,
      "step": 1285
    },
    {
      "epoch": 0.3321318228630278,
      "grad_norm": 190.0,
      "learning_rate": 3.339340885684861e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -273.0,
      "logps/rejected": -258.6000061035156,
      "loss": 0.5945,
      "rewards/accuracies": 0.6384615302085876,
      "rewards/chosen": 0.14582519233226776,
      "rewards/margins": 0.30878907442092896,
      "rewards/rejected": -0.16279296576976776,
      "step": 1290
    },
    {
      "epoch": 0.33341915550978374,
      "grad_norm": 162.0,
      "learning_rate": 3.3329042224510813e-07,
      "logits/chosen": -3.309375047683716,
      "logits/rejected": -3.3375000953674316,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -291.0,
      "loss": 0.6082,
      "rewards/accuracies": 0.6266666650772095,
      "rewards/chosen": 0.10747070610523224,
      "rewards/margins": 0.2769775390625,
      "rewards/rejected": -0.16943359375,
      "step": 1295
    },
    {
      "epoch": 0.33470648815653964,
      "grad_norm": 298.0,
      "learning_rate": 3.326467559217301e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -356.3999938964844,
      "logps/rejected": -326.3999938964844,
      "loss": 0.5945,
      "rewards/accuracies": 0.6352381706237793,
      "rewards/chosen": 0.24519042670726776,
      "rewards/margins": 0.3011718690395355,
      "rewards/rejected": -0.05577850341796875,
      "step": 1300
    },
    {
      "epoch": 0.3359938208032956,
      "grad_norm": 181.0,
      "learning_rate": 3.320030895983522e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.2718749046325684,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -216.8000030517578,
      "loss": 0.6133,
      "rewards/accuracies": 0.6380769610404968,
      "rewards/chosen": 0.13642577826976776,
      "rewards/margins": 0.23183593153953552,
      "rewards/rejected": -0.0953369140625,
      "step": 1305
    },
    {
      "epoch": 0.3372811534500515,
      "grad_norm": 256.0,
      "learning_rate": 3.3135942327497426e-07,
      "logits/chosen": -2.9593749046325684,
      "logits/rejected": -2.7874999046325684,
      "logps/chosen": -245.8000030517578,
      "logps/rejected": -257.1000061035156,
      "loss": 0.6266,
      "rewards/accuracies": 0.5966270565986633,
      "rewards/chosen": 0.1715087890625,
      "rewards/margins": 0.21936646103858948,
      "rewards/rejected": -0.04749755933880806,
      "step": 1310
    },
    {
      "epoch": 0.3385684860968074,
      "grad_norm": 182.0,
      "learning_rate": 3.3071575695159625e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -3.203125,
      "logps/chosen": -295.79998779296875,
      "logps/rejected": -241.89999389648438,
      "loss": 0.5738,
      "rewards/accuracies": 0.684166669845581,
      "rewards/chosen": 0.15863648056983948,
      "rewards/margins": 0.3543457090854645,
      "rewards/rejected": -0.195556640625,
      "step": 1315
    },
    {
      "epoch": 0.33985581874356335,
      "grad_norm": 254.0,
      "learning_rate": 3.3007209062821835e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.112499952316284,
      "logps/chosen": -263.3999938964844,
      "logps/rejected": -320.79998779296875,
      "loss": 0.6367,
      "rewards/accuracies": 0.6026948094367981,
      "rewards/chosen": 0.23881836235523224,
      "rewards/margins": 0.19990234076976776,
      "rewards/rejected": 0.03867187350988388,
      "step": 1320
    },
    {
      "epoch": 0.34114315139031925,
      "grad_norm": 229.0,
      "learning_rate": 3.2942842430484033e-07,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -319.6000061035156,
      "logps/rejected": -229.8000030517578,
      "loss": 0.5859,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": 0.24477538466453552,
      "rewards/margins": 0.30195313692092896,
      "rewards/rejected": -0.05668945237994194,
      "step": 1325
    },
    {
      "epoch": 0.34243048403707516,
      "grad_norm": 245.0,
      "learning_rate": 3.287847579814624e-07,
      "logits/chosen": -2.7890625,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -274.8999938964844,
      "logps/rejected": -245.8000030517578,
      "loss": 0.5254,
      "rewards/accuracies": 0.7701557278633118,
      "rewards/chosen": 0.3785156309604645,
      "rewards/margins": 0.533007800579071,
      "rewards/rejected": -0.15449218451976776,
      "step": 1330
    },
    {
      "epoch": 0.3437178166838311,
      "grad_norm": 220.0,
      "learning_rate": 3.2814109165808447e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -270.0,
      "logps/rejected": -261.3999938964844,
      "loss": 0.6336,
      "rewards/accuracies": 0.609870195388794,
      "rewards/chosen": 0.174560546875,
      "rewards/margins": 0.23603515326976776,
      "rewards/rejected": -0.06162109225988388,
      "step": 1335
    },
    {
      "epoch": 0.345005149330587,
      "grad_norm": 274.0,
      "learning_rate": 3.2749742533470646e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -270.0,
      "logps/rejected": -245.1999969482422,
      "loss": 0.5961,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": 0.38969725370407104,
      "rewards/margins": 0.302642822265625,
      "rewards/rejected": 0.08749999850988388,
      "step": 1340
    },
    {
      "epoch": 0.34629248197734297,
      "grad_norm": 250.0,
      "learning_rate": 3.268537590113285e-07,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.015625,
      "logps/chosen": -230.0,
      "logps/rejected": -232.1999969482422,
      "loss": 0.6586,
      "rewards/accuracies": 0.6147583723068237,
      "rewards/chosen": 0.18654784560203552,
      "rewards/margins": 0.15439453721046448,
      "rewards/rejected": 0.0318603515625,
      "step": 1345
    },
    {
      "epoch": 0.34757981462409887,
      "grad_norm": 274.0,
      "learning_rate": 3.262100926879506e-07,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -2.9000000953674316,
      "logps/chosen": -299.79998779296875,
      "logps/rejected": -329.20001220703125,
      "loss": 0.6047,
      "rewards/accuracies": 0.5646428465843201,
      "rewards/chosen": 0.11248779296875,
      "rewards/margins": 0.8001953363418579,
      "rewards/rejected": -0.6900390386581421,
      "step": 1350
    },
    {
      "epoch": 0.34886714727085477,
      "grad_norm": 177.0,
      "learning_rate": 3.255664263645726e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.25,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -254.1999969482422,
      "loss": 0.5891,
      "rewards/accuracies": 0.6491667032241821,
      "rewards/chosen": 0.0849609375,
      "rewards/margins": 0.31523436307907104,
      "rewards/rejected": -0.23007813096046448,
      "step": 1355
    },
    {
      "epoch": 0.35015447991761073,
      "grad_norm": 237.0,
      "learning_rate": 3.2492276004119463e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.203125,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -232.1999969482422,
      "loss": 0.6164,
      "rewards/accuracies": 0.6409091353416443,
      "rewards/chosen": 0.09165038913488388,
      "rewards/margins": 0.23095703125,
      "rewards/rejected": -0.13955077528953552,
      "step": 1360
    },
    {
      "epoch": 0.35144181256436663,
      "grad_norm": 215.0,
      "learning_rate": 3.242790937178166e-07,
      "logits/chosen": -3.28125,
      "logits/rejected": -3.3062500953674316,
      "logps/chosen": -285.79998779296875,
      "logps/rejected": -310.3999938964844,
      "loss": 0.693,
      "rewards/accuracies": 0.5915476083755493,
      "rewards/chosen": 0.148681640625,
      "rewards/margins": 0.11835937201976776,
      "rewards/rejected": 0.03044433519244194,
      "step": 1365
    },
    {
      "epoch": 0.35272914521112253,
      "grad_norm": 213.0,
      "learning_rate": 3.236354273944387e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.09375,
      "logps/chosen": -241.8000030517578,
      "logps/rejected": -203.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.5683882832527161,
      "rewards/chosen": 0.08613280951976776,
      "rewards/margins": 0.19218750298023224,
      "rewards/rejected": -0.10634765774011612,
      "step": 1370
    },
    {
      "epoch": 0.3540164778578785,
      "grad_norm": 972.0,
      "learning_rate": 3.2299176107106076e-07,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -306.20001220703125,
      "logps/rejected": -344.79998779296875,
      "loss": 0.7414,
      "rewards/accuracies": 0.4910714030265808,
      "rewards/chosen": 0.06113281100988388,
      "rewards/margins": 0.03500976413488388,
      "rewards/rejected": 0.02597656287252903,
      "step": 1375
    },
    {
      "epoch": 0.3553038105046344,
      "grad_norm": 228.0,
      "learning_rate": 3.2234809474768275e-07,
      "logits/chosen": -3.296875,
      "logits/rejected": -3.315624952316284,
      "logps/chosen": -330.6000061035156,
      "logps/rejected": -269.3999938964844,
      "loss": 0.6211,
      "rewards/accuracies": 0.6888186931610107,
      "rewards/chosen": 0.16008301079273224,
      "rewards/margins": 0.235595703125,
      "rewards/rejected": -0.07539062201976776,
      "step": 1380
    },
    {
      "epoch": 0.35659114315139034,
      "grad_norm": 166.0,
      "learning_rate": 3.2170442842430485e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -229.0,
      "logps/rejected": -234.1999969482422,
      "loss": 0.6547,
      "rewards/accuracies": 0.5608766078948975,
      "rewards/chosen": 0.17589111626148224,
      "rewards/margins": 0.15385742485523224,
      "rewards/rejected": 0.02177734300494194,
      "step": 1385
    },
    {
      "epoch": 0.35787847579814624,
      "grad_norm": 223.0,
      "learning_rate": 3.2106076210092684e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -327.6000061035156,
      "logps/rejected": -281.79998779296875,
      "loss": 0.6352,
      "rewards/accuracies": 0.6175000071525574,
      "rewards/chosen": 0.174560546875,
      "rewards/margins": 0.22138671576976776,
      "rewards/rejected": -0.04719238355755806,
      "step": 1390
    },
    {
      "epoch": 0.35916580844490215,
      "grad_norm": 233.0,
      "learning_rate": 3.204170957775489e-07,
      "logits/chosen": -3.003124952316284,
      "logits/rejected": -2.864062547683716,
      "logps/chosen": -221.39999389648438,
      "logps/rejected": -214.1999969482422,
      "loss": 0.6164,
      "rewards/accuracies": 0.6492857336997986,
      "rewards/chosen": 0.16201171278953552,
      "rewards/margins": 0.27216798067092896,
      "rewards/rejected": -0.10964355617761612,
      "step": 1395
    },
    {
      "epoch": 0.3604531410916581,
      "grad_norm": 207.0,
      "learning_rate": 3.19773429454171e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -266.0,
      "loss": 0.6094,
      "rewards/accuracies": 0.6816666722297668,
      "rewards/chosen": 0.17934569716453552,
      "rewards/margins": 0.26630860567092896,
      "rewards/rejected": -0.08659668266773224,
      "step": 1400
    },
    {
      "epoch": 0.361740473738414,
      "grad_norm": 324.0,
      "learning_rate": 3.1912976313079296e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -249.1999969482422,
      "loss": 0.6398,
      "rewards/accuracies": 0.5953571200370789,
      "rewards/chosen": 0.3460937440395355,
      "rewards/margins": 0.20541992783546448,
      "rewards/rejected": 0.14072266221046448,
      "step": 1405
    },
    {
      "epoch": 0.3630278063851699,
      "grad_norm": 235.0,
      "learning_rate": 3.18486096807415e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -2.9937500953674316,
      "logps/chosen": -276.3999938964844,
      "logps/rejected": -299.20001220703125,
      "loss": 0.6297,
      "rewards/accuracies": 0.6278571486473083,
      "rewards/chosen": 0.084716796875,
      "rewards/margins": 0.208740234375,
      "rewards/rejected": -0.12412109225988388,
      "step": 1410
    },
    {
      "epoch": 0.36431513903192586,
      "grad_norm": 258.0,
      "learning_rate": 3.178424304840371e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.090625047683716,
      "logps/chosen": -271.79998779296875,
      "logps/rejected": -245.1999969482422,
      "loss": 0.6148,
      "rewards/accuracies": 0.624981701374054,
      "rewards/chosen": 0.18802228569984436,
      "rewards/margins": 0.25,
      "rewards/rejected": -0.06191406399011612,
      "step": 1415
    },
    {
      "epoch": 0.36560247167868176,
      "grad_norm": 213.0,
      "learning_rate": 3.171987641606591e-07,
      "logits/chosen": -3.0218749046325684,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -266.8999938964844,
      "logps/rejected": -242.5,
      "loss": 0.6438,
      "rewards/accuracies": 0.5582143068313599,
      "rewards/chosen": 0.093505859375,
      "rewards/margins": 0.1788330078125,
      "rewards/rejected": -0.08486328274011612,
      "step": 1420
    },
    {
      "epoch": 0.3668898043254377,
      "grad_norm": 316.0,
      "learning_rate": 3.1655509783728114e-07,
      "logits/chosen": -3.2437500953674316,
      "logits/rejected": -3.28125,
      "logps/chosen": -312.6000061035156,
      "logps/rejected": -313.0,
      "loss": 0.6078,
      "rewards/accuracies": 0.6217460632324219,
      "rewards/chosen": 0.22430419921875,
      "rewards/margins": 0.2886962890625,
      "rewards/rejected": -0.06479492038488388,
      "step": 1425
    },
    {
      "epoch": 0.3681771369721936,
      "grad_norm": 256.0,
      "learning_rate": 3.159114315139032e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -248.8000030517578,
      "logps/rejected": -180.39999389648438,
      "loss": 0.5883,
      "rewards/accuracies": 0.6605769395828247,
      "rewards/chosen": 0.184326171875,
      "rewards/margins": 0.27812498807907104,
      "rewards/rejected": -0.09340820461511612,
      "step": 1430
    },
    {
      "epoch": 0.3694644696189495,
      "grad_norm": 219.0,
      "learning_rate": 3.152677651905252e-07,
      "logits/chosen": -2.9437499046325684,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -226.60000610351562,
      "logps/rejected": -222.8000030517578,
      "loss": 0.5836,
      "rewards/accuracies": 0.7469696998596191,
      "rewards/chosen": 0.15839843451976776,
      "rewards/margins": 0.3148437440395355,
      "rewards/rejected": -0.15634766221046448,
      "step": 1435
    },
    {
      "epoch": 0.3707518022657055,
      "grad_norm": 390.0,
      "learning_rate": 3.1462409886714726e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.059375047683716,
      "logps/chosen": -268.70001220703125,
      "logps/rejected": -227.39999389648438,
      "loss": 0.6391,
      "rewards/accuracies": 0.621666669845581,
      "rewards/chosen": 0.131591796875,
      "rewards/margins": 0.19741210341453552,
      "rewards/rejected": -0.06572265923023224,
      "step": 1440
    },
    {
      "epoch": 0.3720391349124614,
      "grad_norm": 300.0,
      "learning_rate": 3.1398043254376925e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.234375,
      "logps/chosen": -306.79998779296875,
      "logps/rejected": -265.6000061035156,
      "loss": 0.6008,
      "rewards/accuracies": 0.6901190876960754,
      "rewards/chosen": 0.19833984971046448,
      "rewards/margins": 0.3050781190395355,
      "rewards/rejected": -0.10732422024011612,
      "step": 1445
    },
    {
      "epoch": 0.3733264675592173,
      "grad_norm": 262.0,
      "learning_rate": 3.1333676622039135e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -325.20001220703125,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6328,
      "rewards/accuracies": 0.5464102625846863,
      "rewards/chosen": 0.3017578125,
      "rewards/margins": 0.24013671278953552,
      "rewards/rejected": 0.06132812425494194,
      "step": 1450
    },
    {
      "epoch": 0.37461380020597324,
      "grad_norm": 240.0,
      "learning_rate": 3.126930998970134e-07,
      "logits/chosen": -3.3125,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -271.0,
      "logps/rejected": -250.39999389648438,
      "loss": 0.6156,
      "rewards/accuracies": 0.64083331823349,
      "rewards/chosen": 0.04252929612994194,
      "rewards/margins": 0.2786621153354645,
      "rewards/rejected": -0.23671874403953552,
      "step": 1455
    },
    {
      "epoch": 0.37590113285272914,
      "grad_norm": 243.0,
      "learning_rate": 3.120494335736354e-07,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -312.3999938964844,
      "logps/rejected": -274.0,
      "loss": 0.6227,
      "rewards/accuracies": 0.5691666603088379,
      "rewards/chosen": 0.2710205018520355,
      "rewards/margins": 0.24833984673023224,
      "rewards/rejected": 0.02364501915872097,
      "step": 1460
    },
    {
      "epoch": 0.3771884654994851,
      "grad_norm": 247.0,
      "learning_rate": 3.114057672502575e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -304.20001220703125,
      "logps/rejected": -267.0,
      "loss": 0.5867,
      "rewards/accuracies": 0.6886905431747437,
      "rewards/chosen": 0.22285155951976776,
      "rewards/margins": 0.30791014432907104,
      "rewards/rejected": -0.08527831733226776,
      "step": 1465
    },
    {
      "epoch": 0.378475798146241,
      "grad_norm": 328.0,
      "learning_rate": 3.1076210092687947e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.15625,
      "logps/chosen": -296.20001220703125,
      "logps/rejected": -233.0,
      "loss": 0.6008,
      "rewards/accuracies": 0.6306974291801453,
      "rewards/chosen": 0.23310546576976776,
      "rewards/margins": 0.27978515625,
      "rewards/rejected": -0.04692382737994194,
      "step": 1470
    },
    {
      "epoch": 0.3797631307929969,
      "grad_norm": 230.0,
      "learning_rate": 3.101184346035015e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.268749952316284,
      "logps/chosen": -308.3999938964844,
      "logps/rejected": -296.5,
      "loss": 0.6383,
      "rewards/accuracies": 0.6131410598754883,
      "rewards/chosen": 0.17543944716453552,
      "rewards/margins": 0.18251952528953552,
      "rewards/rejected": -0.00727081298828125,
      "step": 1475
    },
    {
      "epoch": 0.38105046343975285,
      "grad_norm": 904.0,
      "learning_rate": 3.094747682801236e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -2.635937452316284,
      "logps/chosen": -197.9250030517578,
      "logps/rejected": -241.6999969482422,
      "loss": 0.5328,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": 0.26567381620407104,
      "rewards/margins": 0.5330566167831421,
      "rewards/rejected": -0.26594239473342896,
      "step": 1480
    },
    {
      "epoch": 0.38233779608650875,
      "grad_norm": 184.0,
      "learning_rate": 3.088311019567456e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -300.79998779296875,
      "logps/rejected": -251.1999969482422,
      "loss": 0.6203,
      "rewards/accuracies": 0.6044480204582214,
      "rewards/chosen": 0.26093751192092896,
      "rewards/margins": 0.2177734375,
      "rewards/rejected": 0.04267578199505806,
      "step": 1485
    },
    {
      "epoch": 0.38362512873326465,
      "grad_norm": 164.0,
      "learning_rate": 3.0818743563336764e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -308.79998779296875,
      "logps/rejected": -248.39999389648438,
      "loss": 0.5629,
      "rewards/accuracies": 0.6826923489570618,
      "rewards/chosen": 0.2548828125,
      "rewards/margins": 0.36835938692092896,
      "rewards/rejected": -0.11306152492761612,
      "step": 1490
    },
    {
      "epoch": 0.3849124613800206,
      "grad_norm": 194.0,
      "learning_rate": 3.0754376930998973e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -241.60000610351562,
      "loss": 0.5961,
      "rewards/accuracies": 0.6291667222976685,
      "rewards/chosen": 0.3558593690395355,
      "rewards/margins": 0.331298828125,
      "rewards/rejected": 0.02460937574505806,
      "step": 1495
    },
    {
      "epoch": 0.3861997940267765,
      "grad_norm": 198.0,
      "learning_rate": 3.069001029866117e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -269.79998779296875,
      "loss": 0.5805,
      "rewards/accuracies": 0.6783333420753479,
      "rewards/chosen": 0.2523437440395355,
      "rewards/margins": 0.3119140565395355,
      "rewards/rejected": -0.05947265774011612,
      "step": 1500
    },
    {
      "epoch": 0.38748712667353247,
      "grad_norm": 232.0,
      "learning_rate": 3.0625643666323377e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.2906250953674316,
      "logps/chosen": -311.6000061035156,
      "logps/rejected": -301.3999938964844,
      "loss": 0.6234,
      "rewards/accuracies": 0.6136904954910278,
      "rewards/chosen": 0.2542968690395355,
      "rewards/margins": 0.26289063692092896,
      "rewards/rejected": -0.008471679873764515,
      "step": 1505
    },
    {
      "epoch": 0.38877445932028837,
      "grad_norm": 276.0,
      "learning_rate": 3.056127703398558e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.049999952316284,
      "logps/chosen": -299.20001220703125,
      "logps/rejected": -233.1999969482422,
      "loss": 0.6086,
      "rewards/accuracies": 0.6130952835083008,
      "rewards/chosen": 0.21630859375,
      "rewards/margins": 0.2632812559604645,
      "rewards/rejected": -0.04672851413488388,
      "step": 1510
    },
    {
      "epoch": 0.39006179196704427,
      "grad_norm": 205.0,
      "learning_rate": 3.0496910401647785e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.09375,
      "logps/chosen": -296.0,
      "logps/rejected": -287.3999938964844,
      "loss": 0.5973,
      "rewards/accuracies": 0.6129545569419861,
      "rewards/chosen": 0.23198242485523224,
      "rewards/margins": 0.2840820252895355,
      "rewards/rejected": -0.05225830152630806,
      "step": 1515
    },
    {
      "epoch": 0.3913491246138002,
      "grad_norm": 254.0,
      "learning_rate": 3.043254376930999e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -304.79998779296875,
      "logps/rejected": -311.6000061035156,
      "loss": 0.6129,
      "rewards/accuracies": 0.6401923298835754,
      "rewards/chosen": 0.12285156548023224,
      "rewards/margins": 0.3070922791957855,
      "rewards/rejected": -0.184326171875,
      "step": 1520
    },
    {
      "epoch": 0.3926364572605561,
      "grad_norm": 247.0,
      "learning_rate": 3.036817713697219e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -245.39999389648438,
      "logps/rejected": -207.1999969482422,
      "loss": 0.6117,
      "rewards/accuracies": 0.6879321336746216,
      "rewards/chosen": 0.27910155057907104,
      "rewards/margins": 0.23542480170726776,
      "rewards/rejected": 0.04390869289636612,
      "step": 1525
    },
    {
      "epoch": 0.393923789907312,
      "grad_norm": 232.0,
      "learning_rate": 3.03038105046344e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -256.79998779296875,
      "logps/rejected": -259.20001220703125,
      "loss": 0.5805,
      "rewards/accuracies": 0.64083331823349,
      "rewards/chosen": 0.20913085341453552,
      "rewards/margins": 0.33769530057907104,
      "rewards/rejected": -0.12812499701976776,
      "step": 1530
    },
    {
      "epoch": 0.395211122554068,
      "grad_norm": 446.0,
      "learning_rate": 3.0239443872296597e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -256.79998779296875,
      "logps/rejected": -238.39999389648438,
      "loss": 0.6266,
      "rewards/accuracies": 0.6385714411735535,
      "rewards/chosen": 0.10858154296875,
      "rewards/margins": 0.23251953721046448,
      "rewards/rejected": -0.12412109225988388,
      "step": 1535
    },
    {
      "epoch": 0.3964984552008239,
      "grad_norm": 284.0,
      "learning_rate": 3.01750772399588e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -304.79998779296875,
      "logps/rejected": -321.3999938964844,
      "loss": 0.6734,
      "rewards/accuracies": 0.5783333778381348,
      "rewards/chosen": 0.156097412109375,
      "rewards/margins": 0.18959960341453552,
      "rewards/rejected": -0.03349609300494194,
      "step": 1540
    },
    {
      "epoch": 0.39778578784757984,
      "grad_norm": 264.0,
      "learning_rate": 3.011071060762101e-07,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -3.043750047683716,
      "logps/chosen": -262.79998779296875,
      "logps/rejected": -290.79998779296875,
      "loss": 0.5977,
      "rewards/accuracies": 0.6600000262260437,
      "rewards/chosen": 0.21271362900733948,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.09228515625,
      "step": 1545
    },
    {
      "epoch": 0.39907312049433574,
      "grad_norm": 254.0,
      "learning_rate": 3.004634397528321e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -263.6000061035156,
      "logps/rejected": -204.60000610351562,
      "loss": 0.5988,
      "rewards/accuracies": 0.5990476012229919,
      "rewards/chosen": 0.17431640625,
      "rewards/margins": 0.29443359375,
      "rewards/rejected": -0.1199951171875,
      "step": 1550
    },
    {
      "epoch": 0.40036045314109164,
      "grad_norm": 208.0,
      "learning_rate": 2.9981977342945414e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -277.0,
      "loss": 0.6078,
      "rewards/accuracies": 0.6384615302085876,
      "rewards/chosen": 0.22597655653953552,
      "rewards/margins": 0.26542967557907104,
      "rewards/rejected": -0.039306640625,
      "step": 1555
    },
    {
      "epoch": 0.4016477857878476,
      "grad_norm": 266.0,
      "learning_rate": 2.9917610710607623e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.15625,
      "logps/chosen": -340.79998779296875,
      "logps/rejected": -279.6000061035156,
      "loss": 0.5695,
      "rewards/accuracies": 0.6325000524520874,
      "rewards/chosen": 0.3399414122104645,
      "rewards/margins": 0.36796873807907104,
      "rewards/rejected": -0.02744140662252903,
      "step": 1560
    },
    {
      "epoch": 0.4029351184346035,
      "grad_norm": 211.0,
      "learning_rate": 2.985324407826982e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -278.6000061035156,
      "logps/rejected": -236.8000030517578,
      "loss": 0.6051,
      "rewards/accuracies": 0.6469155550003052,
      "rewards/chosen": 0.22265625,
      "rewards/margins": 0.26958006620407104,
      "rewards/rejected": -0.04597168043255806,
      "step": 1565
    },
    {
      "epoch": 0.4042224510813594,
      "grad_norm": 270.0,
      "learning_rate": 2.9788877445932027e-07,
      "logits/chosen": -3.34375,
      "logits/rejected": -3.331249952316284,
      "logps/chosen": -296.0,
      "logps/rejected": -272.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.5541666746139526,
      "rewards/chosen": 0.03071289137005806,
      "rewards/margins": 0.14877930283546448,
      "rewards/rejected": -0.11821289360523224,
      "step": 1570
    },
    {
      "epoch": 0.40550978372811536,
      "grad_norm": 226.0,
      "learning_rate": 2.972451081359423e-07,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -266.6000061035156,
      "logps/rejected": -247.39999389648438,
      "loss": 0.6125,
      "rewards/accuracies": 0.6819047927856445,
      "rewards/chosen": -0.02612609788775444,
      "rewards/margins": 0.26347655057907104,
      "rewards/rejected": -0.2896484434604645,
      "step": 1575
    },
    {
      "epoch": 0.40679711637487126,
      "grad_norm": 215.0,
      "learning_rate": 2.9660144181256435e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.325000047683716,
      "logps/chosen": -228.8000030517578,
      "logps/rejected": -162.8000030517578,
      "loss": 0.6305,
      "rewards/accuracies": 0.5698168873786926,
      "rewards/chosen": 0.22531738877296448,
      "rewards/margins": 0.19407348334789276,
      "rewards/rejected": 0.03120117262005806,
      "step": 1580
    },
    {
      "epoch": 0.4080844490216272,
      "grad_norm": 368.0,
      "learning_rate": 2.959577754891864e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -245.1999969482422,
      "loss": 0.6383,
      "rewards/accuracies": 0.6325000524520874,
      "rewards/chosen": 0.2945312559604645,
      "rewards/margins": 0.22592774033546448,
      "rewards/rejected": 0.06845702975988388,
      "step": 1585
    },
    {
      "epoch": 0.4093717816683831,
      "grad_norm": 268.0,
      "learning_rate": 2.9531410916580844e-07,
      "logits/chosen": -3.2906250953674316,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -249.1999969482422,
      "loss": 0.5641,
      "rewards/accuracies": 0.7174603343009949,
      "rewards/chosen": 0.23759765923023224,
      "rewards/margins": 0.35546875,
      "rewards/rejected": -0.11813964694738388,
      "step": 1590
    },
    {
      "epoch": 0.410659114315139,
      "grad_norm": 290.0,
      "learning_rate": 2.946704428424305e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -294.3999938964844,
      "logps/rejected": -245.60000610351562,
      "loss": 0.5797,
      "rewards/accuracies": 0.6758333444595337,
      "rewards/chosen": 0.25078123807907104,
      "rewards/margins": 0.32109373807907104,
      "rewards/rejected": -0.07084961235523224,
      "step": 1595
    },
    {
      "epoch": 0.411946446961895,
      "grad_norm": 374.0,
      "learning_rate": 2.940267765190525e-07,
      "logits/chosen": -3.049999952316284,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -261.20001220703125,
      "logps/rejected": -219.39999389648438,
      "loss": 0.5828,
      "rewards/accuracies": 0.6854060292243958,
      "rewards/chosen": 0.28730469942092896,
      "rewards/margins": 0.30451661348342896,
      "rewards/rejected": -0.01674804650247097,
      "step": 1600
    },
    {
      "epoch": 0.4132337796086509,
      "grad_norm": 192.0,
      "learning_rate": 2.933831101956745e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.0718750953674316,
      "logps/chosen": -266.6000061035156,
      "logps/rejected": -215.1999969482422,
      "loss": 0.5711,
      "rewards/accuracies": 0.7022727131843567,
      "rewards/chosen": 0.2943359315395355,
      "rewards/margins": 0.3423828184604645,
      "rewards/rejected": -0.04810791090130806,
      "step": 1605
    },
    {
      "epoch": 0.4145211122554068,
      "grad_norm": 368.0,
      "learning_rate": 2.927394438722966e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -244.8000030517578,
      "logps/rejected": -231.8000030517578,
      "loss": 0.618,
      "rewards/accuracies": 0.633571445941925,
      "rewards/chosen": 0.20925292372703552,
      "rewards/margins": 0.250244140625,
      "rewards/rejected": -0.04104004055261612,
      "step": 1610
    },
    {
      "epoch": 0.41580844490216273,
      "grad_norm": 216.0,
      "learning_rate": 2.920957775489186e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -301.6000061035156,
      "logps/rejected": -281.6000061035156,
      "loss": 0.6586,
      "rewards/accuracies": 0.5839560627937317,
      "rewards/chosen": 0.19252929091453552,
      "rewards/margins": 0.14958496391773224,
      "rewards/rejected": 0.04294433444738388,
      "step": 1615
    },
    {
      "epoch": 0.41709577754891863,
      "grad_norm": 556.0,
      "learning_rate": 2.9145211122554064e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.096874952316284,
      "logps/chosen": -297.5,
      "logps/rejected": -242.60000610351562,
      "loss": 0.657,
      "rewards/accuracies": 0.5566666722297668,
      "rewards/chosen": 0.25737762451171875,
      "rewards/margins": 0.177734375,
      "rewards/rejected": 0.079833984375,
      "step": 1620
    },
    {
      "epoch": 0.41838311019567453,
      "grad_norm": 204.0,
      "learning_rate": 2.9080844490216274e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -238.0,
      "logps/rejected": -216.60000610351562,
      "loss": 0.6664,
      "rewards/accuracies": 0.534835159778595,
      "rewards/chosen": 0.14179687201976776,
      "rewards/margins": 0.13925781846046448,
      "rewards/rejected": 0.0024291991721838713,
      "step": 1625
    },
    {
      "epoch": 0.4196704428424305,
      "grad_norm": 320.0,
      "learning_rate": 2.9016477857878473e-07,
      "logits/chosen": -3.2874999046325684,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -305.20001220703125,
      "logps/rejected": -253.39999389648438,
      "loss": 0.6133,
      "rewards/accuracies": 0.6263583898544312,
      "rewards/chosen": 0.306640625,
      "rewards/margins": 0.2684570252895355,
      "rewards/rejected": 0.03789062425494194,
      "step": 1630
    },
    {
      "epoch": 0.4209577754891864,
      "grad_norm": 234.0,
      "learning_rate": 2.8952111225540677e-07,
      "logits/chosen": -2.879687547683716,
      "logits/rejected": -2.9000000953674316,
      "logps/chosen": -268.6000061035156,
      "logps/rejected": -246.89999389648438,
      "loss": 0.6074,
      "rewards/accuracies": 0.6816666722297668,
      "rewards/chosen": 0.18281249701976776,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.10844726860523224,
      "step": 1635
    },
    {
      "epoch": 0.42224510813594235,
      "grad_norm": 274.0,
      "learning_rate": 2.8887744593202886e-07,
      "logits/chosen": -3.299999952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -308.79998779296875,
      "logps/rejected": -269.79998779296875,
      "loss": 0.5727,
      "rewards/accuracies": 0.6815034747123718,
      "rewards/chosen": 0.3935546875,
      "rewards/margins": 0.3544921875,
      "rewards/rejected": 0.03925519064068794,
      "step": 1640
    },
    {
      "epoch": 0.42353244078269825,
      "grad_norm": 316.0,
      "learning_rate": 2.8823377960865085e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -285.6000061035156,
      "logps/rejected": -259.79998779296875,
      "loss": 0.6289,
      "rewards/accuracies": 0.5472680330276489,
      "rewards/chosen": 0.2860351502895355,
      "rewards/margins": 0.22377929091453552,
      "rewards/rejected": 0.06337890774011612,
      "step": 1645
    },
    {
      "epoch": 0.42481977342945415,
      "grad_norm": 219.0,
      "learning_rate": 2.875901132852729e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -256.79998779296875,
      "logps/rejected": -245.1999969482422,
      "loss": 0.5687,
      "rewards/accuracies": 0.6658333539962769,
      "rewards/chosen": 0.20302733778953552,
      "rewards/margins": 0.375,
      "rewards/rejected": -0.17194823920726776,
      "step": 1650
    },
    {
      "epoch": 0.4261071060762101,
      "grad_norm": 192.0,
      "learning_rate": 2.8694644696189494e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -247.1999969482422,
      "logps/rejected": -250.60000610351562,
      "loss": 0.6297,
      "rewards/accuracies": 0.5752869844436646,
      "rewards/chosen": 0.23398438096046448,
      "rewards/margins": 0.21875,
      "rewards/rejected": 0.01461181603372097,
      "step": 1655
    },
    {
      "epoch": 0.427394438722966,
      "grad_norm": 262.0,
      "learning_rate": 2.86302780638517e-07,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.1875,
      "logps/chosen": -370.3999938964844,
      "logps/rejected": -335.6000061035156,
      "loss": 0.6281,
      "rewards/accuracies": 0.5842857360839844,
      "rewards/chosen": 0.18720702826976776,
      "rewards/margins": 0.26533204317092896,
      "rewards/rejected": -0.07778320461511612,
      "step": 1660
    },
    {
      "epoch": 0.4286817713697219,
      "grad_norm": 232.0,
      "learning_rate": 2.85659114315139e-07,
      "logits/chosen": -3.0093750953674316,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -331.3999938964844,
      "loss": 0.6109,
      "rewards/accuracies": 0.6433333158493042,
      "rewards/chosen": 0.23115234076976776,
      "rewards/margins": 0.2515625059604645,
      "rewards/rejected": -0.02076416090130806,
      "step": 1665
    },
    {
      "epoch": 0.42996910401647787,
      "grad_norm": 220.0,
      "learning_rate": 2.8501544799176107e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.012500047683716,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -265.6000061035156,
      "loss": 0.6023,
      "rewards/accuracies": 0.7196212410926819,
      "rewards/chosen": 0.32499998807907104,
      "rewards/margins": 0.2764648497104645,
      "rewards/rejected": 0.04902343824505806,
      "step": 1670
    },
    {
      "epoch": 0.43125643666323377,
      "grad_norm": 1296.0,
      "learning_rate": 2.843717816683831e-07,
      "logits/chosen": -3.0250000953674316,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -221.8000030517578,
      "logps/rejected": -178.10000610351562,
      "loss": 0.5805,
      "rewards/accuracies": 0.6686050295829773,
      "rewards/chosen": 0.30021971464157104,
      "rewards/margins": 0.36376953125,
      "rewards/rejected": -0.06394042819738388,
      "step": 1675
    },
    {
      "epoch": 0.4325437693099897,
      "grad_norm": 205.0,
      "learning_rate": 2.837281153450051e-07,
      "logits/chosen": -3.3218750953674316,
      "logits/rejected": -3.3843750953674316,
      "logps/chosen": -281.6000061035156,
      "logps/rejected": -200.5500030517578,
      "loss": 0.6398,
      "rewards/accuracies": 0.6184523701667786,
      "rewards/chosen": 0.15390625596046448,
      "rewards/margins": 0.181396484375,
      "rewards/rejected": -0.02797851525247097,
      "step": 1680
    },
    {
      "epoch": 0.4338311019567456,
      "grad_norm": 298.0,
      "learning_rate": 2.830844490216272e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -332.79998779296875,
      "logps/rejected": -277.20001220703125,
      "loss": 0.6312,
      "rewards/accuracies": 0.5773810148239136,
      "rewards/chosen": 0.27216798067092896,
      "rewards/margins": 0.21757812798023224,
      "rewards/rejected": 0.05527343600988388,
      "step": 1685
    },
    {
      "epoch": 0.4351184346035015,
      "grad_norm": 258.0,
      "learning_rate": 2.8244078269824924e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -311.20001220703125,
      "logps/rejected": -287.20001220703125,
      "loss": 0.6211,
      "rewards/accuracies": 0.6150000095367432,
      "rewards/chosen": 0.14711913466453552,
      "rewards/margins": 0.24501952528953552,
      "rewards/rejected": -0.09844360500574112,
      "step": 1690
    },
    {
      "epoch": 0.4364057672502575,
      "grad_norm": 212.0,
      "learning_rate": 2.8179711637487123e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -270.0,
      "logps/rejected": -254.8000030517578,
      "loss": 0.6109,
      "rewards/accuracies": 0.6658333539962769,
      "rewards/chosen": 0.08159179985523224,
      "rewards/margins": 0.27705079317092896,
      "rewards/rejected": -0.19536133110523224,
      "step": 1695
    },
    {
      "epoch": 0.4376930998970134,
      "grad_norm": 220.0,
      "learning_rate": 2.8115345005149327e-07,
      "logits/chosen": -3.012500047683716,
      "logits/rejected": -2.9046874046325684,
      "logps/chosen": -282.29998779296875,
      "logps/rejected": -254.6999969482422,
      "loss": 0.618,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": 0.3472656309604645,
      "rewards/margins": 0.2427978515625,
      "rewards/rejected": 0.10422363132238388,
      "step": 1700
    },
    {
      "epoch": 0.4389804325437693,
      "grad_norm": 208.0,
      "learning_rate": 2.8050978372811537e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -2.9749999046325684,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -266.3999938964844,
      "loss": 0.632,
      "rewards/accuracies": 0.5653113722801208,
      "rewards/chosen": 0.19208984076976776,
      "rewards/margins": 0.24995116889476776,
      "rewards/rejected": -0.05800781399011612,
      "step": 1705
    },
    {
      "epoch": 0.44026776519052524,
      "grad_norm": 235.0,
      "learning_rate": 2.7986611740473736e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -328.79998779296875,
      "logps/rejected": -318.3999938964844,
      "loss": 0.6258,
      "rewards/accuracies": 0.579880952835083,
      "rewards/chosen": 0.15775147080421448,
      "rewards/margins": 0.2552734315395355,
      "rewards/rejected": -0.09775390475988388,
      "step": 1710
    },
    {
      "epoch": 0.44155509783728114,
      "grad_norm": 212.0,
      "learning_rate": 2.792224510813594e-07,
      "logits/chosen": -3.393749952316284,
      "logits/rejected": -2.700390577316284,
      "logps/chosen": -237.4375,
      "logps/rejected": -261.20001220703125,
      "loss": 0.6273,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.18958739936351776,
      "rewards/margins": 0.2787109315395355,
      "rewards/rejected": -0.08945312350988388,
      "step": 1715
    },
    {
      "epoch": 0.4428424304840371,
      "grad_norm": 214.0,
      "learning_rate": 2.7857878475798144e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -332.3999938964844,
      "logps/rejected": -282.0,
      "loss": 0.6531,
      "rewards/accuracies": 0.5383332967758179,
      "rewards/chosen": 0.2651611268520355,
      "rewards/margins": 0.14516600966453552,
      "rewards/rejected": 0.11962890625,
      "step": 1720
    },
    {
      "epoch": 0.444129763130793,
      "grad_norm": 229.0,
      "learning_rate": 2.779351184346035e-07,
      "logits/chosen": -3.112499952316284,
      "logits/rejected": -3.109375,
      "logps/chosen": -260.20001220703125,
      "logps/rejected": -223.6999969482422,
      "loss": 0.5852,
      "rewards/accuracies": 0.6156044602394104,
      "rewards/chosen": 0.24228516221046448,
      "rewards/margins": 0.29082030057907104,
      "rewards/rejected": -0.04885254055261612,
      "step": 1725
    },
    {
      "epoch": 0.4454170957775489,
      "grad_norm": 196.0,
      "learning_rate": 2.7729145211122553e-07,
      "logits/chosen": -3.2718749046325684,
      "logits/rejected": -3.265625,
      "logps/chosen": -315.0,
      "logps/rejected": -291.20001220703125,
      "loss": 0.6234,
      "rewards/accuracies": 0.6019047498703003,
      "rewards/chosen": 0.23652343451976776,
      "rewards/margins": 0.2533203065395355,
      "rewards/rejected": -0.01663818396627903,
      "step": 1730
    },
    {
      "epoch": 0.44670442842430486,
      "grad_norm": 368.0,
      "learning_rate": 2.7664778578784757e-07,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -2.8453125953674316,
      "logps/chosen": -249.39999389648438,
      "logps/rejected": -203.5,
      "loss": 0.6,
      "rewards/accuracies": 0.6576007604598999,
      "rewards/chosen": 0.3740234375,
      "rewards/margins": 0.2978515625,
      "rewards/rejected": 0.075439453125,
      "step": 1735
    },
    {
      "epoch": 0.44799176107106076,
      "grad_norm": 235.0,
      "learning_rate": 2.760041194644696e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.1875,
      "logps/chosen": -252.1999969482422,
      "logps/rejected": -202.8000030517578,
      "loss": 0.5891,
      "rewards/accuracies": 0.6460806131362915,
      "rewards/chosen": 0.13888998329639435,
      "rewards/margins": 0.33476561307907104,
      "rewards/rejected": -0.19619140028953552,
      "step": 1740
    },
    {
      "epoch": 0.44927909371781666,
      "grad_norm": 235.0,
      "learning_rate": 2.7536045314109166e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.2750000953674316,
      "logps/chosen": -218.6999969482422,
      "logps/rejected": -177.0,
      "loss": 0.5789,
      "rewards/accuracies": 0.6978355646133423,
      "rewards/chosen": 0.32470703125,
      "rewards/margins": 0.32666015625,
      "rewards/rejected": -0.0016601562965661287,
      "step": 1745
    },
    {
      "epoch": 0.4505664263645726,
      "grad_norm": 233.0,
      "learning_rate": 2.747167868177137e-07,
      "logits/chosen": -3.2718749046325684,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -291.20001220703125,
      "loss": 0.65,
      "rewards/accuracies": 0.49000000953674316,
      "rewards/chosen": 0.100341796875,
      "rewards/margins": 0.191986083984375,
      "rewards/rejected": -0.092041015625,
      "step": 1750
    },
    {
      "epoch": 0.4518537590113285,
      "grad_norm": 218.0,
      "learning_rate": 2.7407312049433574e-07,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.234375,
      "logps/chosen": -332.3999938964844,
      "logps/rejected": -241.10000610351562,
      "loss": 0.5949,
      "rewards/accuracies": 0.7434848546981812,
      "rewards/chosen": 0.37968748807907104,
      "rewards/margins": 0.33447265625,
      "rewards/rejected": 0.04541015625,
      "step": 1755
    },
    {
      "epoch": 0.45314109165808447,
      "grad_norm": 304.0,
      "learning_rate": 2.7342945417095773e-07,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -3.0374999046325684,
      "logps/chosen": -250.0,
      "logps/rejected": -238.1999969482422,
      "loss": 0.6082,
      "rewards/accuracies": 0.5987012982368469,
      "rewards/chosen": 0.16679687798023224,
      "rewards/margins": 0.26025390625,
      "rewards/rejected": -0.09335937350988388,
      "step": 1760
    },
    {
      "epoch": 0.4544284243048404,
      "grad_norm": 262.0,
      "learning_rate": 2.727857878475798e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -291.6000061035156,
      "logps/rejected": -327.79998779296875,
      "loss": 0.682,
      "rewards/accuracies": 0.573106050491333,
      "rewards/chosen": 0.12724609673023224,
      "rewards/margins": 0.12423095852136612,
      "rewards/rejected": 0.0029785155784338713,
      "step": 1765
    },
    {
      "epoch": 0.4557157569515963,
      "grad_norm": 266.0,
      "learning_rate": 2.7214212152420187e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -3.0093750953674316,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -259.3999938964844,
      "loss": 0.6078,
      "rewards/accuracies": 0.6943589448928833,
      "rewards/chosen": 0.3211425840854645,
      "rewards/margins": 0.2955078184604645,
      "rewards/rejected": 0.02548828162252903,
      "step": 1770
    },
    {
      "epoch": 0.45700308959835223,
      "grad_norm": 212.0,
      "learning_rate": 2.7149845520082386e-07,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -3.234375,
      "logps/chosen": -338.79998779296875,
      "logps/rejected": -246.5,
      "loss": 0.6039,
      "rewards/accuracies": 0.6589285731315613,
      "rewards/chosen": 0.25684815645217896,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.04822998121380806,
      "step": 1775
    },
    {
      "epoch": 0.45829042224510813,
      "grad_norm": 236.0,
      "learning_rate": 2.708547888774459e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -293.0,
      "logps/rejected": -273.3999938964844,
      "loss": 0.6266,
      "rewards/accuracies": 0.5680769085884094,
      "rewards/chosen": 0.19118651747703552,
      "rewards/margins": 0.24702148139476776,
      "rewards/rejected": -0.055419921875,
      "step": 1780
    },
    {
      "epoch": 0.45957775489186403,
      "grad_norm": 178.0,
      "learning_rate": 2.70211122554068e-07,
      "logits/chosen": -3.3031249046325684,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -221.8000030517578,
      "loss": 0.5852,
      "rewards/accuracies": 0.6824816465377808,
      "rewards/chosen": 0.24851074814796448,
      "rewards/margins": 0.3173828125,
      "rewards/rejected": -0.06992187350988388,
      "step": 1785
    },
    {
      "epoch": 0.46086508753862,
      "grad_norm": 290.0,
      "learning_rate": 2.6956745623069e-07,
      "logits/chosen": -3.0062499046325684,
      "logits/rejected": -3.109375,
      "logps/chosen": -246.3000030517578,
      "logps/rejected": -238.6999969482422,
      "loss": 0.8527,
      "rewards/accuracies": 0.6266666650772095,
      "rewards/chosen": 0.26738280057907104,
      "rewards/margins": 0.07841797173023224,
      "rewards/rejected": 0.19063720107078552,
      "step": 1790
    },
    {
      "epoch": 0.4621524201853759,
      "grad_norm": 306.0,
      "learning_rate": 2.6892378990731203e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -288.20001220703125,
      "logps/rejected": -255.1999969482422,
      "loss": 0.6059,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.27128905057907104,
      "rewards/margins": 0.31010740995407104,
      "rewards/rejected": -0.03925781324505806,
      "step": 1795
    },
    {
      "epoch": 0.46343975283213185,
      "grad_norm": 199.0,
      "learning_rate": 2.6828012358393407e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -257.20001220703125,
      "logps/rejected": -275.3999938964844,
      "loss": 0.5977,
      "rewards/accuracies": 0.6470237970352173,
      "rewards/chosen": 0.24591064453125,
      "rewards/margins": 0.2994140684604645,
      "rewards/rejected": -0.05396728590130806,
      "step": 1800
    },
    {
      "epoch": 0.46472708547888775,
      "grad_norm": 448.0,
      "learning_rate": 2.676364572605561e-07,
      "logits/chosen": -3.012500047683716,
      "logits/rejected": -3.0999999046325684,
      "logps/chosen": -184.39999389648438,
      "logps/rejected": -180.0500030517578,
      "loss": 0.5973,
      "rewards/accuracies": 0.654358983039856,
      "rewards/chosen": 0.2647460997104645,
      "rewards/margins": 0.27783203125,
      "rewards/rejected": -0.01328125037252903,
      "step": 1805
    },
    {
      "epoch": 0.46601441812564365,
      "grad_norm": 346.0,
      "learning_rate": 2.6699279093717816e-07,
      "logits/chosen": -2.799999952316284,
      "logits/rejected": -2.671875,
      "logps/chosen": -225.89999389648438,
      "logps/rejected": -234.89999389648438,
      "loss": 0.5938,
      "rewards/accuracies": 0.6571861505508423,
      "rewards/chosen": 0.22246094048023224,
      "rewards/margins": 0.2998046875,
      "rewards/rejected": -0.07675781100988388,
      "step": 1810
    },
    {
      "epoch": 0.4673017507723996,
      "grad_norm": 236.0,
      "learning_rate": 2.663491246138002e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -242.0,
      "logps/rejected": -251.8000030517578,
      "loss": 0.6047,
      "rewards/accuracies": 0.6619231104850769,
      "rewards/chosen": 0.18027344346046448,
      "rewards/margins": 0.24399414658546448,
      "rewards/rejected": -0.06392822414636612,
      "step": 1815
    },
    {
      "epoch": 0.4685890834191555,
      "grad_norm": 230.0,
      "learning_rate": 2.6570545829042224e-07,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -2.987499952316284,
      "logps/chosen": -247.1999969482422,
      "logps/rejected": -253.1999969482422,
      "loss": 0.5633,
      "rewards/accuracies": 0.677628219127655,
      "rewards/chosen": 0.26484376192092896,
      "rewards/margins": 0.371337890625,
      "rewards/rejected": -0.10576172173023224,
      "step": 1820
    },
    {
      "epoch": 0.4698764160659114,
      "grad_norm": 195.0,
      "learning_rate": 2.6506179196704423e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -319.20001220703125,
      "logps/rejected": -285.6000061035156,
      "loss": 0.55,
      "rewards/accuracies": 0.6614394187927246,
      "rewards/chosen": 0.2451171875,
      "rewards/margins": 0.4296875,
      "rewards/rejected": -0.18480224907398224,
      "step": 1825
    },
    {
      "epoch": 0.47116374871266736,
      "grad_norm": 188.0,
      "learning_rate": 2.6441812564366633e-07,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.3687500953674316,
      "logps/chosen": -216.1999969482422,
      "logps/rejected": -206.1999969482422,
      "loss": 0.6289,
      "rewards/accuracies": 0.5517857074737549,
      "rewards/chosen": 0.06289062649011612,
      "rewards/margins": 0.21048584580421448,
      "rewards/rejected": -0.14777831733226776,
      "step": 1830
    },
    {
      "epoch": 0.47245108135942326,
      "grad_norm": 191.0,
      "learning_rate": 2.6377445932028837e-07,
      "logits/chosen": -3.0374999046325684,
      "logits/rejected": -2.9906249046325684,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -250.1999969482422,
      "loss": 0.607,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": 0.21757812798023224,
      "rewards/margins": 0.24785156548023224,
      "rewards/rejected": -0.03041992150247097,
      "step": 1835
    },
    {
      "epoch": 0.4737384140061792,
      "grad_norm": 240.0,
      "learning_rate": 2.6313079299691036e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.140625,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -222.39999389648438,
      "loss": 0.5531,
      "rewards/accuracies": 0.7155952453613281,
      "rewards/chosen": 0.37736815214157104,
      "rewards/margins": 0.4189453125,
      "rewards/rejected": -0.04169921949505806,
      "step": 1840
    },
    {
      "epoch": 0.4750257466529351,
      "grad_norm": 192.0,
      "learning_rate": 2.6248712667353246e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.3531250953674316,
      "logps/chosen": -317.20001220703125,
      "logps/rejected": -267.6000061035156,
      "loss": 0.5742,
      "rewards/accuracies": 0.6491667032241821,
      "rewards/chosen": 0.22744140028953552,
      "rewards/margins": 0.3519531190395355,
      "rewards/rejected": -0.12441406399011612,
      "step": 1845
    },
    {
      "epoch": 0.476313079299691,
      "grad_norm": 219.0,
      "learning_rate": 2.618434603501545e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.21875,
      "logps/chosen": -316.0,
      "logps/rejected": -273.6000061035156,
      "loss": 0.5695,
      "rewards/accuracies": 0.6824817657470703,
      "rewards/chosen": 0.43437498807907104,
      "rewards/margins": 0.3662109375,
      "rewards/rejected": 0.06877441704273224,
      "step": 1850
    },
    {
      "epoch": 0.477600411946447,
      "grad_norm": 234.0,
      "learning_rate": 2.611997940267765e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.3187499046325684,
      "logps/chosen": -234.3000030517578,
      "logps/rejected": -189.0,
      "loss": 0.6281,
      "rewards/accuracies": 0.5718590021133423,
      "rewards/chosen": 0.1455078125,
      "rewards/margins": 0.231201171875,
      "rewards/rejected": -0.08554687350988388,
      "step": 1855
    },
    {
      "epoch": 0.4788877445932029,
      "grad_norm": 294.0,
      "learning_rate": 2.6055612770339853e-07,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -2.9781250953674316,
      "logps/chosen": -179.0,
      "logps/rejected": -201.14999389648438,
      "loss": 0.5953,
      "rewards/accuracies": 0.6028571128845215,
      "rewards/chosen": 0.3037109375,
      "rewards/margins": 0.29960936307907104,
      "rewards/rejected": 0.0042358399368822575,
      "step": 1860
    },
    {
      "epoch": 0.4801750772399588,
      "grad_norm": 260.0,
      "learning_rate": 2.599124613800206e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -280.79998779296875,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6859,
      "rewards/accuracies": 0.5027564764022827,
      "rewards/chosen": -0.03977050632238388,
      "rewards/margins": 0.09165038913488388,
      "rewards/rejected": -0.13173827528953552,
      "step": 1865
    },
    {
      "epoch": 0.48146240988671474,
      "grad_norm": 169.0,
      "learning_rate": 2.592687950566426e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.34375,
      "logps/chosen": -307.3999938964844,
      "logps/rejected": -276.79998779296875,
      "loss": 0.5398,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.23369140923023224,
      "rewards/margins": 0.4332031309604645,
      "rewards/rejected": -0.19877929985523224,
      "step": 1870
    },
    {
      "epoch": 0.48274974253347064,
      "grad_norm": 282.0,
      "learning_rate": 2.5862512873326466e-07,
      "logits/chosen": -3.112499952316284,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -355.0,
      "logps/rejected": -276.3999938964844,
      "loss": 0.6336,
      "rewards/accuracies": 0.6373625993728638,
      "rewards/chosen": 0.3949218690395355,
      "rewards/margins": 0.22919921576976776,
      "rewards/rejected": 0.16547851264476776,
      "step": 1875
    },
    {
      "epoch": 0.4840370751802266,
      "grad_norm": 628.0,
      "learning_rate": 2.579814624098867e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.03125,
      "logps/chosen": -171.3000030517578,
      "logps/rejected": -151.1999969482422,
      "loss": 0.6547,
      "rewards/accuracies": 0.6464285850524902,
      "rewards/chosen": 0.09588623046875,
      "rewards/margins": 0.16850586235523224,
      "rewards/rejected": -0.072265625,
      "step": 1880
    },
    {
      "epoch": 0.4853244078269825,
      "grad_norm": 242.0,
      "learning_rate": 2.5733779608650874e-07,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.043750047683716,
      "logps/chosen": -331.3999938964844,
      "logps/rejected": -259.79998779296875,
      "loss": 0.5547,
      "rewards/accuracies": 0.6357684135437012,
      "rewards/chosen": 0.45781248807907104,
      "rewards/margins": 0.42695313692092896,
      "rewards/rejected": 0.03007812425494194,
      "step": 1885
    },
    {
      "epoch": 0.4866117404737384,
      "grad_norm": 280.0,
      "learning_rate": 2.566941297631308e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.265625,
      "logps/chosen": -261.79998779296875,
      "logps/rejected": -246.1999969482422,
      "loss": 0.607,
      "rewards/accuracies": 0.6491667032241821,
      "rewards/chosen": 0.07319335639476776,
      "rewards/margins": 0.2955078184604645,
      "rewards/rejected": -0.22182616591453552,
      "step": 1890
    },
    {
      "epoch": 0.48789907312049435,
      "grad_norm": 428.0,
      "learning_rate": 2.5605046343975283e-07,
      "logits/chosen": -2.7171874046325684,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -162.8000030517578,
      "logps/rejected": -164.39999389648438,
      "loss": 0.657,
      "rewards/accuracies": 0.49500003457069397,
      "rewards/chosen": 0.15544433891773224,
      "rewards/margins": 0.13339844346046448,
      "rewards/rejected": 0.02236328087747097,
      "step": 1895
    },
    {
      "epoch": 0.48918640576725025,
      "grad_norm": 342.0,
      "learning_rate": 2.5540679711637487e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -268.20001220703125,
      "logps/rejected": -221.39999389648438,
      "loss": 0.625,
      "rewards/accuracies": 0.5600000023841858,
      "rewards/chosen": 0.22919921576976776,
      "rewards/margins": 0.23222656548023224,
      "rewards/rejected": -0.0029296875,
      "step": 1900
    },
    {
      "epoch": 0.49047373841400616,
      "grad_norm": 290.0,
      "learning_rate": 2.5476313079299686e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.0250000953674316,
      "logps/chosen": -271.0,
      "logps/rejected": -246.0,
      "loss": 0.6648,
      "rewards/accuracies": 0.5719444155693054,
      "rewards/chosen": 0.18436889350414276,
      "rewards/margins": 0.14560547471046448,
      "rewards/rejected": 0.03847656399011612,
      "step": 1905
    },
    {
      "epoch": 0.4917610710607621,
      "grad_norm": 184.0,
      "learning_rate": 2.5411946446961896e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.140625,
      "logps/chosen": -283.0,
      "logps/rejected": -282.0,
      "loss": 0.5551,
      "rewards/accuracies": 0.7118590474128723,
      "rewards/chosen": 0.16923828423023224,
      "rewards/margins": 0.4212890565395355,
      "rewards/rejected": -0.2515625059604645,
      "step": 1910
    },
    {
      "epoch": 0.493048403707518,
      "grad_norm": 260.0,
      "learning_rate": 2.53475798146241e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.203125,
      "logps/chosen": -269.20001220703125,
      "logps/rejected": -273.6000061035156,
      "loss": 0.6102,
      "rewards/accuracies": 0.6557143330574036,
      "rewards/chosen": 0.11699219048023224,
      "rewards/margins": 0.2791992127895355,
      "rewards/rejected": -0.16215820610523224,
      "step": 1915
    },
    {
      "epoch": 0.49433573635427397,
      "grad_norm": 229.0,
      "learning_rate": 2.52832131822863e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -300.79998779296875,
      "logps/rejected": -250.0,
      "loss": 0.5586,
      "rewards/accuracies": 0.6621428728103638,
      "rewards/chosen": 0.2925781309604645,
      "rewards/margins": 0.4017578065395355,
      "rewards/rejected": -0.10830078274011612,
      "step": 1920
    },
    {
      "epoch": 0.49562306900102987,
      "grad_norm": 242.0,
      "learning_rate": 2.521884654994851e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.284374952316284,
      "logps/chosen": -259.3999938964844,
      "logps/rejected": -213.1999969482422,
      "loss": 0.6492,
      "rewards/accuracies": 0.5813636779785156,
      "rewards/chosen": 0.12172241508960724,
      "rewards/margins": 0.18085937201976776,
      "rewards/rejected": -0.05952148512005806,
      "step": 1925
    },
    {
      "epoch": 0.49691040164778577,
      "grad_norm": 260.0,
      "learning_rate": 2.5154479917610713e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -275.6000061035156,
      "logps/rejected": -229.0,
      "loss": 0.6297,
      "rewards/accuracies": 0.6287668347358704,
      "rewards/chosen": 0.14882811903953552,
      "rewards/margins": 0.253662109375,
      "rewards/rejected": -0.10413818061351776,
      "step": 1930
    },
    {
      "epoch": 0.4981977342945417,
      "grad_norm": 752.0,
      "learning_rate": 2.509011328527291e-07,
      "logits/chosen": NaN,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -226.8000030517578,
      "logps/rejected": -170.02499389648438,
      "loss": 0.6562,
      "rewards/accuracies": 0.5560317635536194,
      "rewards/chosen": 0.14756926894187927,
      "rewards/margins": 0.16152343153953552,
      "rewards/rejected": -0.013964843936264515,
      "step": 1935
    },
    {
      "epoch": 0.49948506694129763,
      "grad_norm": 226.0,
      "learning_rate": 2.5025746652935116e-07,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -261.6000061035156,
      "logps/rejected": -236.89999389648438,
      "loss": 0.5984,
      "rewards/accuracies": 0.6150504946708679,
      "rewards/chosen": 0.34453123807907104,
      "rewards/margins": 0.33964842557907104,
      "rewards/rejected": 0.005462646484375,
      "step": 1940
    },
    {
      "epoch": 0.5007723995880535,
      "grad_norm": 238.0,
      "learning_rate": 2.496138002059732e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.078125,
      "logps/chosen": -235.1999969482422,
      "logps/rejected": -192.8000030517578,
      "loss": 0.6258,
      "rewards/accuracies": 0.6430769562721252,
      "rewards/chosen": 0.23334960639476776,
      "rewards/margins": 0.2470703125,
      "rewards/rejected": -0.012988281436264515,
      "step": 1945
    },
    {
      "epoch": 0.5020597322348095,
      "grad_norm": 1012.0,
      "learning_rate": 2.4897013388259525e-07,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -348.3999938964844,
      "logps/rejected": -294.0,
      "loss": 0.6742,
      "rewards/accuracies": 0.5817857384681702,
      "rewards/chosen": 0.18027344346046448,
      "rewards/margins": 0.1876220703125,
      "rewards/rejected": -0.00795898400247097,
      "step": 1950
    },
    {
      "epoch": 0.5033470648815654,
      "grad_norm": 278.0,
      "learning_rate": 2.483264675592173e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -255.8000030517578,
      "logps/rejected": -233.39999389648438,
      "loss": 0.6031,
      "rewards/accuracies": 0.6123135685920715,
      "rewards/chosen": 0.10946045070886612,
      "rewards/margins": 0.25664061307907104,
      "rewards/rejected": -0.147705078125,
      "step": 1955
    },
    {
      "epoch": 0.5046343975283213,
      "grad_norm": 258.0,
      "learning_rate": 2.4768280123583933e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.234375,
      "logps/chosen": -291.20001220703125,
      "logps/rejected": -298.3999938964844,
      "loss": 0.5973,
      "rewards/accuracies": 0.7113461494445801,
      "rewards/chosen": 0.13461914658546448,
      "rewards/margins": 0.3638671934604645,
      "rewards/rejected": -0.22929687798023224,
      "step": 1960
    },
    {
      "epoch": 0.5059217301750772,
      "grad_norm": 326.0,
      "learning_rate": 2.470391349124614e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -241.60000610351562,
      "logps/rejected": -214.89999389648438,
      "loss": 0.6836,
      "rewards/accuracies": 0.5298413038253784,
      "rewards/chosen": 0.20302733778953552,
      "rewards/margins": 0.12607422471046448,
      "rewards/rejected": 0.07666015625,
      "step": 1965
    },
    {
      "epoch": 0.5072090628218332,
      "grad_norm": 262.0,
      "learning_rate": 2.463954685890834e-07,
      "logits/chosen": -3.049999952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -335.79998779296875,
      "logps/rejected": -275.0,
      "loss": 0.5672,
      "rewards/accuracies": 0.7561813592910767,
      "rewards/chosen": 0.4371093809604645,
      "rewards/margins": 0.36964112520217896,
      "rewards/rejected": 0.06684570014476776,
      "step": 1970
    },
    {
      "epoch": 0.508496395468589,
      "grad_norm": 246.0,
      "learning_rate": 2.4575180226570546e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -292.3999938964844,
      "logps/rejected": -276.0,
      "loss": 0.6859,
      "rewards/accuracies": 0.5303571820259094,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 0.15249022841453552,
      "rewards/rejected": -0.0032226562034338713,
      "step": 1975
    },
    {
      "epoch": 0.509783728115345,
      "grad_norm": 270.0,
      "learning_rate": 2.451081359423275e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -278.0,
      "logps/rejected": -309.6000061035156,
      "loss": 0.6547,
      "rewards/accuracies": 0.52543044090271,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 0.18515625596046448,
      "rewards/rejected": -0.02695312537252903,
      "step": 1980
    },
    {
      "epoch": 0.511071060762101,
      "grad_norm": 224.0,
      "learning_rate": 2.4446446961894955e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -258.5,
      "logps/rejected": -269.0,
      "loss": 0.6359,
      "rewards/accuracies": 0.5213095545768738,
      "rewards/chosen": 0.15864257514476776,
      "rewards/margins": 0.20869140326976776,
      "rewards/rejected": -0.049560546875,
      "step": 1985
    },
    {
      "epoch": 0.5123583934088568,
      "grad_norm": 266.0,
      "learning_rate": 2.4382080329557153e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -2.9281249046325684,
      "logps/chosen": -301.79998779296875,
      "logps/rejected": -239.39999389648438,
      "loss": 0.5797,
      "rewards/accuracies": 0.7175641059875488,
      "rewards/chosen": 0.2798828184604645,
      "rewards/margins": 0.31696778535842896,
      "rewards/rejected": -0.03740234300494194,
      "step": 1990
    },
    {
      "epoch": 0.5136457260556128,
      "grad_norm": 250.0,
      "learning_rate": 2.4317713697219363e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.3125,
      "logps/chosen": -250.9499969482422,
      "logps/rejected": -267.20001220703125,
      "loss": 0.7156,
      "rewards/accuracies": 0.4679761826992035,
      "rewards/chosen": 0.06186523288488388,
      "rewards/margins": 0.04365234449505806,
      "rewards/rejected": 0.01801757887005806,
      "step": 1995
    },
    {
      "epoch": 0.5149330587023687,
      "grad_norm": 332.0,
      "learning_rate": 2.425334706488156e-07,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.026562452316284,
      "logps/chosen": -256.3999938964844,
      "logps/rejected": -219.39999389648438,
      "loss": 0.5789,
      "rewards/accuracies": 0.7011722326278687,
      "rewards/chosen": 0.4292968809604645,
      "rewards/margins": 0.33056640625,
      "rewards/rejected": 0.09876708686351776,
      "step": 2000
    },
    {
      "epoch": 0.5162203913491246,
      "grad_norm": 203.0,
      "learning_rate": 2.4188980432543766e-07,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -287.79998779296875,
      "logps/rejected": -259.79998779296875,
      "loss": 0.6,
      "rewards/accuracies": 0.6528571844100952,
      "rewards/chosen": 0.23837891221046448,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.04641113430261612,
      "step": 2005
    },
    {
      "epoch": 0.5175077239958805,
      "grad_norm": 222.0,
      "learning_rate": 2.412461380020597e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -331.20001220703125,
      "logps/rejected": -271.0,
      "loss": 0.5719,
      "rewards/accuracies": 0.7089285850524902,
      "rewards/chosen": 0.2831054627895355,
      "rewards/margins": 0.35468751192092896,
      "rewards/rejected": -0.07211913913488388,
      "step": 2010
    },
    {
      "epoch": 0.5187950566426365,
      "grad_norm": 286.0,
      "learning_rate": 2.4060247167868175e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -2.9437499046325684,
      "logps/chosen": -235.1999969482422,
      "logps/rejected": -234.3000030517578,
      "loss": 0.6527,
      "rewards/accuracies": 0.6745887994766235,
      "rewards/chosen": 0.20039062201976776,
      "rewards/margins": 0.1898193359375,
      "rewards/rejected": 0.010449218563735485,
      "step": 2015
    },
    {
      "epoch": 0.5200823892893924,
      "grad_norm": 524.0,
      "learning_rate": 2.399588053553038e-07,
      "logits/chosen": -2.9906249046325684,
      "logits/rejected": -2.832812547683716,
      "logps/chosen": -226.0,
      "logps/rejected": -203.89999389648438,
      "loss": 0.5789,
      "rewards/accuracies": 0.6466667056083679,
      "rewards/chosen": 0.38554686307907104,
      "rewards/margins": 0.3260742127895355,
      "rewards/rejected": 0.0604248046875,
      "step": 2020
    },
    {
      "epoch": 0.5213697219361483,
      "grad_norm": 274.0,
      "learning_rate": 2.3931513903192583e-07,
      "logits/chosen": -3.2906250953674316,
      "logits/rejected": -3.340625047683716,
      "logps/chosen": -290.0,
      "logps/rejected": -246.0,
      "loss": 0.5723,
      "rewards/accuracies": 0.6336904764175415,
      "rewards/chosen": 0.17322388291358948,
      "rewards/margins": 0.3960815370082855,
      "rewards/rejected": -0.22246094048023224,
      "step": 2025
    },
    {
      "epoch": 0.5226570545829042,
      "grad_norm": 194.0,
      "learning_rate": 2.386714727085479e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -324.3999938964844,
      "logps/rejected": -281.20001220703125,
      "loss": 0.6031,
      "rewards/accuracies": 0.5717424154281616,
      "rewards/chosen": 0.2982421815395355,
      "rewards/margins": 0.29570311307907104,
      "rewards/rejected": 0.003173828125,
      "step": 2030
    },
    {
      "epoch": 0.5239443872296602,
      "grad_norm": 201.0,
      "learning_rate": 2.3802780638516992e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.140625,
      "logps/chosen": -223.10000610351562,
      "logps/rejected": -184.3000030517578,
      "loss": 0.6195,
      "rewards/accuracies": 0.6107600927352905,
      "rewards/chosen": 0.21953125298023224,
      "rewards/margins": 0.24619141221046448,
      "rewards/rejected": -0.02652587927877903,
      "step": 2035
    },
    {
      "epoch": 0.525231719876416,
      "grad_norm": 260.0,
      "learning_rate": 2.3738414006179194e-07,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -2.9625000953674316,
      "logps/chosen": -265.79998779296875,
      "logps/rejected": -270.20001220703125,
      "loss": 0.6227,
      "rewards/accuracies": 0.6259523630142212,
      "rewards/chosen": 0.2529052793979645,
      "rewards/margins": 0.265625,
      "rewards/rejected": -0.012402343563735485,
      "step": 2040
    },
    {
      "epoch": 0.526519052523172,
      "grad_norm": 230.0,
      "learning_rate": 2.36740473738414e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -350.3999938964844,
      "logps/rejected": -276.3999938964844,
      "loss": 0.5898,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": 0.251220703125,
      "rewards/margins": 0.3130859434604645,
      "rewards/rejected": -0.06191406399011612,
      "step": 2045
    },
    {
      "epoch": 0.527806385169928,
      "grad_norm": 224.0,
      "learning_rate": 2.3609680741503605e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -2.8968749046325684,
      "logps/chosen": -255.6999969482422,
      "logps/rejected": -207.8000030517578,
      "loss": 0.5922,
      "rewards/accuracies": 0.7033333778381348,
      "rewards/chosen": 0.25238800048828125,
      "rewards/margins": 0.33525389432907104,
      "rewards/rejected": -0.08256836235523224,
      "step": 2050
    },
    {
      "epoch": 0.5290937178166838,
      "grad_norm": 266.0,
      "learning_rate": 2.3545314109165806e-07,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -257.79998779296875,
      "logps/rejected": -220.0,
      "loss": 0.6625,
      "rewards/accuracies": 0.5829137563705444,
      "rewards/chosen": 0.06203613430261612,
      "rewards/margins": 0.16923828423023224,
      "rewards/rejected": -0.10715331882238388,
      "step": 2055
    },
    {
      "epoch": 0.5303810504634398,
      "grad_norm": 220.0,
      "learning_rate": 2.348094747682801e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -276.20001220703125,
      "logps/rejected": -223.8000030517578,
      "loss": 0.5977,
      "rewards/accuracies": 0.5891667008399963,
      "rewards/chosen": 0.16914062201976776,
      "rewards/margins": 0.30224609375,
      "rewards/rejected": -0.13295897841453552,
      "step": 2060
    },
    {
      "epoch": 0.5316683831101957,
      "grad_norm": 206.0,
      "learning_rate": 2.3416580844490218e-07,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -290.3999938964844,
      "logps/rejected": -288.79998779296875,
      "loss": 0.6156,
      "rewards/accuracies": 0.5577813982963562,
      "rewards/chosen": 0.33203125,
      "rewards/margins": 0.27656251192092896,
      "rewards/rejected": 0.05532226711511612,
      "step": 2065
    },
    {
      "epoch": 0.5329557157569516,
      "grad_norm": 230.0,
      "learning_rate": 2.335221421215242e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.3062500953674316,
      "logps/chosen": -252.60000610351562,
      "logps/rejected": -247.8000030517578,
      "loss": 0.6469,
      "rewards/accuracies": 0.5741666555404663,
      "rewards/chosen": 0.12480469048023224,
      "rewards/margins": 0.19890137016773224,
      "rewards/rejected": -0.07392577826976776,
      "step": 2070
    },
    {
      "epoch": 0.5342430484037075,
      "grad_norm": 282.0,
      "learning_rate": 2.3287847579814623e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -214.8000030517578,
      "logps/rejected": -182.1999969482422,
      "loss": 0.6273,
      "rewards/accuracies": 0.5261188745498657,
      "rewards/chosen": 0.24765625596046448,
      "rewards/margins": 0.19941405951976776,
      "rewards/rejected": 0.04755859449505806,
      "step": 2075
    },
    {
      "epoch": 0.5355303810504635,
      "grad_norm": 260.0,
      "learning_rate": 2.3223480947476825e-07,
      "logits/chosen": -3.112499952316284,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -334.3999938964844,
      "logps/rejected": -253.8000030517578,
      "loss": 0.6625,
      "rewards/accuracies": 0.5983150601387024,
      "rewards/chosen": 0.30830079317092896,
      "rewards/margins": 0.17572021484375,
      "rewards/rejected": 0.13320311903953552,
      "step": 2080
    },
    {
      "epoch": 0.5368177136972193,
      "grad_norm": 288.0,
      "learning_rate": 2.315911431513903e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.03125,
      "logps/chosen": -301.20001220703125,
      "logps/rejected": -263.3999938964844,
      "loss": 0.6727,
      "rewards/accuracies": 0.559166669845581,
      "rewards/chosen": 0.20976562798023224,
      "rewards/margins": 0.13120117783546448,
      "rewards/rejected": 0.078857421875,
      "step": 2085
    },
    {
      "epoch": 0.5381050463439753,
      "grad_norm": 252.0,
      "learning_rate": 2.3094747682801236e-07,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -3.003124952316284,
      "logps/chosen": -242.60000610351562,
      "logps/rejected": -201.39999389648438,
      "loss": 0.6258,
      "rewards/accuracies": 0.5659127235412598,
      "rewards/chosen": 0.25324708223342896,
      "rewards/margins": 0.21518555283546448,
      "rewards/rejected": 0.03767089918255806,
      "step": 2090
    },
    {
      "epoch": 0.5393923789907312,
      "grad_norm": 418.0,
      "learning_rate": 2.3030381050463438e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -270.6000061035156,
      "loss": 0.6367,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.246337890625,
      "rewards/margins": 0.22001953423023224,
      "rewards/rejected": 0.0264892578125,
      "step": 2095
    },
    {
      "epoch": 0.5406797116374872,
      "grad_norm": 232.0,
      "learning_rate": 2.2966014418125642e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -282.6000061035156,
      "logps/rejected": -248.0,
      "loss": 0.6105,
      "rewards/accuracies": 0.6246154308319092,
      "rewards/chosen": 0.23593750596046448,
      "rewards/margins": 0.26689451932907104,
      "rewards/rejected": -0.03178710862994194,
      "step": 2100
    },
    {
      "epoch": 0.541967044284243,
      "grad_norm": 328.0,
      "learning_rate": 2.2901647785787846e-07,
      "logits/chosen": -3.262500047683716,
      "logits/rejected": -3.2906250953674316,
      "logps/chosen": -321.20001220703125,
      "logps/rejected": -299.79998779296875,
      "loss": 0.6062,
      "rewards/accuracies": 0.6490909457206726,
      "rewards/chosen": 0.25288087129592896,
      "rewards/margins": 0.29316407442092896,
      "rewards/rejected": -0.04018554836511612,
      "step": 2105
    },
    {
      "epoch": 0.543254376930999,
      "grad_norm": 294.0,
      "learning_rate": 2.283728115345005e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -217.3000030517578,
      "loss": 0.625,
      "rewards/accuracies": 0.5570346117019653,
      "rewards/chosen": 0.27656251192092896,
      "rewards/margins": 0.23544922471046448,
      "rewards/rejected": 0.04134521633386612,
      "step": 2110
    },
    {
      "epoch": 0.5445417095777549,
      "grad_norm": 292.0,
      "learning_rate": 2.2772914521112255e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.109375,
      "logps/chosen": -306.0,
      "logps/rejected": -268.0,
      "loss": 0.5867,
      "rewards/accuracies": 0.655930757522583,
      "rewards/chosen": 0.3001953065395355,
      "rewards/margins": 0.32402342557907104,
      "rewards/rejected": -0.02348632737994194,
      "step": 2115
    },
    {
      "epoch": 0.5458290422245108,
      "grad_norm": 252.0,
      "learning_rate": 2.2708547888774457e-07,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.299999952316284,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -242.1999969482422,
      "loss": 0.5871,
      "rewards/accuracies": 0.686904788017273,
      "rewards/chosen": 0.22963866591453552,
      "rewards/margins": 0.317626953125,
      "rewards/rejected": -0.08820800483226776,
      "step": 2120
    },
    {
      "epoch": 0.5471163748712667,
      "grad_norm": 256.0,
      "learning_rate": 2.264418125643666e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -314.20001220703125,
      "logps/rejected": -302.20001220703125,
      "loss": 0.5844,
      "rewards/accuracies": 0.6433333158493042,
      "rewards/chosen": 0.31718748807907104,
      "rewards/margins": 0.3617187440395355,
      "rewards/rejected": -0.04409179836511612,
      "step": 2125
    },
    {
      "epoch": 0.5484037075180227,
      "grad_norm": 302.0,
      "learning_rate": 2.2579814624098868e-07,
      "logits/chosen": -2.971874952316284,
      "logits/rejected": -2.965625047683716,
      "logps/chosen": -309.6000061035156,
      "logps/rejected": -222.8000030517578,
      "loss": 0.5711,
      "rewards/accuracies": 0.6917948722839355,
      "rewards/chosen": 0.29643553495407104,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -0.07133789360523224,
      "step": 2130
    },
    {
      "epoch": 0.5496910401647785,
      "grad_norm": 218.0,
      "learning_rate": 2.251544799176107e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -273.79998779296875,
      "loss": 0.6352,
      "rewards/accuracies": 0.6775000691413879,
      "rewards/chosen": 0.21513672173023224,
      "rewards/margins": 0.24173584580421448,
      "rewards/rejected": -0.02656250074505806,
      "step": 2135
    },
    {
      "epoch": 0.5509783728115345,
      "grad_norm": 203.0,
      "learning_rate": 2.2451081359423274e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -228.8000030517578,
      "logps/rejected": -245.89999389648438,
      "loss": 0.55,
      "rewards/accuracies": 0.7047618627548218,
      "rewards/chosen": 0.24914856255054474,
      "rewards/margins": 0.40703123807907104,
      "rewards/rejected": -0.15845413506031036,
      "step": 2140
    },
    {
      "epoch": 0.5522657054582905,
      "grad_norm": 214.0,
      "learning_rate": 2.2386714727085478e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -332.79998779296875,
      "logps/rejected": -249.8000030517578,
      "loss": 0.5852,
      "rewards/accuracies": 0.6577380895614624,
      "rewards/chosen": 0.36640626192092896,
      "rewards/margins": 0.3272460997104645,
      "rewards/rejected": 0.0390625,
      "step": 2145
    },
    {
      "epoch": 0.5535530381050463,
      "grad_norm": 268.0,
      "learning_rate": 2.2322348094747682e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -275.8999938964844,
      "loss": 0.5672,
      "rewards/accuracies": 0.6638461351394653,
      "rewards/chosen": 0.2767578065395355,
      "rewards/margins": 0.37128907442092896,
      "rewards/rejected": -0.09492187201976776,
      "step": 2150
    },
    {
      "epoch": 0.5548403707518023,
      "grad_norm": 177.0,
      "learning_rate": 2.2257981462409886e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -272.3999938964844,
      "loss": 0.5816,
      "rewards/accuracies": 0.6861110925674438,
      "rewards/chosen": 0.23408202826976776,
      "rewards/margins": 0.3404296934604645,
      "rewards/rejected": -0.10654296725988388,
      "step": 2155
    },
    {
      "epoch": 0.5561277033985582,
      "grad_norm": 408.0,
      "learning_rate": 2.219361483007209e-07,
      "logits/chosen": -3.0562500953674316,
      "logits/rejected": -3.0625,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -290.0,
      "loss": 0.6203,
      "rewards/accuracies": 0.6308333277702332,
      "rewards/chosen": 0.28271484375,
      "rewards/margins": 0.2520996034145355,
      "rewards/rejected": 0.02988281287252903,
      "step": 2160
    },
    {
      "epoch": 0.5574150360453141,
      "grad_norm": 274.0,
      "learning_rate": 2.2129248197734292e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -320.79998779296875,
      "logps/rejected": -247.8000030517578,
      "loss": 0.6414,
      "rewards/accuracies": 0.574404776096344,
      "rewards/chosen": 0.23076172173023224,
      "rewards/margins": 0.20791015028953552,
      "rewards/rejected": 0.02265625074505806,
      "step": 2165
    },
    {
      "epoch": 0.55870236869207,
      "grad_norm": 312.0,
      "learning_rate": 2.20648815653965e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -251.1999969482422,
      "logps/rejected": -218.39999389648438,
      "loss": 0.6211,
      "rewards/accuracies": 0.6708974838256836,
      "rewards/chosen": 0.19882813096046448,
      "rewards/margins": 0.23710937798023224,
      "rewards/rejected": -0.03774414211511612,
      "step": 2170
    },
    {
      "epoch": 0.559989701338826,
      "grad_norm": 280.0,
      "learning_rate": 2.20005149330587e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.1875,
      "logps/chosen": -265.20001220703125,
      "logps/rejected": -223.8000030517578,
      "loss": 0.568,
      "rewards/accuracies": 0.7163461446762085,
      "rewards/chosen": 0.19594725966453552,
      "rewards/margins": 0.3716796934604645,
      "rewards/rejected": -0.17607422173023224,
      "step": 2175
    },
    {
      "epoch": 0.5612770339855818,
      "grad_norm": 234.0,
      "learning_rate": 2.1936148300720905e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.315624952316284,
      "logps/chosen": -268.6000061035156,
      "logps/rejected": -251.0,
      "loss": 0.5547,
      "rewards/accuracies": 0.726190447807312,
      "rewards/chosen": 0.218994140625,
      "rewards/margins": 0.3857421875,
      "rewards/rejected": -0.1669921875,
      "step": 2180
    },
    {
      "epoch": 0.5625643666323378,
      "grad_norm": 227.0,
      "learning_rate": 2.187178166838311e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -242.8000030517578,
      "logps/rejected": -245.39999389648438,
      "loss": 0.5984,
      "rewards/accuracies": 0.6083028316497803,
      "rewards/chosen": 0.26191407442092896,
      "rewards/margins": 0.3203125,
      "rewards/rejected": -0.05878906324505806,
      "step": 2185
    },
    {
      "epoch": 0.5638516992790937,
      "grad_norm": 280.0,
      "learning_rate": 2.1807415036045314e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.140625,
      "logps/chosen": -295.79998779296875,
      "logps/rejected": -271.79998779296875,
      "loss": 0.5711,
      "rewards/accuracies": 0.6706410646438599,
      "rewards/chosen": 0.32084959745407104,
      "rewards/margins": 0.3681640625,
      "rewards/rejected": -0.04794921725988388,
      "step": 2190
    },
    {
      "epoch": 0.5651390319258497,
      "grad_norm": 236.0,
      "learning_rate": 2.1743048403707518e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": NaN,
      "logps/chosen": -250.89999389648438,
      "logps/rejected": -227.0,
      "loss": 0.6047,
      "rewards/accuracies": 0.6425000429153442,
      "rewards/chosen": 0.23403319716453552,
      "rewards/margins": 0.2718749940395355,
      "rewards/rejected": -0.03798828274011612,
      "step": 2195
    },
    {
      "epoch": 0.5664263645726055,
      "grad_norm": 324.0,
      "learning_rate": 2.1678681771369722e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -280.3999938964844,
      "logps/rejected": -252.1999969482422,
      "loss": 0.6039,
      "rewards/accuracies": 0.6309981942176819,
      "rewards/chosen": 0.16875000298023224,
      "rewards/margins": 0.3359375,
      "rewards/rejected": -0.16783447563648224,
      "step": 2200
    },
    {
      "epoch": 0.5677136972193615,
      "grad_norm": 206.0,
      "learning_rate": 2.1614315139031924e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -282.0,
      "loss": 0.6484,
      "rewards/accuracies": 0.5557509064674377,
      "rewards/chosen": 0.36503905057907104,
      "rewards/margins": 0.20712891221046448,
      "rewards/rejected": 0.15761718153953552,
      "step": 2205
    },
    {
      "epoch": 0.5690010298661174,
      "grad_norm": 270.0,
      "learning_rate": 2.154994850669413e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -236.39999389648438,
      "loss": 0.5691,
      "rewards/accuracies": 0.7252747416496277,
      "rewards/chosen": 0.2506347596645355,
      "rewards/margins": 0.36054688692092896,
      "rewards/rejected": -0.11044921725988388,
      "step": 2210
    },
    {
      "epoch": 0.5702883625128733,
      "grad_norm": 245.0,
      "learning_rate": 2.1485581874356332e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -275.20001220703125,
      "logps/rejected": -266.6000061035156,
      "loss": 0.5785,
      "rewards/accuracies": 0.6678571701049805,
      "rewards/chosen": 0.308370977640152,
      "rewards/margins": 0.37089842557907104,
      "rewards/rejected": -0.062255859375,
      "step": 2215
    },
    {
      "epoch": 0.5715756951596292,
      "grad_norm": 247.0,
      "learning_rate": 2.1421215242018537e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -256.20001220703125,
      "logps/rejected": -244.0,
      "loss": 0.5504,
      "rewards/accuracies": 0.7111905217170715,
      "rewards/chosen": 0.202880859375,
      "rewards/margins": 0.4052734375,
      "rewards/rejected": -0.20283202826976776,
      "step": 2220
    },
    {
      "epoch": 0.5728630278063852,
      "grad_norm": 232.0,
      "learning_rate": 2.135684860968074e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -237.8000030517578,
      "logps/rejected": -280.6000061035156,
      "loss": 0.5992,
      "rewards/accuracies": 0.6042624115943909,
      "rewards/chosen": 0.2776855528354645,
      "rewards/margins": 0.28925782442092896,
      "rewards/rejected": -0.011322021484375,
      "step": 2225
    },
    {
      "epoch": 0.574150360453141,
      "grad_norm": 232.0,
      "learning_rate": 2.1292481977342942e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -250.0,
      "loss": 0.5594,
      "rewards/accuracies": 0.6670330166816711,
      "rewards/chosen": 0.30351561307907104,
      "rewards/margins": 0.3857421875,
      "rewards/rejected": -0.081787109375,
      "step": 2230
    },
    {
      "epoch": 0.575437693099897,
      "grad_norm": 302.0,
      "learning_rate": 2.122811534500515e-07,
      "logits/chosen": -3.049999952316284,
      "logits/rejected": -3.112499952316284,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -247.1999969482422,
      "loss": 0.6281,
      "rewards/accuracies": 0.6481776833534241,
      "rewards/chosen": 0.27099609375,
      "rewards/margins": 0.22968749701976776,
      "rewards/rejected": 0.04177246242761612,
      "step": 2235
    },
    {
      "epoch": 0.576725025746653,
      "grad_norm": 250.0,
      "learning_rate": 2.1163748712667354e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.0406250953674316,
      "logps/chosen": -223.85000610351562,
      "logps/rejected": -194.5500030517578,
      "loss": 0.6609,
      "rewards/accuracies": 0.570457935333252,
      "rewards/chosen": 0.23085936903953552,
      "rewards/margins": 0.17294922471046448,
      "rewards/rejected": 0.05864257737994194,
      "step": 2240
    },
    {
      "epoch": 0.5780123583934088,
      "grad_norm": 220.0,
      "learning_rate": 2.1099382080329555e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.25,
      "logps/chosen": -276.6000061035156,
      "logps/rejected": -227.8000030517578,
      "loss": 0.5762,
      "rewards/accuracies": 0.6498810052871704,
      "rewards/chosen": 0.3814453184604645,
      "rewards/margins": 0.38359373807907104,
      "rewards/rejected": -0.0015625000232830644,
      "step": 2245
    },
    {
      "epoch": 0.5792996910401648,
      "grad_norm": 272.0,
      "learning_rate": 2.103501544799176e-07,
      "logits/chosen": -2.9296875,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -258.3999938964844,
      "logps/rejected": -299.20001220703125,
      "loss": 0.6727,
      "rewards/accuracies": 0.5082265138626099,
      "rewards/chosen": 0.23476561903953552,
      "rewards/margins": 0.12045898288488388,
      "rewards/rejected": 0.1142578125,
      "step": 2250
    },
    {
      "epoch": 0.5805870236869207,
      "grad_norm": 162.0,
      "learning_rate": 2.0970648815653964e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -352.6000061035156,
      "logps/rejected": -268.79998779296875,
      "loss": 0.5711,
      "rewards/accuracies": 0.6530769467353821,
      "rewards/chosen": 0.4056640565395355,
      "rewards/margins": 0.3695312440395355,
      "rewards/rejected": 0.03647460788488388,
      "step": 2255
    },
    {
      "epoch": 0.5818743563336766,
      "grad_norm": 258.0,
      "learning_rate": 2.0906282183316168e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.112499952316284,
      "logps/chosen": -255.39999389648438,
      "logps/rejected": -229.10000610351562,
      "loss": 0.6047,
      "rewards/accuracies": 0.7079545259475708,
      "rewards/chosen": 0.24240723252296448,
      "rewards/margins": 0.26152342557907104,
      "rewards/rejected": -0.01906738243997097,
      "step": 2260
    },
    {
      "epoch": 0.5831616889804325,
      "grad_norm": 548.0,
      "learning_rate": 2.0841915550978372e-07,
      "logits/chosen": -2.934375047683716,
      "logits/rejected": -3.140625,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -227.10000610351562,
      "loss": 0.6422,
      "rewards/accuracies": 0.5818290114402771,
      "rewards/chosen": 0.306640625,
      "rewards/margins": 0.24667969346046448,
      "rewards/rejected": 0.05980529636144638,
      "step": 2265
    },
    {
      "epoch": 0.5844490216271885,
      "grad_norm": 203.0,
      "learning_rate": 2.0777548918640574e-07,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.109375,
      "logps/chosen": -256.79998779296875,
      "logps/rejected": -239.1999969482422,
      "loss": 0.6078,
      "rewards/accuracies": 0.6091667413711548,
      "rewards/chosen": 0.12119140475988388,
      "rewards/margins": 0.2767578065395355,
      "rewards/rejected": -0.156005859375,
      "step": 2270
    },
    {
      "epoch": 0.5857363542739444,
      "grad_norm": 258.0,
      "learning_rate": 2.071318228630278e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -299.79998779296875,
      "logps/rejected": -291.79998779296875,
      "loss": 0.5898,
      "rewards/accuracies": 0.658049464225769,
      "rewards/chosen": 0.36250001192092896,
      "rewards/margins": 0.3248046934604645,
      "rewards/rejected": 0.03701172024011612,
      "step": 2275
    },
    {
      "epoch": 0.5870236869207003,
      "grad_norm": 215.0,
      "learning_rate": 2.0648815653964985e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -250.0,
      "logps/rejected": -222.1999969482422,
      "loss": 0.6664,
      "rewards/accuracies": 0.6267949342727661,
      "rewards/chosen": -0.14121094346046448,
      "rewards/margins": 0.1591796875,
      "rewards/rejected": -0.3001953065395355,
      "step": 2280
    },
    {
      "epoch": 0.5883110195674562,
      "grad_norm": 218.0,
      "learning_rate": 2.0584449021627187e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -291.3999938964844,
      "logps/rejected": -243.8000030517578,
      "loss": 0.632,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": 0.30908203125,
      "rewards/margins": 0.20859375596046448,
      "rewards/rejected": 0.10039062798023224,
      "step": 2285
    },
    {
      "epoch": 0.5895983522142122,
      "grad_norm": 238.0,
      "learning_rate": 2.052008238928939e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -288.8999938964844,
      "logps/rejected": -279.3999938964844,
      "loss": 0.593,
      "rewards/accuracies": 0.678928554058075,
      "rewards/chosen": 0.2685546875,
      "rewards/margins": 0.3021484315395355,
      "rewards/rejected": -0.03334961086511612,
      "step": 2290
    },
    {
      "epoch": 0.590885684860968,
      "grad_norm": 201.0,
      "learning_rate": 2.0455715756951595e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -254.1999969482422,
      "logps/rejected": -254.0,
      "loss": 0.6383,
      "rewards/accuracies": 0.7069047689437866,
      "rewards/chosen": 0.19707031548023224,
      "rewards/margins": 0.19121094048023224,
      "rewards/rejected": 0.0058837891556322575,
      "step": 2295
    },
    {
      "epoch": 0.592173017507724,
      "grad_norm": 246.0,
      "learning_rate": 2.03913491246138e-07,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -286.3999938964844,
      "logps/rejected": -255.39999389648438,
      "loss": 0.5473,
      "rewards/accuracies": 0.7696429491043091,
      "rewards/chosen": 0.43793946504592896,
      "rewards/margins": 0.42460936307907104,
      "rewards/rejected": 0.012890624813735485,
      "step": 2300
    },
    {
      "epoch": 0.59346035015448,
      "grad_norm": 276.0,
      "learning_rate": 2.0326982492276004e-07,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -314.3999938964844,
      "logps/rejected": -301.79998779296875,
      "loss": 0.5699,
      "rewards/accuracies": 0.59333336353302,
      "rewards/chosen": 0.4027343690395355,
      "rewards/margins": 0.3560546934604645,
      "rewards/rejected": 0.04618988186120987,
      "step": 2305
    },
    {
      "epoch": 0.5947476828012358,
      "grad_norm": 204.0,
      "learning_rate": 2.0262615859938205e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -363.79998779296875,
      "logps/rejected": -336.3999938964844,
      "loss": 0.5797,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2412109375,
      "rewards/margins": 0.36601561307907104,
      "rewards/rejected": -0.12451171875,
      "step": 2310
    },
    {
      "epoch": 0.5960350154479918,
      "grad_norm": 220.0,
      "learning_rate": 2.0198249227600412e-07,
      "logits/chosen": -3.0250000953674316,
      "logits/rejected": -3.0625,
      "logps/chosen": -266.0,
      "logps/rejected": -226.39999389648438,
      "loss": 0.6305,
      "rewards/accuracies": 0.6131318807601929,
      "rewards/chosen": 0.30302733182907104,
      "rewards/margins": 0.232421875,
      "rewards/rejected": 0.07099609076976776,
      "step": 2315
    },
    {
      "epoch": 0.5973223480947477,
      "grad_norm": 182.0,
      "learning_rate": 2.0133882595262617e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -2.953125,
      "logps/chosen": -294.79998779296875,
      "logps/rejected": -269.3999938964844,
      "loss": 0.6477,
      "rewards/accuracies": 0.5907575488090515,
      "rewards/chosen": 0.09145507961511612,
      "rewards/margins": 0.1861572265625,
      "rewards/rejected": -0.09531249850988388,
      "step": 2320
    },
    {
      "epoch": 0.5986096807415036,
      "grad_norm": 209.0,
      "learning_rate": 2.0069515962924818e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -223.1999969482422,
      "logps/rejected": -257.79998779296875,
      "loss": 0.6023,
      "rewards/accuracies": 0.6796428561210632,
      "rewards/chosen": 0.16440430283546448,
      "rewards/margins": 0.27308958768844604,
      "rewards/rejected": -0.10872497409582138,
      "step": 2325
    },
    {
      "epoch": 0.5998970133882595,
      "grad_norm": 308.0,
      "learning_rate": 2.0005149330587023e-07,
      "logits/chosen": -3.0718750953674316,
      "logits/rejected": -3.018749952316284,
      "logps/chosen": -180.5,
      "logps/rejected": -163.6999969482422,
      "loss": 0.5516,
      "rewards/accuracies": 0.7195237874984741,
      "rewards/chosen": 0.3900390565395355,
      "rewards/margins": 0.40937501192092896,
      "rewards/rejected": -0.01889648474752903,
      "step": 2330
    },
    {
      "epoch": 0.6011843460350155,
      "grad_norm": 316.0,
      "learning_rate": 1.9940782698249227e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -281.20001220703125,
      "logps/rejected": -302.6000061035156,
      "loss": 0.6105,
      "rewards/accuracies": 0.6313888430595398,
      "rewards/chosen": 0.2763671875,
      "rewards/margins": 0.2833007872104645,
      "rewards/rejected": -0.006518554873764515,
      "step": 2335
    },
    {
      "epoch": 0.6024716786817713,
      "grad_norm": 243.0,
      "learning_rate": 1.987641606591143e-07,
      "logits/chosen": -3.0093750953674316,
      "logits/rejected": -2.971874952316284,
      "logps/chosen": -293.3999938964844,
      "logps/rejected": -311.79998779296875,
      "loss": 0.593,
      "rewards/accuracies": 0.6114010810852051,
      "rewards/chosen": 0.36259764432907104,
      "rewards/margins": 0.40058594942092896,
      "rewards/rejected": -0.03764648362994194,
      "step": 2340
    },
    {
      "epoch": 0.6037590113285273,
      "grad_norm": 298.0,
      "learning_rate": 1.9812049433573635e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -268.6000061035156,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6062,
      "rewards/accuracies": 0.5627778172492981,
      "rewards/chosen": 0.2099609375,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.07490234076976776,
      "step": 2345
    },
    {
      "epoch": 0.6050463439752832,
      "grad_norm": 280.0,
      "learning_rate": 1.9747682801235837e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -2.8031249046325684,
      "logps/chosen": -229.39999389648438,
      "logps/rejected": -221.1999969482422,
      "loss": 0.6027,
      "rewards/accuracies": 0.5875757932662964,
      "rewards/chosen": 0.12739257514476776,
      "rewards/margins": 0.28399658203125,
      "rewards/rejected": -0.15654297173023224,
      "step": 2350
    },
    {
      "epoch": 0.6063336766220392,
      "grad_norm": 286.0,
      "learning_rate": 1.9683316168898044e-07,
      "logits/chosen": -3.0718750953674316,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -279.20001220703125,
      "logps/rejected": -245.1999969482422,
      "loss": 0.5664,
      "rewards/accuracies": 0.6626282334327698,
      "rewards/chosen": 0.33338624238967896,
      "rewards/margins": 0.357421875,
      "rewards/rejected": -0.02358398400247097,
      "step": 2355
    },
    {
      "epoch": 0.607621009268795,
      "grad_norm": 314.0,
      "learning_rate": 1.9618949536560248e-07,
      "logits/chosen": -3.2437500953674316,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -286.0,
      "logps/rejected": -287.6000061035156,
      "loss": 0.6297,
      "rewards/accuracies": 0.602738082408905,
      "rewards/chosen": 0.02690429612994194,
      "rewards/margins": 0.22597655653953552,
      "rewards/rejected": -0.19945068657398224,
      "step": 2360
    },
    {
      "epoch": 0.608908341915551,
      "grad_norm": 212.0,
      "learning_rate": 1.955458290422245e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -251.60000610351562,
      "logps/rejected": -220.0,
      "loss": 0.5746,
      "rewards/accuracies": 0.6993743181228638,
      "rewards/chosen": 0.29594725370407104,
      "rewards/margins": 0.35302734375,
      "rewards/rejected": -0.05662841722369194,
      "step": 2365
    },
    {
      "epoch": 0.6101956745623069,
      "grad_norm": 266.0,
      "learning_rate": 1.9490216271884654e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.831249952316284,
      "logps/chosen": -259.0,
      "logps/rejected": -239.1999969482422,
      "loss": 0.5844,
      "rewards/accuracies": 0.6728030443191528,
      "rewards/chosen": 0.29106444120407104,
      "rewards/margins": 0.3515625,
      "rewards/rejected": -0.06118164211511612,
      "step": 2370
    },
    {
      "epoch": 0.6114830072090628,
      "grad_norm": 208.0,
      "learning_rate": 1.9425849639546856e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.109375,
      "logps/chosen": -221.8000030517578,
      "logps/rejected": -227.60000610351562,
      "loss": 0.618,
      "rewards/accuracies": 0.6118406653404236,
      "rewards/chosen": 0.2236328125,
      "rewards/margins": 0.2701660096645355,
      "rewards/rejected": -0.04599609225988388,
      "step": 2375
    },
    {
      "epoch": 0.6127703398558187,
      "grad_norm": 408.0,
      "learning_rate": 1.9361483007209063e-07,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -286.6000061035156,
      "logps/rejected": -258.20001220703125,
      "loss": 0.5746,
      "rewards/accuracies": 0.6657692193984985,
      "rewards/chosen": 0.330078125,
      "rewards/margins": 0.32890623807907104,
      "rewards/rejected": 0.0002929687616415322,
      "step": 2380
    },
    {
      "epoch": 0.6140576725025747,
      "grad_norm": 208.0,
      "learning_rate": 1.9297116374871267e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -2.9281249046325684,
      "logps/chosen": -246.1999969482422,
      "logps/rejected": -188.75,
      "loss": 0.6141,
      "rewards/accuracies": 0.5500000715255737,
      "rewards/chosen": 0.19760742783546448,
      "rewards/margins": 0.22895507514476776,
      "rewards/rejected": -0.03194580227136612,
      "step": 2385
    },
    {
      "epoch": 0.6153450051493305,
      "grad_norm": 796.0,
      "learning_rate": 1.9232749742533468e-07,
      "logits/chosen": -3.0218749046325684,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -217.60000610351562,
      "logps/rejected": -214.3000030517578,
      "loss": 0.6703,
      "rewards/accuracies": 0.5186654925346375,
      "rewards/chosen": 0.21074219048023224,
      "rewards/margins": 0.14453125,
      "rewards/rejected": 0.06564941257238388,
      "step": 2390
    },
    {
      "epoch": 0.6166323377960865,
      "grad_norm": 454.0,
      "learning_rate": 1.9168383110195673e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -338.79998779296875,
      "logps/rejected": -298.6000061035156,
      "loss": 0.6305,
      "rewards/accuracies": 0.6235714554786682,
      "rewards/chosen": 0.212158203125,
      "rewards/margins": 0.2699218690395355,
      "rewards/rejected": -0.05712890625,
      "step": 2395
    },
    {
      "epoch": 0.6179196704428425,
      "grad_norm": 612.0,
      "learning_rate": 1.910401647785788e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.0687499046325684,
      "logps/chosen": -261.6000061035156,
      "logps/rejected": -279.6000061035156,
      "loss": 0.6305,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.10830078274011612,
      "rewards/margins": 0.26982420682907104,
      "rewards/rejected": -0.16191406548023224,
      "step": 2400
    },
    {
      "epoch": 0.6192070030895983,
      "grad_norm": 229.0,
      "learning_rate": 1.903964984552008e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -259.20001220703125,
      "logps/rejected": -201.1999969482422,
      "loss": 0.6406,
      "rewards/accuracies": 0.5967949032783508,
      "rewards/chosen": 0.13901367783546448,
      "rewards/margins": 0.19091796875,
      "rewards/rejected": -0.05190429836511612,
      "step": 2405
    },
    {
      "epoch": 0.6204943357363543,
      "grad_norm": 312.0,
      "learning_rate": 1.8975283213182286e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.265625,
      "logps/chosen": -354.79998779296875,
      "logps/rejected": -282.6000061035156,
      "loss": 0.5469,
      "rewards/accuracies": 0.7439286112785339,
      "rewards/chosen": 0.24858398735523224,
      "rewards/margins": 0.41484373807907104,
      "rewards/rejected": -0.16623535752296448,
      "step": 2410
    },
    {
      "epoch": 0.6217816683831102,
      "grad_norm": 242.0,
      "learning_rate": 1.8910916580844487e-07,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -288.79998779296875,
      "logps/rejected": -236.1999969482422,
      "loss": 0.6516,
      "rewards/accuracies": 0.5587302446365356,
      "rewards/chosen": 0.31171876192092896,
      "rewards/margins": 0.16249999403953552,
      "rewards/rejected": 0.14902344346046448,
      "step": 2415
    },
    {
      "epoch": 0.6230690010298661,
      "grad_norm": 215.0,
      "learning_rate": 1.8846549948506694e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -272.0,
      "logps/rejected": -247.0,
      "loss": 0.5992,
      "rewards/accuracies": 0.5805194973945618,
      "rewards/chosen": 0.230224609375,
      "rewards/margins": 0.2983459532260895,
      "rewards/rejected": -0.06816406548023224,
      "step": 2420
    },
    {
      "epoch": 0.624356333676622,
      "grad_norm": 218.0,
      "learning_rate": 1.8782183316168898e-07,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -301.79998779296875,
      "logps/rejected": -274.3999938964844,
      "loss": 0.607,
      "rewards/accuracies": 0.576785683631897,
      "rewards/chosen": 0.15175780653953552,
      "rewards/margins": 0.29374998807907104,
      "rewards/rejected": -0.14179687201976776,
      "step": 2425
    },
    {
      "epoch": 0.625643666323378,
      "grad_norm": 358.0,
      "learning_rate": 1.87178166838311e-07,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -298.0,
      "logps/rejected": -266.6000061035156,
      "loss": 0.65,
      "rewards/accuracies": 0.5467033386230469,
      "rewards/chosen": 0.2625366151332855,
      "rewards/margins": 0.18217773735523224,
      "rewards/rejected": 0.08039550483226776,
      "step": 2430
    },
    {
      "epoch": 0.6269309989701339,
      "grad_norm": 316.0,
      "learning_rate": 1.8653450051493304e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.25,
      "logps/chosen": -303.3999938964844,
      "logps/rejected": -267.20001220703125,
      "loss": 0.6891,
      "rewards/accuracies": 0.5019047856330872,
      "rewards/chosen": 0.21860352158546448,
      "rewards/margins": 0.104248046875,
      "rewards/rejected": 0.11435546725988388,
      "step": 2435
    },
    {
      "epoch": 0.6282183316168898,
      "grad_norm": 266.0,
      "learning_rate": 1.858908341915551e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -289.20001220703125,
      "loss": 0.6258,
      "rewards/accuracies": 0.6885714530944824,
      "rewards/chosen": 0.15154418349266052,
      "rewards/margins": 0.24882812798023224,
      "rewards/rejected": -0.09736327826976776,
      "step": 2440
    },
    {
      "epoch": 0.6295056642636457,
      "grad_norm": 272.0,
      "learning_rate": 1.8524716786817713e-07,
      "logits/chosen": -2.956249952316284,
      "logits/rejected": -3.03125,
      "logps/chosen": -220.1999969482422,
      "logps/rejected": -212.0,
      "loss": 0.6344,
      "rewards/accuracies": 0.5760822296142578,
      "rewards/chosen": 0.30937498807907104,
      "rewards/margins": 0.20543213188648224,
      "rewards/rejected": 0.10366211086511612,
      "step": 2445
    },
    {
      "epoch": 0.6307929969104017,
      "grad_norm": 268.0,
      "learning_rate": 1.8460350154479917e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -282.79998779296875,
      "logps/rejected": -249.1999969482422,
      "loss": 0.5867,
      "rewards/accuracies": 0.6816667318344116,
      "rewards/chosen": 0.25419920682907104,
      "rewards/margins": 0.31494140625,
      "rewards/rejected": -0.06065673753619194,
      "step": 2450
    },
    {
      "epoch": 0.6320803295571575,
      "grad_norm": 249.0,
      "learning_rate": 1.8395983522142119e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -310.0,
      "logps/rejected": -305.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.5648809671401978,
      "rewards/chosen": 0.10151366889476776,
      "rewards/margins": 0.21550293266773224,
      "rewards/rejected": -0.11357422173023224,
      "step": 2455
    },
    {
      "epoch": 0.6333676622039135,
      "grad_norm": 203.0,
      "learning_rate": 1.8331616889804326e-07,
      "logits/chosen": -3.200000047683716,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -307.20001220703125,
      "logps/rejected": -292.79998779296875,
      "loss": 0.5867,
      "rewards/accuracies": 0.71833336353302,
      "rewards/chosen": 0.20148925483226776,
      "rewards/margins": 0.36992186307907104,
      "rewards/rejected": -0.16887207329273224,
      "step": 2460
    },
    {
      "epoch": 0.6346549948506695,
      "grad_norm": 264.0,
      "learning_rate": 1.826725025746653e-07,
      "logits/chosen": -3.246875047683716,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -330.3999938964844,
      "logps/rejected": -320.20001220703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.5800000429153442,
      "rewards/chosen": 0.20505371689796448,
      "rewards/margins": 0.18601074814796448,
      "rewards/rejected": 0.01865234412252903,
      "step": 2465
    },
    {
      "epoch": 0.6359423274974253,
      "grad_norm": 230.0,
      "learning_rate": 1.8202883625128731e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -249.10000610351562,
      "logps/rejected": -241.8000030517578,
      "loss": 0.6332,
      "rewards/accuracies": 0.5596367716789246,
      "rewards/chosen": 0.2612060606479645,
      "rewards/margins": 0.20429687201976776,
      "rewards/rejected": 0.05665283277630806,
      "step": 2470
    },
    {
      "epoch": 0.6372296601441813,
      "grad_norm": 268.0,
      "learning_rate": 1.8138516992790936e-07,
      "logits/chosen": -3.112499952316284,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -301.79998779296875,
      "logps/rejected": -262.79998779296875,
      "loss": 0.5898,
      "rewards/accuracies": 0.6352380514144897,
      "rewards/chosen": 0.16748046875,
      "rewards/margins": 0.3228515684604645,
      "rewards/rejected": -0.15546874701976776,
      "step": 2475
    },
    {
      "epoch": 0.6385169927909372,
      "grad_norm": 406.0,
      "learning_rate": 1.8074150360453143e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -282.20001220703125,
      "loss": 0.6016,
      "rewards/accuracies": 0.6691666841506958,
      "rewards/chosen": 0.19179686903953552,
      "rewards/margins": 0.32343751192092896,
      "rewards/rejected": -0.13242188096046448,
      "step": 2480
    },
    {
      "epoch": 0.6398043254376931,
      "grad_norm": 236.0,
      "learning_rate": 1.8009783728115344e-07,
      "logits/chosen": -3.2874999046325684,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -294.20001220703125,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6102,
      "rewards/accuracies": 0.5817857980728149,
      "rewards/chosen": 0.06982421875,
      "rewards/margins": 0.27519530057907104,
      "rewards/rejected": -0.20463867485523224,
      "step": 2485
    },
    {
      "epoch": 0.641091658084449,
      "grad_norm": 251.0,
      "learning_rate": 1.7945417095777549e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -227.8000030517578,
      "logps/rejected": -242.8000030517578,
      "loss": 0.5781,
      "rewards/accuracies": 0.6945238709449768,
      "rewards/chosen": 0.17210082709789276,
      "rewards/margins": 0.3695312440395355,
      "rewards/rejected": -0.19785156846046448,
      "step": 2490
    },
    {
      "epoch": 0.642378990731205,
      "grad_norm": 230.0,
      "learning_rate": 1.788105046343975e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.034374952316284,
      "logps/chosen": -222.5,
      "logps/rejected": -198.60000610351562,
      "loss": 0.6227,
      "rewards/accuracies": 0.606859028339386,
      "rewards/chosen": 0.18034668266773224,
      "rewards/margins": 0.24726562201976776,
      "rewards/rejected": -0.06657715141773224,
      "step": 2495
    },
    {
      "epoch": 0.6436663233779608,
      "grad_norm": 230.0,
      "learning_rate": 1.7816683831101954e-07,
      "logits/chosen": -3.3125,
      "logits/rejected": -3.296875,
      "logps/chosen": -315.79998779296875,
      "logps/rejected": -280.79998779296875,
      "loss": 0.6312,
      "rewards/accuracies": 0.6403571963310242,
      "rewards/chosen": 0.18137207627296448,
      "rewards/margins": 0.24921874701976776,
      "rewards/rejected": -0.06723632663488388,
      "step": 2500
    },
    {
      "epoch": 0.6449536560247168,
      "grad_norm": 380.0,
      "learning_rate": 1.7752317198764161e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -274.3999938964844,
      "logps/rejected": -219.89999389648438,
      "loss": 0.6508,
      "rewards/accuracies": 0.5783333778381348,
      "rewards/chosen": 0.24665527045726776,
      "rewards/margins": 0.18710938096046448,
      "rewards/rejected": 0.05887451022863388,
      "step": 2505
    },
    {
      "epoch": 0.6462409886714727,
      "grad_norm": 218.0,
      "learning_rate": 1.7687950566426363e-07,
      "logits/chosen": -2.9124999046325684,
      "logits/rejected": -3.0062499046325684,
      "logps/chosen": -251.1999969482422,
      "logps/rejected": -221.85000610351562,
      "loss": 0.6406,
      "rewards/accuracies": 0.659166693687439,
      "rewards/chosen": 0.34033203125,
      "rewards/margins": 0.21083983778953552,
      "rewards/rejected": 0.129669189453125,
      "step": 2510
    },
    {
      "epoch": 0.6475283213182287,
      "grad_norm": 258.0,
      "learning_rate": 1.7623583934088567e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -338.79998779296875,
      "logps/rejected": -247.8000030517578,
      "loss": 0.6508,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.2810302674770355,
      "rewards/margins": 0.15449218451976776,
      "rewards/rejected": 0.12636718153953552,
      "step": 2515
    },
    {
      "epoch": 0.6488156539649845,
      "grad_norm": 236.0,
      "learning_rate": 1.7559217301750771e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -319.79998779296875,
      "logps/rejected": -280.79998779296875,
      "loss": 0.598,
      "rewards/accuracies": 0.6616666913032532,
      "rewards/chosen": 0.2972168028354645,
      "rewards/margins": 0.2869628965854645,
      "rewards/rejected": 0.0091552734375,
      "step": 2520
    },
    {
      "epoch": 0.6501029866117405,
      "grad_norm": 216.0,
      "learning_rate": 1.7494850669412976e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -336.0,
      "logps/rejected": -289.0,
      "loss": 0.6164,
      "rewards/accuracies": 0.6304944753646851,
      "rewards/chosen": 0.23652343451976776,
      "rewards/margins": 0.27167969942092896,
      "rewards/rejected": -0.03535156324505806,
      "step": 2525
    },
    {
      "epoch": 0.6513903192584964,
      "grad_norm": 215.0,
      "learning_rate": 1.743048403707518e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -312.3999938964844,
      "logps/rejected": -279.79998779296875,
      "loss": 0.557,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.20158691704273224,
      "rewards/margins": 0.4222656190395355,
      "rewards/rejected": -0.22043457627296448,
      "step": 2530
    },
    {
      "epoch": 0.6526776519052523,
      "grad_norm": 219.0,
      "learning_rate": 1.7366117404737382e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -3.203125,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -170.8000030517578,
      "loss": 0.5859,
      "rewards/accuracies": 0.6675524711608887,
      "rewards/chosen": 0.3359375,
      "rewards/margins": 0.3246093690395355,
      "rewards/rejected": 0.011608886532485485,
      "step": 2535
    },
    {
      "epoch": 0.6539649845520082,
      "grad_norm": 178.0,
      "learning_rate": 1.7301750772399586e-07,
      "logits/chosen": -3.3531250953674316,
      "logits/rejected": -3.3375000953674316,
      "logps/chosen": -305.3999938964844,
      "logps/rejected": -257.79998779296875,
      "loss": 0.643,
      "rewards/accuracies": 0.6355769038200378,
      "rewards/chosen": 0.0076660155318677425,
      "rewards/margins": 0.23125000298023224,
      "rewards/rejected": -0.22402343153953552,
      "step": 2540
    },
    {
      "epoch": 0.6552523171987642,
      "grad_norm": 356.0,
      "learning_rate": 1.7237384140061793e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -218.8000030517578,
      "logps/rejected": -210.1999969482422,
      "loss": 0.6215,
      "rewards/accuracies": 0.5900000333786011,
      "rewards/chosen": -0.0234375,
      "rewards/margins": 0.22932128608226776,
      "rewards/rejected": -0.25239259004592896,
      "step": 2545
    },
    {
      "epoch": 0.65653964984552,
      "grad_norm": 204.0,
      "learning_rate": 1.7173017507723994e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.325000047683716,
      "logps/chosen": -269.20001220703125,
      "logps/rejected": -239.39999389648438,
      "loss": 0.5953,
      "rewards/accuracies": 0.5846154093742371,
      "rewards/chosen": 0.18593749403953552,
      "rewards/margins": 0.353515625,
      "rewards/rejected": -0.16764526069164276,
      "step": 2550
    },
    {
      "epoch": 0.657826982492276,
      "grad_norm": 684.0,
      "learning_rate": 1.71086508753862e-07,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -2.9593749046325684,
      "logps/chosen": -274.3999938964844,
      "logps/rejected": -224.8000030517578,
      "loss": 0.5492,
      "rewards/accuracies": 0.7120635509490967,
      "rewards/chosen": 0.21660156548023224,
      "rewards/margins": 0.4117187559604645,
      "rewards/rejected": -0.1950603425502777,
      "step": 2555
    },
    {
      "epoch": 0.659114315139032,
      "grad_norm": 286.0,
      "learning_rate": 1.7044284243048403e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.140625,
      "logps/chosen": -224.39999389648438,
      "logps/rejected": -226.5,
      "loss": 0.6984,
      "rewards/accuracies": 0.48059526085853577,
      "rewards/chosen": 0.21708984673023224,
      "rewards/margins": 0.08806762844324112,
      "rewards/rejected": 0.12910155951976776,
      "step": 2560
    },
    {
      "epoch": 0.6604016477857878,
      "grad_norm": 248.0,
      "learning_rate": 1.6979917610710607e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -217.10000610351562,
      "logps/rejected": -245.0,
      "loss": 0.5828,
      "rewards/accuracies": 0.7119048237800598,
      "rewards/chosen": 0.28046876192092896,
      "rewards/margins": 0.3267578184604645,
      "rewards/rejected": -0.04599609225988388,
      "step": 2565
    },
    {
      "epoch": 0.6616889804325438,
      "grad_norm": 250.0,
      "learning_rate": 1.6915550978372812e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -347.6000061035156,
      "logps/rejected": -264.0,
      "loss": 0.6,
      "rewards/accuracies": 0.6480768918991089,
      "rewards/chosen": 0.34941405057907104,
      "rewards/margins": 0.29462891817092896,
      "rewards/rejected": 0.05478515475988388,
      "step": 2570
    },
    {
      "epoch": 0.6629763130792997,
      "grad_norm": 207.0,
      "learning_rate": 1.6851184346035013e-07,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -317.3999938964844,
      "logps/rejected": -232.60000610351562,
      "loss": 0.5578,
      "rewards/accuracies": 0.7400000691413879,
      "rewards/chosen": 0.3421874940395355,
      "rewards/margins": 0.39375001192092896,
      "rewards/rejected": -0.05178222805261612,
      "step": 2575
    },
    {
      "epoch": 0.6642636457260556,
      "grad_norm": 278.0,
      "learning_rate": 1.6786817713697217e-07,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -340.0,
      "logps/rejected": -345.20001220703125,
      "loss": 0.6117,
      "rewards/accuracies": 0.6266666650772095,
      "rewards/chosen": 0.11490478366613388,
      "rewards/margins": 0.28339844942092896,
      "rewards/rejected": -0.16828612983226776,
      "step": 2580
    },
    {
      "epoch": 0.6655509783728115,
      "grad_norm": 235.0,
      "learning_rate": 1.6722451081359424e-07,
      "logits/chosen": -3.3218750953674316,
      "logits/rejected": -3.2750000953674316,
      "logps/chosen": -247.60000610351562,
      "logps/rejected": -264.3999938964844,
      "loss": 0.5617,
      "rewards/accuracies": 0.6866666674613953,
      "rewards/chosen": 0.18039551377296448,
      "rewards/margins": 0.37890625,
      "rewards/rejected": -0.19819335639476776,
      "step": 2585
    },
    {
      "epoch": 0.6668383110195675,
      "grad_norm": 251.0,
      "learning_rate": 1.6658084449021626e-07,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -237.39999389648438,
      "logps/rejected": -194.3000030517578,
      "loss": 0.5621,
      "rewards/accuracies": 0.7234920263290405,
      "rewards/chosen": 0.3407226502895355,
      "rewards/margins": 0.37675780057907104,
      "rewards/rejected": -0.03570556640625,
      "step": 2590
    },
    {
      "epoch": 0.6681256436663234,
      "grad_norm": 254.0,
      "learning_rate": 1.659371781668383e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.2718749046325684,
      "logps/chosen": -264.79998779296875,
      "logps/rejected": -253.3000030517578,
      "loss": 0.5977,
      "rewards/accuracies": 0.6641666889190674,
      "rewards/chosen": 0.24819335341453552,
      "rewards/margins": 0.3275390565395355,
      "rewards/rejected": -0.07949218899011612,
      "step": 2595
    },
    {
      "epoch": 0.6694129763130793,
      "grad_norm": 180.0,
      "learning_rate": 1.6529351184346034e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -313.6000061035156,
      "logps/rejected": -257.3999938964844,
      "loss": 0.5297,
      "rewards/accuracies": 0.7010822296142578,
      "rewards/chosen": 0.2799316346645355,
      "rewards/margins": 0.46171873807907104,
      "rewards/rejected": -0.18212890625,
      "step": 2600
    },
    {
      "epoch": 0.6707003089598352,
      "grad_norm": 258.0,
      "learning_rate": 1.646498455200824e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.109375,
      "logps/chosen": -252.39999389648438,
      "logps/rejected": -249.39999389648438,
      "loss": 0.5891,
      "rewards/accuracies": 0.7012879252433777,
      "rewards/chosen": 0.17333984375,
      "rewards/margins": 0.3031249940395355,
      "rewards/rejected": -0.12966307997703552,
      "step": 2605
    },
    {
      "epoch": 0.6719876416065912,
      "grad_norm": 314.0,
      "learning_rate": 1.6400617919670443e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": NaN,
      "logps/chosen": -278.5,
      "logps/rejected": -209.8000030517578,
      "loss": 0.6344,
      "rewards/accuracies": 0.5876190662384033,
      "rewards/chosen": 0.2730674743652344,
      "rewards/margins": 0.20930175483226776,
      "rewards/rejected": 0.06391601264476776,
      "step": 2610
    },
    {
      "epoch": 0.673274974253347,
      "grad_norm": 165.0,
      "learning_rate": 1.6336251287332645e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -315.3999938964844,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6445,
      "rewards/accuracies": 0.5699999928474426,
      "rewards/chosen": 0.16787108778953552,
      "rewards/margins": 0.24199219048023224,
      "rewards/rejected": -0.07436523586511612,
      "step": 2615
    },
    {
      "epoch": 0.674562306900103,
      "grad_norm": 239.0,
      "learning_rate": 1.627188465499485e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -274.3999938964844,
      "logps/rejected": -293.3999938964844,
      "loss": 0.6188,
      "rewards/accuracies": 0.6496212482452393,
      "rewards/chosen": 0.24277344346046448,
      "rewards/margins": 0.27363282442092896,
      "rewards/rejected": -0.03078613243997097,
      "step": 2620
    },
    {
      "epoch": 0.675849639546859,
      "grad_norm": 242.0,
      "learning_rate": 1.6207518022657056e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -2.7046875953674316,
      "logps/chosen": -217.33749389648438,
      "logps/rejected": -169.8000030517578,
      "loss": 0.6523,
      "rewards/accuracies": 0.5661446452140808,
      "rewards/chosen": 0.3585449159145355,
      "rewards/margins": 0.24335937201976776,
      "rewards/rejected": 0.115966796875,
      "step": 2625
    },
    {
      "epoch": 0.6771369721936148,
      "grad_norm": 206.0,
      "learning_rate": 1.6143151390319257e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.2593750953674316,
      "logps/chosen": -255.1999969482422,
      "logps/rejected": -241.8000030517578,
      "loss": 0.5977,
      "rewards/accuracies": 0.6660256385803223,
      "rewards/chosen": 0.041259765625,
      "rewards/margins": 0.2763671875,
      "rewards/rejected": -0.23496094346046448,
      "step": 2630
    },
    {
      "epoch": 0.6784243048403708,
      "grad_norm": 264.0,
      "learning_rate": 1.6078784757981462e-07,
      "logits/chosen": -3.0374999046325684,
      "logits/rejected": -3.059375047683716,
      "logps/chosen": -228.5,
      "logps/rejected": -178.1999969482422,
      "loss": 0.6227,
      "rewards/accuracies": 0.6184090971946716,
      "rewards/chosen": 0.24082031846046448,
      "rewards/margins": 0.24863281846046448,
      "rewards/rejected": -0.007855224423110485,
      "step": 2635
    },
    {
      "epoch": 0.6797116374871267,
      "grad_norm": 209.0,
      "learning_rate": 1.6014418125643666e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.28125,
      "logps/chosen": -279.6000061035156,
      "logps/rejected": -256.0,
      "loss": 0.5805,
      "rewards/accuracies": 0.6483333706855774,
      "rewards/chosen": 0.39661866426467896,
      "rewards/margins": 0.37910157442092896,
      "rewards/rejected": 0.01806640625,
      "step": 2640
    },
    {
      "epoch": 0.6809989701338826,
      "grad_norm": 728.0,
      "learning_rate": 1.5950051493305868e-07,
      "logits/chosen": -2.971874952316284,
      "logits/rejected": -3.018749952316284,
      "logps/chosen": -200.8000030517578,
      "logps/rejected": -172.3000030517578,
      "loss": 0.6125,
      "rewards/accuracies": 0.6106060743331909,
      "rewards/chosen": 0.3890624940395355,
      "rewards/margins": 0.29218751192092896,
      "rewards/rejected": 0.09775390475988388,
      "step": 2645
    },
    {
      "epoch": 0.6822863027806385,
      "grad_norm": 226.0,
      "learning_rate": 1.5885684860968075e-07,
      "logits/chosen": -3.296875,
      "logits/rejected": -3.3031249046325684,
      "logps/chosen": -281.79998779296875,
      "logps/rejected": -249.39999389648438,
      "loss": 0.6,
      "rewards/accuracies": 0.6274999380111694,
      "rewards/chosen": 0.09735107421875,
      "rewards/margins": 0.28691405057907104,
      "rewards/rejected": -0.18881836533546448,
      "step": 2650
    },
    {
      "epoch": 0.6835736354273945,
      "grad_norm": 235.0,
      "learning_rate": 1.5821318228630276e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -287.20001220703125,
      "logps/rejected": -282.20001220703125,
      "loss": 0.5773,
      "rewards/accuracies": 0.6584523916244507,
      "rewards/chosen": 0.18505859375,
      "rewards/margins": 0.3744140565395355,
      "rewards/rejected": -0.18916015326976776,
      "step": 2655
    },
    {
      "epoch": 0.6848609680741503,
      "grad_norm": 191.0,
      "learning_rate": 1.575695159629248e-07,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.3031249046325684,
      "logps/chosen": -335.6000061035156,
      "logps/rejected": -245.0,
      "loss": 0.5977,
      "rewards/accuracies": 0.6440476179122925,
      "rewards/chosen": 0.06640625,
      "rewards/margins": 0.35624998807907104,
      "rewards/rejected": -0.2901611328125,
      "step": 2660
    },
    {
      "epoch": 0.6861483007209063,
      "grad_norm": 215.0,
      "learning_rate": 1.5692584963954685e-07,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.1875,
      "logps/chosen": -234.6999969482422,
      "logps/rejected": -227.1999969482422,
      "loss": 0.6133,
      "rewards/accuracies": 0.5758333206176758,
      "rewards/chosen": 0.21406249701976776,
      "rewards/margins": 0.2826904356479645,
      "rewards/rejected": -0.06796874850988388,
      "step": 2665
    },
    {
      "epoch": 0.6874356333676622,
      "grad_norm": 215.0,
      "learning_rate": 1.562821833161689e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -269.20001220703125,
      "logps/rejected": -241.1999969482422,
      "loss": 0.6203,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.18359375,
      "rewards/margins": 0.26484376192092896,
      "rewards/rejected": -0.08134765923023224,
      "step": 2670
    },
    {
      "epoch": 0.6887229660144182,
      "grad_norm": 300.0,
      "learning_rate": 1.5563851699279093e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -252.60000610351562,
      "logps/rejected": -238.0,
      "loss": 0.6023,
      "rewards/accuracies": 0.5994445085525513,
      "rewards/chosen": 0.2562499940395355,
      "rewards/margins": 0.2772216796875,
      "rewards/rejected": -0.0205078125,
      "step": 2675
    },
    {
      "epoch": 0.690010298661174,
      "grad_norm": 234.0,
      "learning_rate": 1.5499485066941297e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -315.79998779296875,
      "logps/rejected": -310.3999938964844,
      "loss": 0.5855,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.13344725966453552,
      "rewards/margins": 0.3543945252895355,
      "rewards/rejected": -0.2216796875,
      "step": 2680
    },
    {
      "epoch": 0.69129763130793,
      "grad_norm": 186.0,
      "learning_rate": 1.54351184346035e-07,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -253.60000610351562,
      "logps/rejected": -256.0,
      "loss": 0.609,
      "rewards/accuracies": 0.6637337803840637,
      "rewards/chosen": 0.21083983778953552,
      "rewards/margins": 0.25810545682907104,
      "rewards/rejected": -0.04736328125,
      "step": 2685
    },
    {
      "epoch": 0.6925849639546859,
      "grad_norm": 176.0,
      "learning_rate": 1.5370751802265706e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -300.79998779296875,
      "logps/rejected": -277.20001220703125,
      "loss": 0.5645,
      "rewards/accuracies": 0.6775000691413879,
      "rewards/chosen": 0.24924317002296448,
      "rewards/margins": 0.3763671815395355,
      "rewards/rejected": -0.12722167372703552,
      "step": 2690
    },
    {
      "epoch": 0.6938722966014418,
      "grad_norm": 276.0,
      "learning_rate": 1.5306385169927908e-07,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.125,
      "logps/chosen": -303.79998779296875,
      "logps/rejected": -229.0,
      "loss": 0.6227,
      "rewards/accuracies": 0.6304395794868469,
      "rewards/chosen": 0.24653320014476776,
      "rewards/margins": 0.29951173067092896,
      "rewards/rejected": -0.05361328274011612,
      "step": 2695
    },
    {
      "epoch": 0.6951596292481977,
      "grad_norm": 288.0,
      "learning_rate": 1.5242018537590112e-07,
      "logits/chosen": -2.9906249046325684,
      "logits/rejected": -2.549999952316284,
      "logps/chosen": -255.1999969482422,
      "logps/rejected": -243.1999969482422,
      "loss": 0.6148,
      "rewards/accuracies": 0.5482295751571655,
      "rewards/chosen": 0.3382568359375,
      "rewards/margins": 0.23876953125,
      "rewards/rejected": 0.09953613579273224,
      "step": 2700
    },
    {
      "epoch": 0.6964469618949537,
      "grad_norm": 245.0,
      "learning_rate": 1.5177651905252316e-07,
      "logits/chosen": -3.0875000953674316,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -297.0,
      "logps/rejected": -237.39999389648438,
      "loss": 0.5758,
      "rewards/accuracies": 0.6982142925262451,
      "rewards/chosen": 0.3514648377895355,
      "rewards/margins": 0.36835938692092896,
      "rewards/rejected": -0.016357421875,
      "step": 2705
    },
    {
      "epoch": 0.6977342945417095,
      "grad_norm": 202.0,
      "learning_rate": 1.511328527291452e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -298.3999938964844,
      "logps/rejected": -270.6000061035156,
      "loss": 0.6352,
      "rewards/accuracies": 0.6127289533615112,
      "rewards/chosen": 0.15688475966453552,
      "rewards/margins": 0.24609375,
      "rewards/rejected": -0.08872070163488388,
      "step": 2710
    },
    {
      "epoch": 0.6990216271884655,
      "grad_norm": 204.0,
      "learning_rate": 1.5048918640576725e-07,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.106250047683716,
      "logps/chosen": -282.3999938964844,
      "logps/rejected": -284.20001220703125,
      "loss": 0.5746,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.11391601711511612,
      "rewards/margins": 0.37822264432907104,
      "rewards/rejected": -0.26484376192092896,
      "step": 2715
    },
    {
      "epoch": 0.7003089598352215,
      "grad_norm": 251.0,
      "learning_rate": 1.498455200823893e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -330.79998779296875,
      "logps/rejected": -273.6000061035156,
      "loss": 0.6023,
      "rewards/accuracies": 0.64083331823349,
      "rewards/chosen": 0.359701544046402,
      "rewards/margins": 0.3304687440395355,
      "rewards/rejected": 0.02895507775247097,
      "step": 2720
    },
    {
      "epoch": 0.7015962924819773,
      "grad_norm": 199.0,
      "learning_rate": 1.492018537590113e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.315624952316284,
      "logps/chosen": -301.3999938964844,
      "logps/rejected": -230.0,
      "loss": 0.5969,
      "rewards/accuracies": 0.6633334159851074,
      "rewards/chosen": 0.2804199159145355,
      "rewards/margins": 0.32050782442092896,
      "rewards/rejected": -0.04047851637005806,
      "step": 2725
    },
    {
      "epoch": 0.7028836251287333,
      "grad_norm": 282.0,
      "learning_rate": 1.4855818743563338e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -291.0,
      "logps/rejected": -259.3999938964844,
      "loss": 0.6258,
      "rewards/accuracies": 0.5661172270774841,
      "rewards/chosen": 0.3995117247104645,
      "rewards/margins": 0.24082031846046448,
      "rewards/rejected": 0.15761718153953552,
      "step": 2730
    },
    {
      "epoch": 0.7041709577754892,
      "grad_norm": 278.0,
      "learning_rate": 1.4791452111225542e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -268.3999938964844,
      "logps/rejected": -258.0,
      "loss": 0.6223,
      "rewards/accuracies": 0.5821428894996643,
      "rewards/chosen": 0.25410157442092896,
      "rewards/margins": 0.308837890625,
      "rewards/rejected": -0.05483398586511612,
      "step": 2735
    },
    {
      "epoch": 0.7054582904222451,
      "grad_norm": 264.0,
      "learning_rate": 1.4727085478887743e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -264.79998779296875,
      "logps/rejected": -230.1999969482422,
      "loss": 0.607,
      "rewards/accuracies": 0.6061722040176392,
      "rewards/chosen": 0.239013671875,
      "rewards/margins": 0.31328123807907104,
      "rewards/rejected": -0.07421875,
      "step": 2740
    },
    {
      "epoch": 0.706745623069001,
      "grad_norm": 174.0,
      "learning_rate": 1.4662718846549948e-07,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -268.79998779296875,
      "logps/rejected": -259.6000061035156,
      "loss": 0.6133,
      "rewards/accuracies": 0.651190459728241,
      "rewards/chosen": 0.17734375596046448,
      "rewards/margins": 0.2517456114292145,
      "rewards/rejected": -0.07457885891199112,
      "step": 2745
    },
    {
      "epoch": 0.708032955715757,
      "grad_norm": 336.0,
      "learning_rate": 1.4598352214212152e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -2.953125,
      "logps/chosen": -293.0,
      "logps/rejected": -273.6000061035156,
      "loss": 0.6766,
      "rewards/accuracies": 0.5939285159111023,
      "rewards/chosen": 0.1756591796875,
      "rewards/margins": 0.13037109375,
      "rewards/rejected": 0.04509277269244194,
      "step": 2750
    },
    {
      "epoch": 0.7093202883625128,
      "grad_norm": 191.0,
      "learning_rate": 1.4533985581874356e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -265.3999938964844,
      "logps/rejected": -233.0,
      "loss": 0.6109,
      "rewards/accuracies": 0.6417948603630066,
      "rewards/chosen": 0.3140625059604645,
      "rewards/margins": 0.2682128846645355,
      "rewards/rejected": 0.0460205078125,
      "step": 2755
    },
    {
      "epoch": 0.7106076210092688,
      "grad_norm": 200.0,
      "learning_rate": 1.446961894953656e-07,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -234.8000030517578,
      "logps/rejected": -235.8000030517578,
      "loss": 0.6055,
      "rewards/accuracies": 0.5759344100952148,
      "rewards/chosen": 0.17604979872703552,
      "rewards/margins": 0.28144532442092896,
      "rewards/rejected": -0.10572204738855362,
      "step": 2760
    },
    {
      "epoch": 0.7118949536560247,
      "grad_norm": 632.0,
      "learning_rate": 1.4405252317198762e-07,
      "logits/chosen": -3.081249952316284,
      "logits/rejected": -2.832812547683716,
      "logps/chosen": -202.5500030517578,
      "logps/rejected": -210.3000030517578,
      "loss": 0.6562,
      "rewards/accuracies": 0.5390475988388062,
      "rewards/chosen": 0.15605469048023224,
      "rewards/margins": 0.1555328369140625,
      "rewards/rejected": 0.00048828125,
      "step": 2765
    },
    {
      "epoch": 0.7131822863027807,
      "grad_norm": 316.0,
      "learning_rate": 1.434088568486097e-07,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -304.3999938964844,
      "logps/rejected": -283.3999938964844,
      "loss": 0.6898,
      "rewards/accuracies": 0.5691666603088379,
      "rewards/chosen": 0.16572265326976776,
      "rewards/margins": 0.09526367485523224,
      "rewards/rejected": 0.07053222507238388,
      "step": 2770
    },
    {
      "epoch": 0.7144696189495365,
      "grad_norm": 266.0,
      "learning_rate": 1.4276519052523173e-07,
      "logits/chosen": -3.0531249046325684,
      "logits/rejected": -3.293750047683716,
      "logps/chosen": -215.39999389648438,
      "logps/rejected": -189.89999389648438,
      "loss": 0.7281,
      "rewards/accuracies": 0.5338661670684814,
      "rewards/chosen": 0.08579101413488388,
      "rewards/margins": -0.0034698485396802425,
      "rewards/rejected": 0.089111328125,
      "step": 2775
    },
    {
      "epoch": 0.7157569515962925,
      "grad_norm": 191.0,
      "learning_rate": 1.4212152420185375e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.21875,
      "logps/chosen": -306.79998779296875,
      "logps/rejected": -256.20001220703125,
      "loss": 0.5797,
      "rewards/accuracies": 0.7321428656578064,
      "rewards/chosen": 0.22568359971046448,
      "rewards/margins": 0.35175782442092896,
      "rewards/rejected": -0.12567749619483948,
      "step": 2780
    },
    {
      "epoch": 0.7170442842430484,
      "grad_norm": 236.0,
      "learning_rate": 1.414778578784758e-07,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -2.875,
      "logps/chosen": -220.0,
      "logps/rejected": -250.0,
      "loss": 0.625,
      "rewards/accuracies": 0.6492856740951538,
      "rewards/chosen": 0.18732909858226776,
      "rewards/margins": 0.22934570908546448,
      "rewards/rejected": -0.04165039211511612,
      "step": 2785
    },
    {
      "epoch": 0.7183316168898043,
      "grad_norm": 201.0,
      "learning_rate": 1.408341915550978e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -304.3999938964844,
      "logps/rejected": -281.0,
      "loss": 0.5539,
      "rewards/accuracies": 0.7639743685722351,
      "rewards/chosen": 0.2721191346645355,
      "rewards/margins": 0.3863281309604645,
      "rewards/rejected": -0.11445312201976776,
      "step": 2790
    },
    {
      "epoch": 0.7196189495365602,
      "grad_norm": 255.0,
      "learning_rate": 1.4019052523171988e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -279.0,
      "logps/rejected": -288.79998779296875,
      "loss": 0.5984,
      "rewards/accuracies": 0.605238139629364,
      "rewards/chosen": 0.12509766221046448,
      "rewards/margins": 0.32441407442092896,
      "rewards/rejected": -0.19914551079273224,
      "step": 2795
    },
    {
      "epoch": 0.7209062821833162,
      "grad_norm": 282.0,
      "learning_rate": 1.3954685890834192e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -205.39999389648438,
      "logps/rejected": -223.60000610351562,
      "loss": 0.5922,
      "rewards/accuracies": 0.6561616063117981,
      "rewards/chosen": 0.17661133408546448,
      "rewards/margins": 0.26582032442092896,
      "rewards/rejected": -0.08914794772863388,
      "step": 2800
    },
    {
      "epoch": 0.722193614830072,
      "grad_norm": 240.0,
      "learning_rate": 1.3890319258496394e-07,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -263.20001220703125,
      "logps/rejected": -255.10000610351562,
      "loss": 0.6953,
      "rewards/accuracies": 0.46547621488571167,
      "rewards/chosen": 0.21147461235523224,
      "rewards/margins": 0.06716308742761612,
      "rewards/rejected": 0.14414063096046448,
      "step": 2805
    },
    {
      "epoch": 0.723480947476828,
      "grad_norm": 434.0,
      "learning_rate": 1.3825952626158598e-07,
      "logits/chosen": -3.296875,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -251.39999389648438,
      "logps/rejected": -222.5,
      "loss": 0.6148,
      "rewards/accuracies": 0.6073626279830933,
      "rewards/chosen": 0.28242188692092896,
      "rewards/margins": 0.3050781190395355,
      "rewards/rejected": -0.02276611328125,
      "step": 2810
    },
    {
      "epoch": 0.724768280123584,
      "grad_norm": 442.0,
      "learning_rate": 1.3761585993820805e-07,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -279.0,
      "logps/rejected": -250.39999389648438,
      "loss": 0.6422,
      "rewards/accuracies": 0.5985256433486938,
      "rewards/chosen": 0.29414063692092896,
      "rewards/margins": 0.21284179389476776,
      "rewards/rejected": 0.08149413764476776,
      "step": 2815
    },
    {
      "epoch": 0.7260556127703398,
      "grad_norm": 184.0,
      "learning_rate": 1.3697219361483006e-07,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.003124952316284,
      "logps/chosen": -260.20001220703125,
      "logps/rejected": -235.0,
      "loss": 0.6125,
      "rewards/accuracies": 0.6334524154663086,
      "rewards/chosen": 0.10059203952550888,
      "rewards/margins": 0.2691406309604645,
      "rewards/rejected": -0.16877441108226776,
      "step": 2820
    },
    {
      "epoch": 0.7273429454170958,
      "grad_norm": 172.0,
      "learning_rate": 1.363285272914521e-07,
      "logits/chosen": -3.25,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -293.20001220703125,
      "logps/rejected": -257.6000061035156,
      "loss": 0.5953,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.04213867336511612,
      "rewards/margins": 0.31367188692092896,
      "rewards/rejected": -0.271484375,
      "step": 2825
    },
    {
      "epoch": 0.7286302780638517,
      "grad_norm": 326.0,
      "learning_rate": 1.3568486096807412e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.075000047683716,
      "logps/chosen": -290.79998779296875,
      "logps/rejected": -226.8000030517578,
      "loss": 0.5383,
      "rewards/accuracies": 0.6758334040641785,
      "rewards/chosen": 0.3246093690395355,
      "rewards/margins": 0.447265625,
      "rewards/rejected": -0.12260742485523224,
      "step": 2830
    },
    {
      "epoch": 0.7299176107106076,
      "grad_norm": 296.0,
      "learning_rate": 1.350411946446962e-07,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -272.3999938964844,
      "loss": 0.5848,
      "rewards/accuracies": 0.6433333158493042,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 0.3450683653354645,
      "rewards/rejected": -0.17050781846046448,
      "step": 2835
    },
    {
      "epoch": 0.7312049433573635,
      "grad_norm": 218.0,
      "learning_rate": 1.3439752832131823e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -367.20001220703125,
      "logps/rejected": -296.79998779296875,
      "loss": 0.5461,
      "rewards/accuracies": 0.7595238089561462,
      "rewards/chosen": 0.3204589784145355,
      "rewards/margins": 0.42167967557907104,
      "rewards/rejected": -0.10109863430261612,
      "step": 2840
    },
    {
      "epoch": 0.7324922760041195,
      "grad_norm": 233.0,
      "learning_rate": 1.3375386199794025e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.328125,
      "logps/chosen": -254.0,
      "logps/rejected": -204.8000030517578,
      "loss": 0.6297,
      "rewards/accuracies": 0.5515384674072266,
      "rewards/chosen": 0.16806641221046448,
      "rewards/margins": 0.29364013671875,
      "rewards/rejected": -0.12529297173023224,
      "step": 2845
    },
    {
      "epoch": 0.7337796086508754,
      "grad_norm": 198.0,
      "learning_rate": 1.331101956745623e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -256.3999938964844,
      "logps/rejected": -240.1999969482422,
      "loss": 0.6438,
      "rewards/accuracies": 0.5525457859039307,
      "rewards/chosen": 0.13979491591453552,
      "rewards/margins": 0.18417969346046448,
      "rewards/rejected": -0.04409179836511612,
      "step": 2850
    },
    {
      "epoch": 0.7350669412976313,
      "grad_norm": 193.0,
      "learning_rate": 1.3246652935118436e-07,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.296875,
      "logps/chosen": -280.6000061035156,
      "logps/rejected": -239.0,
      "loss": 0.6059,
      "rewards/accuracies": 0.6311904788017273,
      "rewards/chosen": 0.19482421875,
      "rewards/margins": 0.3238769471645355,
      "rewards/rejected": -0.12871094048023224,
      "step": 2855
    },
    {
      "epoch": 0.7363542739443872,
      "grad_norm": 260.0,
      "learning_rate": 1.3182286302780638e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -236.6999969482422,
      "logps/rejected": -205.39999389648438,
      "loss": 0.6156,
      "rewards/accuracies": 0.6553571820259094,
      "rewards/chosen": 0.09306640923023224,
      "rewards/margins": 0.2531066834926605,
      "rewards/rejected": -0.15966796875,
      "step": 2860
    },
    {
      "epoch": 0.7376416065911432,
      "grad_norm": 262.0,
      "learning_rate": 1.3117919670442842e-07,
      "logits/chosen": -3.0562500953674316,
      "logits/rejected": -3.125,
      "logps/chosen": -212.60000610351562,
      "logps/rejected": -211.39999389648438,
      "loss": 0.6922,
      "rewards/accuracies": 0.5383332967758179,
      "rewards/chosen": 0.09638671576976776,
      "rewards/margins": 0.08613280951976776,
      "rewards/rejected": 0.010668945498764515,
      "step": 2865
    },
    {
      "epoch": 0.738928939237899,
      "grad_norm": 236.0,
      "learning_rate": 1.3053553038105044e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -292.6000061035156,
      "logps/rejected": -266.79998779296875,
      "loss": 0.6375,
      "rewards/accuracies": 0.5954869985580444,
      "rewards/chosen": 0.22034911811351776,
      "rewards/margins": 0.18208007514476776,
      "rewards/rejected": 0.03803711012005806,
      "step": 2870
    },
    {
      "epoch": 0.740216271884655,
      "grad_norm": 212.0,
      "learning_rate": 1.298918640576725e-07,
      "logits/chosen": -3.28125,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -349.3999938964844,
      "logps/rejected": -297.6000061035156,
      "loss": 0.5734,
      "rewards/accuracies": 0.6767857074737549,
      "rewards/chosen": 0.2100830078125,
      "rewards/margins": 0.43085938692092896,
      "rewards/rejected": -0.22148437798023224,
      "step": 2875
    },
    {
      "epoch": 0.741503604531411,
      "grad_norm": 239.0,
      "learning_rate": 1.2924819773429455e-07,
      "logits/chosen": -3.0218749046325684,
      "logits/rejected": -2.96875,
      "logps/chosen": -232.8000030517578,
      "logps/rejected": -185.60000610351562,
      "loss": 0.6266,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.15249022841453552,
      "rewards/margins": 0.21750488877296448,
      "rewards/rejected": -0.06484375149011612,
      "step": 2880
    },
    {
      "epoch": 0.7427909371781668,
      "grad_norm": 218.0,
      "learning_rate": 1.2860453141091657e-07,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -3.0843749046325684,
      "logps/chosen": -233.1999969482422,
      "logps/rejected": -228.60000610351562,
      "loss": 0.6602,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.200439453125,
      "rewards/margins": 0.14768067002296448,
      "rewards/rejected": 0.05263672024011612,
      "step": 2885
    },
    {
      "epoch": 0.7440782698249228,
      "grad_norm": 217.0,
      "learning_rate": 1.279608650875386e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.203125,
      "logps/chosen": -269.3999938964844,
      "logps/rejected": -268.6000061035156,
      "loss": 0.632,
      "rewards/accuracies": 0.5576923489570618,
      "rewards/chosen": 0.17324218153953552,
      "rewards/margins": 0.23203125596046448,
      "rewards/rejected": -0.05832519382238388,
      "step": 2890
    },
    {
      "epoch": 0.7453656024716787,
      "grad_norm": 302.0,
      "learning_rate": 1.2731719876416068e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -358.6000061035156,
      "logps/rejected": -255.0,
      "loss": 0.7805,
      "rewards/accuracies": 0.708492636680603,
      "rewards/chosen": 0.11328125,
      "rewards/margins": 0.13037109375,
      "rewards/rejected": -0.01796874962747097,
      "step": 2895
    },
    {
      "epoch": 0.7466529351184346,
      "grad_norm": 294.0,
      "learning_rate": 1.266735324407827e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -350.3999938964844,
      "loss": 0.6266,
      "rewards/accuracies": 0.5430768728256226,
      "rewards/chosen": 0.09697265923023224,
      "rewards/margins": 0.225341796875,
      "rewards/rejected": -0.128662109375,
      "step": 2900
    },
    {
      "epoch": 0.7479402677651905,
      "grad_norm": 238.0,
      "learning_rate": 1.2602986611740474e-07,
      "logits/chosen": -3.0843749046325684,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -279.6000061035156,
      "logps/rejected": -217.8000030517578,
      "loss": 0.5945,
      "rewards/accuracies": 0.6320878863334656,
      "rewards/chosen": 0.19658203423023224,
      "rewards/margins": 0.3382812440395355,
      "rewards/rejected": -0.1417900025844574,
      "step": 2905
    },
    {
      "epoch": 0.7492276004119465,
      "grad_norm": 274.0,
      "learning_rate": 1.2538619979402675e-07,
      "logits/chosen": -2.971874952316284,
      "logits/rejected": -2.8125,
      "logps/chosen": -219.3000030517578,
      "logps/rejected": -178.60000610351562,
      "loss": 0.5719,
      "rewards/accuracies": 0.6951282024383545,
      "rewards/chosen": 0.41289061307907104,
      "rewards/margins": 0.3414062559604645,
      "rewards/rejected": 0.07182617485523224,
      "step": 2910
    },
    {
      "epoch": 0.7505149330587023,
      "grad_norm": 167.0,
      "learning_rate": 1.247425334706488e-07,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -233.60000610351562,
      "logps/rejected": -231.8000030517578,
      "loss": 0.6203,
      "rewards/accuracies": 0.6271368265151978,
      "rewards/chosen": 0.1759033203125,
      "rewards/margins": 0.24531249701976776,
      "rewards/rejected": -0.06889648735523224,
      "step": 2915
    },
    {
      "epoch": 0.7518022657054583,
      "grad_norm": 249.0,
      "learning_rate": 1.2409886714727084e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -321.79998779296875,
      "logps/rejected": -275.20001220703125,
      "loss": 0.5813,
      "rewards/accuracies": 0.6170996427536011,
      "rewards/chosen": 0.3934326171875,
      "rewards/margins": 0.388671875,
      "rewards/rejected": 0.005175781436264515,
      "step": 2920
    },
    {
      "epoch": 0.7530895983522142,
      "grad_norm": 229.0,
      "learning_rate": 1.2345520082389288e-07,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -256.0,
      "loss": 0.6008,
      "rewards/accuracies": 0.6467949151992798,
      "rewards/chosen": 0.24335937201976776,
      "rewards/margins": 0.28056639432907104,
      "rewards/rejected": -0.037261962890625,
      "step": 2925
    },
    {
      "epoch": 0.7543769309989702,
      "grad_norm": 270.0,
      "learning_rate": 1.2281153450051492e-07,
      "logits/chosen": -3.0531249046325684,
      "logits/rejected": -3.0406250953674316,
      "logps/chosen": -274.0,
      "logps/rejected": -249.8000030517578,
      "loss": 0.6836,
      "rewards/accuracies": 0.4635256826877594,
      "rewards/chosen": 0.2310943603515625,
      "rewards/margins": 0.10380859673023224,
      "rewards/rejected": 0.126953125,
      "step": 2930
    },
    {
      "epoch": 0.755664263645726,
      "grad_norm": 194.0,
      "learning_rate": 1.2216786817713697e-07,
      "logits/chosen": -3.231250047683716,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -263.3999938964844,
      "logps/rejected": -220.6999969482422,
      "loss": 0.5219,
      "rewards/accuracies": 0.8264935612678528,
      "rewards/chosen": 0.27220457792282104,
      "rewards/margins": 0.4351562559604645,
      "rewards/rejected": -0.16396483778953552,
      "step": 2935
    },
    {
      "epoch": 0.756951596292482,
      "grad_norm": 378.0,
      "learning_rate": 1.21524201853759e-07,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -317.0,
      "logps/rejected": -267.20001220703125,
      "loss": 0.6047,
      "rewards/accuracies": 0.6328662633895874,
      "rewards/chosen": 0.37773436307907104,
      "rewards/margins": 0.25654298067092896,
      "rewards/rejected": 0.12082519382238388,
      "step": 2940
    },
    {
      "epoch": 0.7582389289392379,
      "grad_norm": 248.0,
      "learning_rate": 1.2088053553038105e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.065624952316284,
      "logps/chosen": -272.79998779296875,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6492,
      "rewards/accuracies": 0.5709340572357178,
      "rewards/chosen": 0.2664550840854645,
      "rewards/margins": 0.20390625298023224,
      "rewards/rejected": 0.06240234524011612,
      "step": 2945
    },
    {
      "epoch": 0.7595262615859938,
      "grad_norm": 189.0,
      "learning_rate": 1.202368692070031e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -275.20001220703125,
      "logps/rejected": -247.0,
      "loss": 0.5555,
      "rewards/accuracies": 0.7592857480049133,
      "rewards/chosen": 0.19072265923023224,
      "rewards/margins": 0.41015625,
      "rewards/rejected": -0.21953125298023224,
      "step": 2950
    },
    {
      "epoch": 0.7608135942327497,
      "grad_norm": 466.0,
      "learning_rate": 1.195932028836251e-07,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -277.0,
      "logps/rejected": -238.8000030517578,
      "loss": 0.6086,
      "rewards/accuracies": 0.5958333611488342,
      "rewards/chosen": 0.28369140625,
      "rewards/margins": 0.2787109315395355,
      "rewards/rejected": 0.0048828125,
      "step": 2955
    },
    {
      "epoch": 0.7621009268795057,
      "grad_norm": 264.0,
      "learning_rate": 1.1894953656024715e-07,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.0999999046325684,
      "logps/chosen": -254.8000030517578,
      "logps/rejected": -271.0,
      "loss": 0.6469,
      "rewards/accuracies": 0.6781318783760071,
      "rewards/chosen": 0.082275390625,
      "rewards/margins": 0.23458556830883026,
      "rewards/rejected": -0.15239258110523224,
      "step": 2960
    },
    {
      "epoch": 0.7633882595262615,
      "grad_norm": 254.0,
      "learning_rate": 1.1830587023686921e-07,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -265.3999938964844,
      "logps/rejected": -218.10000610351562,
      "loss": 0.5867,
      "rewards/accuracies": 0.6386147737503052,
      "rewards/chosen": 0.4095703065395355,
      "rewards/margins": 0.3047851622104645,
      "rewards/rejected": 0.10488281399011612,
      "step": 2965
    },
    {
      "epoch": 0.7646755921730175,
      "grad_norm": 182.0,
      "learning_rate": 1.1766220391349124e-07,
      "logits/chosen": -2.831249952316284,
      "logits/rejected": -3.028125047683716,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -209.89999389648438,
      "loss": 0.5492,
      "rewards/accuracies": 0.7116667032241821,
      "rewards/chosen": 0.21875,
      "rewards/margins": 0.44316405057907104,
      "rewards/rejected": -0.2236328125,
      "step": 2970
    },
    {
      "epoch": 0.7659629248197735,
      "grad_norm": 262.0,
      "learning_rate": 1.1701853759011328e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -307.6000061035156,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6305,
      "rewards/accuracies": 0.6901981234550476,
      "rewards/chosen": 0.15791015326976776,
      "rewards/margins": 0.22341307997703552,
      "rewards/rejected": -0.06533203274011612,
      "step": 2975
    },
    {
      "epoch": 0.7672502574665293,
      "grad_norm": 165.0,
      "learning_rate": 1.1637487126673531e-07,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -234.3000030517578,
      "loss": 0.5207,
      "rewards/accuracies": 0.815151572227478,
      "rewards/chosen": 0.29707032442092896,
      "rewards/margins": 0.48066407442092896,
      "rewards/rejected": -0.18378905951976776,
      "step": 2980
    },
    {
      "epoch": 0.7685375901132853,
      "grad_norm": 482.0,
      "learning_rate": 1.1573120494335737e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -248.0,
      "logps/rejected": -202.10000610351562,
      "loss": 0.6289,
      "rewards/accuracies": 0.5611110925674438,
      "rewards/chosen": 0.3179687559604645,
      "rewards/margins": 0.23650388419628143,
      "rewards/rejected": 0.08134765923023224,
      "step": 2985
    },
    {
      "epoch": 0.7698249227600412,
      "grad_norm": 360.0,
      "learning_rate": 1.150875386199794e-07,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -267.6000061035156,
      "logps/rejected": -185.89999389648438,
      "loss": 0.5844,
      "rewards/accuracies": 0.6701298952102661,
      "rewards/chosen": 0.47480469942092896,
      "rewards/margins": 0.2953124940395355,
      "rewards/rejected": 0.17937889695167542,
      "step": 2990
    },
    {
      "epoch": 0.7711122554067971,
      "grad_norm": 342.0,
      "learning_rate": 1.1444387229660144e-07,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.171875,
      "logps/chosen": -260.20001220703125,
      "logps/rejected": -239.1999969482422,
      "loss": 0.5945,
      "rewards/accuracies": 0.6660140156745911,
      "rewards/chosen": 0.3111328184604645,
      "rewards/margins": 0.2501220703125,
      "rewards/rejected": 0.06083984300494194,
      "step": 2995
    },
    {
      "epoch": 0.772399588053553,
      "grad_norm": 344.0,
      "learning_rate": 1.1380020597322347e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.140625,
      "logps/chosen": -336.0,
      "logps/rejected": -281.6000061035156,
      "loss": 0.6641,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.3548828065395355,
      "rewards/margins": 0.18623046576976776,
      "rewards/rejected": 0.16855469346046448,
      "step": 3000
    },
    {
      "epoch": 0.773686920700309,
      "grad_norm": 210.0,
      "learning_rate": 1.1315653964984552e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.137500047683716,
      "logps/chosen": -314.0,
      "logps/rejected": -249.60000610351562,
      "loss": 0.5992,
      "rewards/accuracies": 0.6541666984558105,
      "rewards/chosen": 0.29052734375,
      "rewards/margins": 0.31718748807907104,
      "rewards/rejected": -0.0263671875,
      "step": 3005
    },
    {
      "epoch": 0.7749742533470649,
      "grad_norm": 688.0,
      "learning_rate": 1.1251287332646755e-07,
      "logits/chosen": -2.901562452316284,
      "logits/rejected": -2.9000000953674316,
      "logps/chosen": -258.6000061035156,
      "logps/rejected": -232.60000610351562,
      "loss": 0.5492,
      "rewards/accuracies": 0.671818196773529,
      "rewards/chosen": 0.24516601860523224,
      "rewards/margins": 0.4061523377895355,
      "rewards/rejected": -0.16132812201976776,
      "step": 3010
    },
    {
      "epoch": 0.7762615859938208,
      "grad_norm": 1392.0,
      "learning_rate": 1.118692070030896e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -282.79998779296875,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6203,
      "rewards/accuracies": 0.6720085144042969,
      "rewards/chosen": 0.36494141817092896,
      "rewards/margins": 0.28186243772506714,
      "rewards/rejected": 0.08269043266773224,
      "step": 3015
    },
    {
      "epoch": 0.7775489186405767,
      "grad_norm": 170.0,
      "learning_rate": 1.1122554067971163e-07,
      "logits/chosen": -3.3187499046325684,
      "logits/rejected": -3.387500047683716,
      "logps/chosen": -283.6000061035156,
      "logps/rejected": -193.10000610351562,
      "loss": 0.6016,
      "rewards/accuracies": 0.6478021740913391,
      "rewards/chosen": 0.11776123195886612,
      "rewards/margins": 0.32441407442092896,
      "rewards/rejected": -0.20717772841453552,
      "step": 3020
    },
    {
      "epoch": 0.7788362512873327,
      "grad_norm": 388.0,
      "learning_rate": 1.1058187435633368e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -298.6000061035156,
      "logps/rejected": -241.39999389648438,
      "loss": 0.5687,
      "rewards/accuracies": 0.6832051277160645,
      "rewards/chosen": 0.35869139432907104,
      "rewards/margins": 0.3624023497104645,
      "rewards/rejected": -0.0035644532181322575,
      "step": 3025
    },
    {
      "epoch": 0.7801235839340885,
      "grad_norm": 242.0,
      "learning_rate": 1.0993820803295571e-07,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.090625047683716,
      "logps/chosen": -293.79998779296875,
      "logps/rejected": -266.20001220703125,
      "loss": 0.6062,
      "rewards/accuracies": 0.6395238041877747,
      "rewards/chosen": 0.19243164360523224,
      "rewards/margins": 0.29765623807907104,
      "rewards/rejected": -0.10419921576976776,
      "step": 3030
    },
    {
      "epoch": 0.7814109165808445,
      "grad_norm": 264.0,
      "learning_rate": 1.0929454170957775e-07,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.203125,
      "logps/chosen": -258.29998779296875,
      "logps/rejected": -205.5749969482422,
      "loss": 0.5801,
      "rewards/accuracies": 0.70333331823349,
      "rewards/chosen": 0.08554687350988388,
      "rewards/margins": 0.36005860567092896,
      "rewards/rejected": -0.27490234375,
      "step": 3035
    },
    {
      "epoch": 0.7826982492276005,
      "grad_norm": 173.0,
      "learning_rate": 1.0865087538619978e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -233.0,
      "logps/rejected": -263.6000061035156,
      "loss": 0.5977,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": 0.10458984225988388,
      "rewards/margins": 0.3110595643520355,
      "rewards/rejected": -0.20624999701976776,
      "step": 3040
    },
    {
      "epoch": 0.7839855818743563,
      "grad_norm": 414.0,
      "learning_rate": 1.0800720906282184e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -261.5,
      "logps/rejected": -281.79998779296875,
      "loss": 0.6766,
      "rewards/accuracies": 0.5652777552604675,
      "rewards/chosen": 0.15537109971046448,
      "rewards/margins": 0.14877930283546448,
      "rewards/rejected": 0.006542968563735485,
      "step": 3045
    },
    {
      "epoch": 0.7852729145211123,
      "grad_norm": 13376.0,
      "learning_rate": 1.0736354273944387e-07,
      "logits/chosen": -3.121875047683716,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -214.10000610351562,
      "logps/rejected": -212.3000030517578,
      "loss": 1.0437,
      "rewards/accuracies": 0.5251947641372681,
      "rewards/chosen": 0.22480468451976776,
      "rewards/margins": -0.21682128310203552,
      "rewards/rejected": 0.44271546602249146,
      "step": 3050
    },
    {
      "epoch": 0.7865602471678682,
      "grad_norm": 246.0,
      "learning_rate": 1.067198764160659e-07,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.096874952316284,
      "logps/chosen": -299.3999938964844,
      "logps/rejected": -308.6000061035156,
      "loss": 0.6395,
      "rewards/accuracies": 0.5061172246932983,
      "rewards/chosen": 0.3182128965854645,
      "rewards/margins": 0.23369140923023224,
      "rewards/rejected": 0.08448486030101776,
      "step": 3055
    },
    {
      "epoch": 0.787847579814624,
      "grad_norm": 239.0,
      "learning_rate": 1.0607621009268794e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -267.3999938964844,
      "loss": 0.5574,
      "rewards/accuracies": 0.7157509326934814,
      "rewards/chosen": 0.3030761778354645,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -0.12343750149011612,
      "step": 3060
    },
    {
      "epoch": 0.78913491246138,
      "grad_norm": 230.0,
      "learning_rate": 1.0543254376930998e-07,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.106250047683716,
      "logps/chosen": -254.1999969482422,
      "logps/rejected": -238.0,
      "loss": 0.6109,
      "rewards/accuracies": 0.6700000166893005,
      "rewards/chosen": 0.16958007216453552,
      "rewards/margins": 0.24912109971046448,
      "rewards/rejected": -0.07929687201976776,
      "step": 3065
    },
    {
      "epoch": 0.790422245108136,
      "grad_norm": 406.0,
      "learning_rate": 1.0478887744593203e-07,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.0625,
      "logps/chosen": -274.20001220703125,
      "logps/rejected": -208.8000030517578,
      "loss": 0.6297,
      "rewards/accuracies": 0.5657575726509094,
      "rewards/chosen": 0.4417968690395355,
      "rewards/margins": 0.24716797471046448,
      "rewards/rejected": 0.19374999403953552,
      "step": 3070
    },
    {
      "epoch": 0.7917095777548918,
      "grad_norm": 241.0,
      "learning_rate": 1.0414521112255406e-07,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.046875,
      "logps/chosen": -255.60000610351562,
      "logps/rejected": -222.3000030517578,
      "loss": 0.6125,
      "rewards/accuracies": 0.5921717882156372,
      "rewards/chosen": 0.2655273377895355,
      "rewards/margins": 0.26806640625,
      "rewards/rejected": -0.0025390624068677425,
      "step": 3075
    },
    {
      "epoch": 0.7929969104016478,
      "grad_norm": 262.0,
      "learning_rate": 1.035015447991761e-07,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.0218749046325684,
      "logps/chosen": -232.60000610351562,
      "logps/rejected": -242.8000030517578,
      "loss": 0.6117,
      "rewards/accuracies": 0.5684524178504944,
      "rewards/chosen": 0.22111816704273224,
      "rewards/margins": 0.2368011474609375,
      "rewards/rejected": -0.015332031063735485,
      "step": 3080
    },
    {
      "epoch": 0.7942842430484037,
      "grad_norm": 193.0,
      "learning_rate": 1.0285787847579814e-07,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.25,
      "logps/chosen": -294.0,
      "logps/rejected": -263.20001220703125,
      "loss": 0.6609,
      "rewards/accuracies": 0.5141667127609253,
      "rewards/chosen": 0.23710937798023224,
      "rewards/margins": 0.16555175185203552,
      "rewards/rejected": 0.07187499850988388,
      "step": 3085
    },
    {
      "epoch": 0.7955715756951597,
      "grad_norm": 217.0,
      "learning_rate": 1.0221421215242018e-07,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.2906250953674316,
      "logps/chosen": -342.79998779296875,
      "logps/rejected": -294.0,
      "loss": 0.5656,
      "rewards/accuracies": 0.6491667032241821,
      "rewards/chosen": 0.14960937201976776,
      "rewards/margins": 0.3882812559604645,
      "rewards/rejected": -0.23916015028953552,
      "step": 3090
    },
    {
      "epoch": 0.7968589083419155,
      "grad_norm": 224.0,
      "learning_rate": 1.0157054582904221e-07,
      "logits/chosen": -3.0718750953674316,
      "logits/rejected": -2.887500047683716,
      "logps/chosen": -164.25,
      "logps/rejected": -157.1875,
      "loss": 0.5926,
      "rewards/accuracies": 0.576785683631897,
      "rewards/chosen": 0.2530273497104645,
      "rewards/margins": 0.27233582735061646,
      "rewards/rejected": -0.01894531212747097,
      "step": 3095
    },
    {
      "epoch": 0.7981462409886715,
      "grad_norm": 208.0,
      "learning_rate": 1.0092687950566426e-07,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -246.0,
      "logps/rejected": -213.10000610351562,
      "loss": 0.5379,
      "rewards/accuracies": 0.7465476393699646,
      "rewards/chosen": 0.14284667372703552,
      "rewards/margins": 0.40214842557907104,
      "rewards/rejected": -0.2587890625,
      "step": 3100
    },
    {
      "epoch": 0.7994335736354274,
      "grad_norm": 332.0,
      "learning_rate": 1.002832131822863e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.7718749046325684,
      "logps/chosen": -263.6000061035156,
      "logps/rejected": -242.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.5383332967758179,
      "rewards/chosen": 0.2789062559604645,
      "rewards/margins": 0.19228515028953552,
      "rewards/rejected": 0.08647461235523224,
      "step": 3105
    },
    {
      "epoch": 0.8007209062821833,
      "grad_norm": 185.0,
      "learning_rate": 9.963954685890834e-08,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -275.6000061035156,
      "logps/rejected": -268.20001220703125,
      "loss": 0.5742,
      "rewards/accuracies": 0.6988095641136169,
      "rewards/chosen": 0.230224609375,
      "rewards/margins": 0.34003907442092896,
      "rewards/rejected": -0.10979004204273224,
      "step": 3110
    },
    {
      "epoch": 0.8020082389289392,
      "grad_norm": 376.0,
      "learning_rate": 9.899588053553037e-08,
      "logits/chosen": -2.9749999046325684,
      "logits/rejected": -2.9906249046325684,
      "logps/chosen": -235.1999969482422,
      "logps/rejected": -210.10000610351562,
      "loss": 0.6453,
      "rewards/accuracies": 0.5826190710067749,
      "rewards/chosen": 0.09555663913488388,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": -0.0814208984375,
      "step": 3115
    },
    {
      "epoch": 0.8032955715756952,
      "grad_norm": 196.0,
      "learning_rate": 9.835221421215241e-08,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -240.1999969482422,
      "logps/rejected": -247.0,
      "loss": 0.6219,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.17849120497703552,
      "rewards/margins": 0.22529296576976776,
      "rewards/rejected": -0.04658203199505806,
      "step": 3120
    },
    {
      "epoch": 0.804582904222451,
      "grad_norm": 233.0,
      "learning_rate": 9.770854788877446e-08,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.012500047683716,
      "logps/chosen": -283.6000061035156,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6008,
      "rewards/accuracies": 0.7211905717849731,
      "rewards/chosen": 0.10703124850988388,
      "rewards/margins": 0.3340820372104645,
      "rewards/rejected": -0.227294921875,
      "step": 3125
    },
    {
      "epoch": 0.805870236869207,
      "grad_norm": 192.0,
      "learning_rate": 9.70648815653965e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.065624952316284,
      "logps/chosen": -338.0,
      "logps/rejected": -290.6000061035156,
      "loss": 0.6004,
      "rewards/accuracies": 0.5866667032241821,
      "rewards/chosen": 0.28828126192092896,
      "rewards/margins": 0.34160155057907104,
      "rewards/rejected": -0.05332031100988388,
      "step": 3130
    },
    {
      "epoch": 0.807157569515963,
      "grad_norm": 264.0,
      "learning_rate": 9.642121524201853e-08,
      "logits/chosen": -3.1812500953674316,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6477,
      "rewards/accuracies": 0.679413914680481,
      "rewards/chosen": 0.15234375,
      "rewards/margins": 0.22596435248851776,
      "rewards/rejected": -0.07343749701976776,
      "step": 3135
    },
    {
      "epoch": 0.8084449021627188,
      "grad_norm": 404.0,
      "learning_rate": 9.577754891864057e-08,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.1968750953674316,
      "logps/chosen": -272.6000061035156,
      "logps/rejected": -252.39999389648438,
      "loss": 0.682,
      "rewards/accuracies": 0.4937121272087097,
      "rewards/chosen": 0.029296875,
      "rewards/margins": 0.11757812649011612,
      "rewards/rejected": -0.08740234375,
      "step": 3140
    },
    {
      "epoch": 0.8097322348094748,
      "grad_norm": 251.0,
      "learning_rate": 9.513388259526261e-08,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.0687499046325684,
      "logps/chosen": -231.1999969482422,
      "logps/rejected": -196.8000030517578,
      "loss": 0.6469,
      "rewards/accuracies": 0.6284066438674927,
      "rewards/chosen": 0.07724609225988388,
      "rewards/margins": 0.17849120497703552,
      "rewards/rejected": -0.10117797553539276,
      "step": 3145
    },
    {
      "epoch": 0.8110195674562307,
      "grad_norm": 219.0,
      "learning_rate": 9.449021627188466e-08,
      "logits/chosen": -3.065624952316284,
      "logits/rejected": -2.984375,
      "logps/chosen": -291.20001220703125,
      "logps/rejected": -213.60000610351562,
      "loss": 0.6148,
      "rewards/accuracies": 0.5915043950080872,
      "rewards/chosen": 0.20068359375,
      "rewards/margins": 0.26298826932907104,
      "rewards/rejected": -0.06269530951976776,
      "step": 3150
    },
    {
      "epoch": 0.8123069001029866,
      "grad_norm": 258.0,
      "learning_rate": 9.384654994850669e-08,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -3.171875,
      "logps/chosen": -236.39999389648438,
      "logps/rejected": -213.60000610351562,
      "loss": 0.5781,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.23825684189796448,
      "rewards/margins": 0.3609375059604645,
      "rewards/rejected": -0.12221679836511612,
      "step": 3155
    },
    {
      "epoch": 0.8135942327497425,
      "grad_norm": 227.0,
      "learning_rate": 9.320288362512873e-08,
      "logits/chosen": -3.3125,
      "logits/rejected": -3.268749952316284,
      "logps/chosen": -251.0,
      "logps/rejected": -276.79998779296875,
      "loss": 0.6109,
      "rewards/accuracies": 0.623452365398407,
      "rewards/chosen": 0.23769530653953552,
      "rewards/margins": 0.2955078184604645,
      "rewards/rejected": -0.05826415866613388,
      "step": 3160
    },
    {
      "epoch": 0.8148815653964985,
      "grad_norm": 231.0,
      "learning_rate": 9.255921730175077e-08,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -256.6000061035156,
      "logps/rejected": -242.0,
      "loss": 0.6305,
      "rewards/accuracies": 0.6394230723381042,
      "rewards/chosen": 0.03666992112994194,
      "rewards/margins": 0.20156249403953552,
      "rewards/rejected": -0.16464844346046448,
      "step": 3165
    },
    {
      "epoch": 0.8161688980432544,
      "grad_norm": 204.0,
      "learning_rate": 9.191555097837281e-08,
      "logits/chosen": -3.265625,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -263.0,
      "logps/rejected": -196.60000610351562,
      "loss": 0.6312,
      "rewards/accuracies": 0.5944805145263672,
      "rewards/chosen": 0.20170898735523224,
      "rewards/margins": 0.23808594048023224,
      "rewards/rejected": -0.0361328125,
      "step": 3170
    },
    {
      "epoch": 0.8174562306900103,
      "grad_norm": 194.0,
      "learning_rate": 9.127188465499484e-08,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.096874952316284,
      "logps/chosen": -327.0,
      "logps/rejected": -218.8000030517578,
      "loss": 0.5617,
      "rewards/accuracies": 0.6938096284866333,
      "rewards/chosen": 0.4222656190395355,
      "rewards/margins": 0.4068359434604645,
      "rewards/rejected": 0.014758300967514515,
      "step": 3175
    },
    {
      "epoch": 0.8187435633367662,
      "grad_norm": 326.0,
      "learning_rate": 9.062821833161689e-08,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -263.6000061035156,
      "loss": 0.6617,
      "rewards/accuracies": 0.5432808995246887,
      "rewards/chosen": 0.29707032442092896,
      "rewards/margins": 0.19013671576976776,
      "rewards/rejected": 0.10786132514476776,
      "step": 3180
    },
    {
      "epoch": 0.8200308959835222,
      "grad_norm": 516.0,
      "learning_rate": 8.998455200823893e-08,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -273.3999938964844,
      "logps/rejected": -204.60000610351562,
      "loss": 0.5781,
      "rewards/accuracies": 0.6349999904632568,
      "rewards/chosen": 0.18183593451976776,
      "rewards/margins": 0.36152344942092896,
      "rewards/rejected": -0.17885741591453552,
      "step": 3185
    },
    {
      "epoch": 0.821318228630278,
      "grad_norm": 251.0,
      "learning_rate": 8.934088568486097e-08,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.315624952316284,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -231.60000610351562,
      "loss": 0.6852,
      "rewards/accuracies": 0.4824908673763275,
      "rewards/chosen": 0.22285155951976776,
      "rewards/margins": 0.12148437649011612,
      "rewards/rejected": 0.10102538764476776,
      "step": 3190
    },
    {
      "epoch": 0.822605561277034,
      "grad_norm": 232.0,
      "learning_rate": 8.8697219361483e-08,
      "logits/chosen": -3.1031250953674316,
      "logits/rejected": -3.2093749046325684,
      "logps/chosen": -281.6000061035156,
      "logps/rejected": -237.89999389648438,
      "loss": 0.6102,
      "rewards/accuracies": 0.6133333444595337,
      "rewards/chosen": 0.20161132514476776,
      "rewards/margins": 0.28227537870407104,
      "rewards/rejected": -0.08110351860523224,
      "step": 3195
    },
    {
      "epoch": 0.82389289392379,
      "grad_norm": 318.0,
      "learning_rate": 8.805355303810503e-08,
      "logits/chosen": -3.0562500953674316,
      "logits/rejected": -3.1187500953674316,
      "logps/chosen": -295.20001220703125,
      "logps/rejected": -280.1000061035156,
      "loss": 0.6086,
      "rewards/accuracies": 0.6222222447395325,
      "rewards/chosen": 0.3173828125,
      "rewards/margins": 0.30097657442092896,
      "rewards/rejected": 0.01677856408059597,
      "step": 3200
    },
    {
      "epoch": 0.8251802265705458,
      "grad_norm": 240.0,
      "learning_rate": 8.740988671472709e-08,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -246.1999969482422,
      "logps/rejected": -237.39999389648438,
      "loss": 0.657,
      "rewards/accuracies": 0.5191666483879089,
      "rewards/chosen": 0.14101561903953552,
      "rewards/margins": 0.17911987006664276,
      "rewards/rejected": -0.03837890550494194,
      "step": 3205
    },
    {
      "epoch": 0.8264675592173018,
      "grad_norm": 225.0,
      "learning_rate": 8.676622039134912e-08,
      "logits/chosen": -3.293750047683716,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -359.0,
      "logps/rejected": -301.6000061035156,
      "loss": 0.5891,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": 0.2511230409145355,
      "rewards/margins": 0.3268981873989105,
      "rewards/rejected": -0.07611083984375,
      "step": 3210
    },
    {
      "epoch": 0.8277548918640577,
      "grad_norm": 221.0,
      "learning_rate": 8.612255406797116e-08,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.096874952316284,
      "logps/chosen": -288.0,
      "logps/rejected": -239.89999389648438,
      "loss": 0.5703,
      "rewards/accuracies": 0.6794871687889099,
      "rewards/chosen": 0.24697265028953552,
      "rewards/margins": 0.38593751192092896,
      "rewards/rejected": -0.1396484375,
      "step": 3215
    },
    {
      "epoch": 0.8290422245108136,
      "grad_norm": 203.0,
      "learning_rate": 8.547888774459319e-08,
      "logits/chosen": -2.996875047683716,
      "logits/rejected": -2.9625000953674316,
      "logps/chosen": -234.39999389648438,
      "logps/rejected": -192.8000030517578,
      "loss": 0.6297,
      "rewards/accuracies": 0.5562987327575684,
      "rewards/chosen": 0.17460937798023224,
      "rewards/margins": 0.19033202528953552,
      "rewards/rejected": -0.01567382737994194,
      "step": 3220
    },
    {
      "epoch": 0.8303295571575695,
      "grad_norm": 231.0,
      "learning_rate": 8.483522142121524e-08,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.1937499046325684,
      "logps/chosen": -294.3999938964844,
      "logps/rejected": -255.60000610351562,
      "loss": 0.5848,
      "rewards/accuracies": 0.6467857360839844,
      "rewards/chosen": 0.18919678032398224,
      "rewards/margins": 0.32421875,
      "rewards/rejected": -0.13496093451976776,
      "step": 3225
    },
    {
      "epoch": 0.8316168898043255,
      "grad_norm": 211.0,
      "learning_rate": 8.419155509783727e-08,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -299.6000061035156,
      "logps/rejected": -268.3999938964844,
      "loss": 0.5625,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": 0.3277831971645355,
      "rewards/margins": 0.38671875,
      "rewards/rejected": -0.05908203125,
      "step": 3230
    },
    {
      "epoch": 0.8329042224510813,
      "grad_norm": 188.0,
      "learning_rate": 8.354788877445932e-08,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -307.3999938964844,
      "logps/rejected": -264.20001220703125,
      "loss": 0.6117,
      "rewards/accuracies": 0.6870238184928894,
      "rewards/chosen": 0.13243408501148224,
      "rewards/margins": 0.24082031846046448,
      "rewards/rejected": -0.10859374701976776,
      "step": 3235
    },
    {
      "epoch": 0.8341915550978373,
      "grad_norm": 162.0,
      "learning_rate": 8.290422245108136e-08,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.2750000953674316,
      "logps/chosen": -252.60000610351562,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6023,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": 0.2881835997104645,
      "rewards/margins": 0.291259765625,
      "rewards/rejected": -0.0023437500931322575,
      "step": 3240
    },
    {
      "epoch": 0.8354788877445932,
      "grad_norm": 216.0,
      "learning_rate": 8.22605561277034e-08,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.140625,
      "logps/chosen": -258.3999938964844,
      "logps/rejected": -256.6000061035156,
      "loss": 0.6031,
      "rewards/accuracies": 0.5835497975349426,
      "rewards/chosen": 0.22792968153953552,
      "rewards/margins": 0.27714842557907104,
      "rewards/rejected": -0.04899902269244194,
      "step": 3245
    },
    {
      "epoch": 0.8367662203913491,
      "grad_norm": 274.0,
      "learning_rate": 8.161688980432543e-08,
      "logits/chosen": -3.278125047683716,
      "logits/rejected": -3.3125,
      "logps/chosen": -295.79998779296875,
      "logps/rejected": -218.1999969482422,
      "loss": 0.5789,
      "rewards/accuracies": 0.7264102697372437,
      "rewards/chosen": 0.18183593451976776,
      "rewards/margins": 0.37285155057907104,
      "rewards/rejected": -0.19091796875,
      "step": 3250
    },
    {
      "epoch": 0.838053553038105,
      "grad_norm": 250.0,
      "learning_rate": 8.097322348094747e-08,
      "logits/chosen": -3.268749952316284,
      "logits/rejected": -2.627734422683716,
      "logps/chosen": -219.1750030517578,
      "logps/rejected": -217.8000030517578,
      "loss": 0.6359,
      "rewards/accuracies": 0.6584615707397461,
      "rewards/chosen": 0.20423583686351776,
      "rewards/margins": 0.22666016221046448,
      "rewards/rejected": -0.02211914025247097,
      "step": 3255
    },
    {
      "epoch": 0.839340885684861,
      "grad_norm": 232.0,
      "learning_rate": 8.032955715756952e-08,
      "logits/chosen": -3.1968750953674316,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -281.3999938964844,
      "logps/rejected": -311.3999938964844,
      "loss": 0.657,
      "rewards/accuracies": 0.576923131942749,
      "rewards/chosen": 0.20292969048023224,
      "rewards/margins": 0.18706054985523224,
      "rewards/rejected": 0.01596679724752903,
      "step": 3260
    },
    {
      "epoch": 0.8406282183316169,
      "grad_norm": 215.0,
      "learning_rate": 7.968589083419156e-08,
      "logits/chosen": -3.3031249046325684,
      "logits/rejected": -3.265625,
      "logps/chosen": -303.79998779296875,
      "logps/rejected": -263.3999938964844,
      "loss": 0.5953,
      "rewards/accuracies": 0.6699999570846558,
      "rewards/chosen": 0.08562012016773224,
      "rewards/margins": 0.3101562559604645,
      "rewards/rejected": -0.224609375,
      "step": 3265
    },
    {
      "epoch": 0.8419155509783728,
      "grad_norm": 254.0,
      "learning_rate": 7.904222451081359e-08,
      "logits/chosen": -3.003124952316284,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -312.6000061035156,
      "logps/rejected": -251.8000030517578,
      "loss": 0.6125,
      "rewards/accuracies": 0.6442424058914185,
      "rewards/chosen": 0.3506835997104645,
      "rewards/margins": 0.3115234375,
      "rewards/rejected": 0.038330078125,
      "step": 3270
    },
    {
      "epoch": 0.8432028836251287,
      "grad_norm": 224.0,
      "learning_rate": 7.839855818743563e-08,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -281.79998779296875,
      "logps/rejected": -223.5,
      "loss": 0.5656,
      "rewards/accuracies": 0.7294048070907593,
      "rewards/chosen": 0.3167968690395355,
      "rewards/margins": 0.38261717557907104,
      "rewards/rejected": -0.06561279296875,
      "step": 3275
    },
    {
      "epoch": 0.8444902162718847,
      "grad_norm": 205.0,
      "learning_rate": 7.775489186405767e-08,
      "logits/chosen": -2.9781250953674316,
      "logits/rejected": -3.018749952316284,
      "logps/chosen": -274.8999938964844,
      "logps/rejected": -226.60000610351562,
      "loss": 0.6305,
      "rewards/accuracies": 0.5991666913032532,
      "rewards/chosen": 0.24238280951976776,
      "rewards/margins": 0.23896484076976776,
      "rewards/rejected": 0.0029296875,
      "step": 3280
    },
    {
      "epoch": 0.8457775489186405,
      "grad_norm": 211.0,
      "learning_rate": 7.711122554067972e-08,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.143749952316284,
      "logps/chosen": -258.8999938964844,
      "logps/rejected": -261.3999938964844,
      "loss": 0.6602,
      "rewards/accuracies": 0.6017857789993286,
      "rewards/chosen": 0.16318359971046448,
      "rewards/margins": 0.16871337592601776,
      "rewards/rejected": -0.005847168155014515,
      "step": 3285
    },
    {
      "epoch": 0.8470648815653965,
      "grad_norm": 249.0,
      "learning_rate": 7.646755921730175e-08,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -303.6000061035156,
      "logps/rejected": -304.70001220703125,
      "loss": 0.6297,
      "rewards/accuracies": 0.5732142925262451,
      "rewards/chosen": 0.22802734375,
      "rewards/margins": 0.23765869438648224,
      "rewards/rejected": -0.009765625,
      "step": 3290
    },
    {
      "epoch": 0.8483522142121525,
      "grad_norm": 196.0,
      "learning_rate": 7.582389289392379e-08,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.309375047683716,
      "logps/chosen": -332.79998779296875,
      "logps/rejected": -325.6000061035156,
      "loss": 0.5215,
      "rewards/accuracies": 0.7725000381469727,
      "rewards/chosen": 0.27617186307907104,
      "rewards/margins": 0.508593738079071,
      "rewards/rejected": -0.23271484673023224,
      "step": 3295
    },
    {
      "epoch": 0.8496395468589083,
      "grad_norm": 268.0,
      "learning_rate": 7.518022657054583e-08,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -2.9937500953674316,
      "logps/chosen": -287.0,
      "logps/rejected": -250.1999969482422,
      "loss": 0.6398,
      "rewards/accuracies": 0.6048809885978699,
      "rewards/chosen": 0.2682861387729645,
      "rewards/margins": 0.21826171875,
      "rewards/rejected": 0.05002441257238388,
      "step": 3300
    },
    {
      "epoch": 0.8509268795056643,
      "grad_norm": 536.0,
      "learning_rate": 7.453656024716787e-08,
      "logits/chosen": -3.296875,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -310.79998779296875,
      "logps/rejected": -301.79998779296875,
      "loss": 0.657,
      "rewards/accuracies": 0.565833330154419,
      "rewards/chosen": 0.34868162870407104,
      "rewards/margins": 0.17753906548023224,
      "rewards/rejected": 0.17099609971046448,
      "step": 3305
    },
    {
      "epoch": 0.8522142121524202,
      "grad_norm": 338.0,
      "learning_rate": 7.38928939237899e-08,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.190624952316284,
      "logps/chosen": -338.20001220703125,
      "logps/rejected": -269.29998779296875,
      "loss": 0.5754,
      "rewards/accuracies": 0.6638095378875732,
      "rewards/chosen": 0.3662109375,
      "rewards/margins": 0.3714843690395355,
      "rewards/rejected": -0.0044921874068677425,
      "step": 3310
    },
    {
      "epoch": 0.8535015447991761,
      "grad_norm": 191.0,
      "learning_rate": 7.324922760041195e-08,
      "logits/chosen": -3.1656250953674316,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -295.20001220703125,
      "logps/rejected": -302.79998779296875,
      "loss": 0.6055,
      "rewards/accuracies": 0.6614802479743958,
      "rewards/chosen": 0.3568359315395355,
      "rewards/margins": 0.29863280057907104,
      "rewards/rejected": 0.05812988430261612,
      "step": 3315
    },
    {
      "epoch": 0.854788877445932,
      "grad_norm": 233.0,
      "learning_rate": 7.260556127703399e-08,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.1875,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -241.39999389648438,
      "loss": 0.5992,
      "rewards/accuracies": 0.596666693687439,
      "rewards/chosen": 0.129638671875,
      "rewards/margins": 0.2984375059604645,
      "rewards/rejected": -0.1689453125,
      "step": 3320
    },
    {
      "epoch": 0.856076210092688,
      "grad_norm": 236.0,
      "learning_rate": 7.196189495365603e-08,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -296.20001220703125,
      "logps/rejected": -279.20001220703125,
      "loss": 0.6203,
      "rewards/accuracies": 0.5719047784805298,
      "rewards/chosen": 0.06796874850988388,
      "rewards/margins": 0.2982421815395355,
      "rewards/rejected": -0.23007813096046448,
      "step": 3325
    },
    {
      "epoch": 0.8573635427394438,
      "grad_norm": 209.0,
      "learning_rate": 7.131822863027806e-08,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -341.6000061035156,
      "logps/rejected": -290.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.6035714149475098,
      "rewards/chosen": 0.135009765625,
      "rewards/margins": 0.2718749940395355,
      "rewards/rejected": -0.13671875,
      "step": 3330
    },
    {
      "epoch": 0.8586508753861998,
      "grad_norm": 178.0,
      "learning_rate": 7.067456230690009e-08,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.174999952316284,
      "logps/chosen": -223.89999389648438,
      "logps/rejected": -202.4499969482422,
      "loss": 0.5664,
      "rewards/accuracies": 0.6825000047683716,
      "rewards/chosen": 0.15278320014476776,
      "rewards/margins": 0.3589843809604645,
      "rewards/rejected": -0.20600585639476776,
      "step": 3335
    },
    {
      "epoch": 0.8599382080329557,
      "grad_norm": 454.0,
      "learning_rate": 7.003089598352215e-08,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.206249952316284,
      "logps/chosen": -263.20001220703125,
      "logps/rejected": -233.60000610351562,
      "loss": 0.632,
      "rewards/accuracies": 0.6107392907142639,
      "rewards/chosen": 0.2603515684604645,
      "rewards/margins": 0.22724609076976776,
      "rewards/rejected": 0.03300781175494194,
      "step": 3340
    },
    {
      "epoch": 0.8612255406797117,
      "grad_norm": 213.0,
      "learning_rate": 6.938722966014417e-08,
      "logits/chosen": -3.215625047683716,
      "logits/rejected": -3.299999952316284,
      "logps/chosen": -269.6000061035156,
      "logps/rejected": -221.1999969482422,
      "loss": 0.5906,
      "rewards/accuracies": 0.7131136059761047,
      "rewards/chosen": 0.13642577826976776,
      "rewards/margins": 0.3128906190395355,
      "rewards/rejected": -0.17714843153953552,
      "step": 3345
    },
    {
      "epoch": 0.8625128733264675,
      "grad_norm": 300.0,
      "learning_rate": 6.874356333676622e-08,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.090625047683716,
      "logps/chosen": -325.20001220703125,
      "logps/rejected": -282.0,
      "loss": 0.6219,
      "rewards/accuracies": 0.5849999785423279,
      "rewards/chosen": 0.2177734375,
      "rewards/margins": 0.2874999940395355,
      "rewards/rejected": -0.0697021484375,
      "step": 3350
    },
    {
      "epoch": 0.8638002059732235,
      "grad_norm": 214.0,
      "learning_rate": 6.809989701338825e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.168750047683716,
      "logps/chosen": -304.79998779296875,
      "logps/rejected": -289.79998779296875,
      "loss": 0.6172,
      "rewards/accuracies": 0.596666693687439,
      "rewards/chosen": 0.12565918266773224,
      "rewards/margins": 0.28242188692092896,
      "rewards/rejected": -0.15673828125,
      "step": 3355
    },
    {
      "epoch": 0.8650875386199794,
      "grad_norm": 230.0,
      "learning_rate": 6.74562306900103e-08,
      "logits/chosen": NaN,
      "logits/rejected": -2.7593750953674316,
      "logps/chosen": -271.6000061035156,
      "logps/rejected": -234.5,
      "loss": 0.5988,
      "rewards/accuracies": 0.5566450357437134,
      "rewards/chosen": 0.23515625298023224,
      "rewards/margins": 0.27019041776657104,
      "rewards/rejected": -0.03502197191119194,
      "step": 3360
    },
    {
      "epoch": 0.8663748712667353,
      "grad_norm": 214.0,
      "learning_rate": 6.681256436663233e-08,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.1031250953674316,
      "logps/chosen": -284.6000061035156,
      "logps/rejected": -250.39999389648438,
      "loss": 0.5203,
      "rewards/accuracies": 0.7978030443191528,
      "rewards/chosen": 0.32207030057907104,
      "rewards/margins": 0.49492186307907104,
      "rewards/rejected": -0.17268066108226776,
      "step": 3365
    },
    {
      "epoch": 0.8676622039134912,
      "grad_norm": 370.0,
      "learning_rate": 6.616889804325438e-08,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -292.29998779296875,
      "logps/rejected": -208.64999389648438,
      "loss": 0.5852,
      "rewards/accuracies": 0.6412087678909302,
      "rewards/chosen": 0.3709960877895355,
      "rewards/margins": 0.341796875,
      "rewards/rejected": 0.02863769605755806,
      "step": 3370
    },
    {
      "epoch": 0.8689495365602472,
      "grad_norm": 220.0,
      "learning_rate": 6.55252317198764e-08,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -274.20001220703125,
      "logps/rejected": -239.0,
      "loss": 0.6211,
      "rewards/accuracies": 0.6385531425476074,
      "rewards/chosen": 0.18740233778953552,
      "rewards/margins": 0.24123534560203552,
      "rewards/rejected": -0.05413818359375,
      "step": 3375
    },
    {
      "epoch": 0.870236869207003,
      "grad_norm": 232.0,
      "learning_rate": 6.488156539649846e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -296.6000061035156,
      "logps/rejected": -222.1999969482422,
      "loss": 0.6031,
      "rewards/accuracies": 0.590238094329834,
      "rewards/chosen": 0.283203125,
      "rewards/margins": 0.28447264432907104,
      "rewards/rejected": -0.0011230468517169356,
      "step": 3380
    },
    {
      "epoch": 0.871524201853759,
      "grad_norm": 163.0,
      "learning_rate": 6.423789907312049e-08,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.128124952316284,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -257.79998779296875,
      "loss": 0.6195,
      "rewards/accuracies": 0.7007617950439453,
      "rewards/chosen": 0.08039550483226776,
      "rewards/margins": 0.24111327528953552,
      "rewards/rejected": -0.16059570014476776,
      "step": 3385
    },
    {
      "epoch": 0.872811534500515,
      "grad_norm": 222.0,
      "learning_rate": 6.359423274974253e-08,
      "logits/chosen": -3.0562500953674316,
      "logits/rejected": -3.2125000953674316,
      "logps/chosen": -276.79998779296875,
      "logps/rejected": -230.39999389648438,
      "loss": 0.5691,
      "rewards/accuracies": 0.7250640988349915,
      "rewards/chosen": 0.12944336235523224,
      "rewards/margins": 0.3566650450229645,
      "rewards/rejected": -0.22749023139476776,
      "step": 3390
    },
    {
      "epoch": 0.8740988671472708,
      "grad_norm": 241.0,
      "learning_rate": 6.295056642636456e-08,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.125,
      "logps/chosen": -373.6000061035156,
      "logps/rejected": -299.0,
      "loss": 0.5852,
      "rewards/accuracies": 0.6377472877502441,
      "rewards/chosen": 0.4556640684604645,
      "rewards/margins": 0.31074219942092896,
      "rewards/rejected": 0.14535827934741974,
      "step": 3395
    },
    {
      "epoch": 0.8753861997940268,
      "grad_norm": 205.0,
      "learning_rate": 6.23069001029866e-08,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -2.642578125,
      "logps/chosen": -262.20001220703125,
      "logps/rejected": -234.39999389648438,
      "loss": 0.7516,
      "rewards/accuracies": 0.6241666674613953,
      "rewards/chosen": 0.15620116889476776,
      "rewards/margins": 0.1026611328125,
      "rewards/rejected": 0.05351562425494194,
      "step": 3400
    },
    {
      "epoch": 0.8766735324407827,
      "grad_norm": 288.0,
      "learning_rate": 6.166323377960865e-08,
      "logits/chosen": -3.075000047683716,
      "logits/rejected": -3.109375,
      "logps/chosen": -351.3999938964844,
      "logps/rejected": -304.6000061035156,
      "loss": 0.5906,
      "rewards/accuracies": 0.6791666746139526,
      "rewards/chosen": 0.285888671875,
      "rewards/margins": 0.3427734375,
      "rewards/rejected": -0.05644531175494194,
      "step": 3405
    },
    {
      "epoch": 0.8779608650875386,
      "grad_norm": 193.0,
      "learning_rate": 6.101956745623069e-08,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.328125,
      "logps/chosen": -304.79998779296875,
      "logps/rejected": -257.70001220703125,
      "loss": 0.5855,
      "rewards/accuracies": 0.6550000309944153,
      "rewards/chosen": 0.15927734971046448,
      "rewards/margins": 0.3941406309604645,
      "rewards/rejected": -0.23520508408546448,
      "step": 3410
    },
    {
      "epoch": 0.8792481977342945,
      "grad_norm": 330.0,
      "learning_rate": 6.037590113285273e-08,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -2.8531250953674316,
      "logps/chosen": -204.5,
      "logps/rejected": -226.1999969482422,
      "loss": 0.6086,
      "rewards/accuracies": 0.6843223571777344,
      "rewards/chosen": 0.27392578125,
      "rewards/margins": 0.31328123807907104,
      "rewards/rejected": -0.03925781324505806,
      "step": 3415
    },
    {
      "epoch": 0.8805355303810505,
      "grad_norm": 772.0,
      "learning_rate": 5.973223480947476e-08,
      "logits/chosen": -2.8687500953674316,
      "logits/rejected": -2.598437547683716,
      "logps/chosen": -234.8000030517578,
      "logps/rejected": -224.8000030517578,
      "loss": 0.643,
      "rewards/accuracies": 0.5811966061592102,
      "rewards/chosen": 0.2943359315395355,
      "rewards/margins": 0.15361328423023224,
      "rewards/rejected": 0.14089354872703552,
      "step": 3420
    },
    {
      "epoch": 0.8818228630278064,
      "grad_norm": 210.0,
      "learning_rate": 5.9088568486096805e-08,
      "logits/chosen": -3.0875000953674316,
      "logits/rejected": -3.0250000953674316,
      "logps/chosen": -325.20001220703125,
      "logps/rejected": -231.0,
      "loss": 0.5953,
      "rewards/accuracies": 0.6646212339401245,
      "rewards/chosen": 0.30488282442092896,
      "rewards/margins": 0.302734375,
      "rewards/rejected": 0.0022705078590661287,
      "step": 3425
    },
    {
      "epoch": 0.8831101956745623,
      "grad_norm": 174.0,
      "learning_rate": 5.844490216271884e-08,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -325.3999938964844,
      "logps/rejected": -291.0,
      "loss": 0.5359,
      "rewards/accuracies": 0.7498534917831421,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 0.4449218809604645,
      "rewards/rejected": -0.16708984971046448,
      "step": 3430
    },
    {
      "epoch": 0.8843975283213182,
      "grad_norm": 300.0,
      "learning_rate": 5.7801235839340884e-08,
      "logits/chosen": -2.762500047683716,
      "logits/rejected": -2.9515624046325684,
      "logps/chosen": -217.8000030517578,
      "logps/rejected": -169.1999969482422,
      "loss": 0.6195,
      "rewards/accuracies": 0.5838095545768738,
      "rewards/chosen": 0.4378906190395355,
      "rewards/margins": 0.2640624940395355,
      "rewards/rejected": 0.17441406846046448,
      "step": 3435
    },
    {
      "epoch": 0.8856848609680742,
      "grad_norm": 243.0,
      "learning_rate": 5.715756951596292e-08,
      "logits/chosen": -3.1156249046325684,
      "logits/rejected": -3.1781249046325684,
      "logps/chosen": -248.39999389648438,
      "logps/rejected": -254.39999389648438,
      "loss": 0.5813,
      "rewards/accuracies": 0.6055555939674377,
      "rewards/chosen": 0.2699218690395355,
      "rewards/margins": 0.35136717557907104,
      "rewards/rejected": -0.08161620795726776,
      "step": 3440
    },
    {
      "epoch": 0.88697219361483,
      "grad_norm": 202.0,
      "learning_rate": 5.651390319258496e-08,
      "logits/chosen": -2.9906249046325684,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -250.1999969482422,
      "logps/rejected": -280.6000061035156,
      "loss": 0.5742,
      "rewards/accuracies": 0.6660714745521545,
      "rewards/chosen": 0.36015623807907104,
      "rewards/margins": 0.3681640625,
      "rewards/rejected": -0.007641601376235485,
      "step": 3445
    },
    {
      "epoch": 0.888259526261586,
      "grad_norm": 224.0,
      "learning_rate": 5.5870236869207e-08,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.2281250953674316,
      "logps/chosen": -265.3999938964844,
      "logps/rejected": -177.6999969482422,
      "loss": 0.5512,
      "rewards/accuracies": 0.7477778196334839,
      "rewards/chosen": 0.3373046815395355,
      "rewards/margins": 0.38789063692092896,
      "rewards/rejected": -0.05073242262005806,
      "step": 3450
    },
    {
      "epoch": 0.889546858908342,
      "grad_norm": 210.0,
      "learning_rate": 5.522657054582904e-08,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.234375,
      "logps/chosen": -268.79998779296875,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6508,
      "rewards/accuracies": 0.6035714149475098,
      "rewards/chosen": 0.07784423977136612,
      "rewards/margins": 0.15727539360523224,
      "rewards/rejected": -0.07929687201976776,
      "step": 3455
    },
    {
      "epoch": 0.8908341915550978,
      "grad_norm": 235.0,
      "learning_rate": 5.458290422245108e-08,
      "logits/chosen": -3.1781249046325684,
      "logits/rejected": -3.215625047683716,
      "logps/chosen": -313.20001220703125,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6297,
      "rewards/accuracies": 0.6428571939468384,
      "rewards/chosen": 0.14921875298023224,
      "rewards/margins": 0.22041015326976776,
      "rewards/rejected": -0.07109375298023224,
      "step": 3460
    },
    {
      "epoch": 0.8921215242018538,
      "grad_norm": 334.0,
      "learning_rate": 5.393923789907312e-08,
      "logits/chosen": -3.2406249046325684,
      "logits/rejected": -3.125,
      "logps/chosen": -202.0,
      "logps/rejected": -222.60000610351562,
      "loss": 0.6469,
      "rewards/accuracies": 0.5457051992416382,
      "rewards/chosen": 0.23906250298023224,
      "rewards/margins": 0.24657592177391052,
      "rewards/rejected": -0.007861328311264515,
      "step": 3465
    },
    {
      "epoch": 0.8934088568486097,
      "grad_norm": 222.0,
      "learning_rate": 5.3295571575695156e-08,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.159374952316284,
      "logps/chosen": -244.1999969482422,
      "logps/rejected": -219.0,
      "loss": 0.5957,
      "rewards/accuracies": 0.6475000381469727,
      "rewards/chosen": 0.18940429389476776,
      "rewards/margins": 0.30107420682907104,
      "rewards/rejected": -0.11152343451976776,
      "step": 3470
    },
    {
      "epoch": 0.8946961894953656,
      "grad_norm": 308.0,
      "learning_rate": 5.26519052523172e-08,
      "logits/chosen": -3.168750047683716,
      "logits/rejected": -3.078125,
      "logps/chosen": -302.0,
      "logps/rejected": -281.6000061035156,
      "loss": 0.5547,
      "rewards/accuracies": 0.631794810295105,
      "rewards/chosen": 0.3700805604457855,
      "rewards/margins": 0.3958984315395355,
      "rewards/rejected": -0.02626953087747097,
      "step": 3475
    },
    {
      "epoch": 0.8959835221421215,
      "grad_norm": 286.0,
      "learning_rate": 5.2008238928939235e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.0250000953674316,
      "logps/chosen": -296.79998779296875,
      "logps/rejected": -286.79998779296875,
      "loss": 0.6102,
      "rewards/accuracies": 0.6378968358039856,
      "rewards/chosen": 0.33050537109375,
      "rewards/margins": 0.32255858182907104,
      "rewards/rejected": 0.00795898400247097,
      "step": 3480
    },
    {
      "epoch": 0.8972708547888775,
      "grad_norm": 203.0,
      "learning_rate": 5.136457260556128e-08,
      "logits/chosen": -2.996875047683716,
      "logits/rejected": -3.0562500953674316,
      "logps/chosen": -255.8000030517578,
      "logps/rejected": -240.60000610351562,
      "loss": 0.5961,
      "rewards/accuracies": 0.6924999952316284,
      "rewards/chosen": 0.23725585639476776,
      "rewards/margins": 0.3597656190395355,
      "rewards/rejected": -0.12226562201976776,
      "step": 3485
    },
    {
      "epoch": 0.8985581874356333,
      "grad_norm": 270.0,
      "learning_rate": 5.0720906282183313e-08,
      "logits/chosen": -3.0406250953674316,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -284.6000061035156,
      "logps/rejected": -288.79998779296875,
      "loss": 0.5742,
      "rewards/accuracies": 0.678611159324646,
      "rewards/chosen": 0.09154357761144638,
      "rewards/margins": 0.39375001192092896,
      "rewards/rejected": -0.3021484315395355,
      "step": 3490
    },
    {
      "epoch": 0.8998455200823893,
      "grad_norm": 240.0,
      "learning_rate": 5.0077239958805356e-08,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -340.79998779296875,
      "logps/rejected": -257.3999938964844,
      "loss": 0.6055,
      "rewards/accuracies": 0.5900000333786011,
      "rewards/chosen": 0.388671875,
      "rewards/margins": 0.33232420682907104,
      "rewards/rejected": 0.05659179762005806,
      "step": 3495
    },
    {
      "epoch": 0.9011328527291452,
      "grad_norm": 264.0,
      "learning_rate": 4.943357363542739e-08,
      "logits/chosen": -3.203125,
      "logits/rejected": -3.1156249046325684,
      "logps/chosen": -288.70001220703125,
      "logps/rejected": -234.39999389648438,
      "loss": 0.6016,
      "rewards/accuracies": 0.6208333373069763,
      "rewards/chosen": 0.15810546278953552,
      "rewards/margins": 0.29864501953125,
      "rewards/rejected": -0.139892578125,
      "step": 3500
    },
    {
      "epoch": 0.9024201853759012,
      "grad_norm": 223.0,
      "learning_rate": 4.8789907312049435e-08,
      "logits/chosen": -3.2562499046325684,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -332.79998779296875,
      "logps/rejected": -298.0,
      "loss": 0.5738,
      "rewards/accuracies": 0.6710256338119507,
      "rewards/chosen": 0.221923828125,
      "rewards/margins": 0.38847655057907104,
      "rewards/rejected": -0.16667480766773224,
      "step": 3505
    },
    {
      "epoch": 0.903707518022657,
      "grad_norm": 223.0,
      "learning_rate": 4.814624098867147e-08,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.231250047683716,
      "logps/chosen": -259.5,
      "logps/rejected": -231.6999969482422,
      "loss": 0.6141,
      "rewards/accuracies": 0.6720238327980042,
      "rewards/chosen": 0.353515625,
      "rewards/margins": 0.30512696504592896,
      "rewards/rejected": 0.04816894605755806,
      "step": 3510
    },
    {
      "epoch": 0.904994850669413,
      "grad_norm": 288.0,
      "learning_rate": 4.7502574665293514e-08,
      "logits/chosen": -3.234375,
      "logits/rejected": -3.3343749046325684,
      "logps/chosen": -348.0,
      "logps/rejected": -290.6000061035156,
      "loss": 0.6078,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": 0.08984375,
      "rewards/margins": 0.30878907442092896,
      "rewards/rejected": -0.21914061903953552,
      "step": 3515
    },
    {
      "epoch": 0.9062821833161689,
      "grad_norm": 300.0,
      "learning_rate": 4.685890834191555e-08,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -280.79998779296875,
      "logps/rejected": -262.79998779296875,
      "loss": 0.5762,
      "rewards/accuracies": 0.7180952429771423,
      "rewards/chosen": 0.16191406548023224,
      "rewards/margins": 0.4022460877895355,
      "rewards/rejected": -0.240234375,
      "step": 3520
    },
    {
      "epoch": 0.9075695159629248,
      "grad_norm": 238.0,
      "learning_rate": 4.621524201853759e-08,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.265625,
      "logps/chosen": -282.5,
      "logps/rejected": -285.79998779296875,
      "loss": 0.6102,
      "rewards/accuracies": 0.6038095355033875,
      "rewards/chosen": 0.19160155951976776,
      "rewards/margins": 0.27070313692092896,
      "rewards/rejected": -0.07968749850988388,
      "step": 3525
    },
    {
      "epoch": 0.9088568486096807,
      "grad_norm": 200.0,
      "learning_rate": 4.557157569515963e-08,
      "logits/chosen": -3.184375047683716,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -308.20001220703125,
      "logps/rejected": -269.79998779296875,
      "loss": 0.632,
      "rewards/accuracies": 0.6006410717964172,
      "rewards/chosen": 0.17695312201976776,
      "rewards/margins": 0.2275390625,
      "rewards/rejected": -0.05051269382238388,
      "step": 3530
    },
    {
      "epoch": 0.9101441812564367,
      "grad_norm": 228.0,
      "learning_rate": 4.492790937178167e-08,
      "logits/chosen": -3.3062500953674316,
      "logits/rejected": -3.3031249046325684,
      "logps/chosen": -225.1999969482422,
      "logps/rejected": -234.8000030517578,
      "loss": 0.6195,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": 0.0026367187965661287,
      "rewards/margins": 0.2865234315395355,
      "rewards/rejected": -0.28349608182907104,
      "step": 3535
    },
    {
      "epoch": 0.9114315139031925,
      "grad_norm": 172.0,
      "learning_rate": 4.42842430484037e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.090625047683716,
      "logps/chosen": -293.79998779296875,
      "logps/rejected": -256.0,
      "loss": 0.5918,
      "rewards/accuracies": 0.64000004529953,
      "rewards/chosen": 0.2728515565395355,
      "rewards/margins": 0.3277343809604645,
      "rewards/rejected": -0.05549316480755806,
      "step": 3540
    },
    {
      "epoch": 0.9127188465499485,
      "grad_norm": 242.0,
      "learning_rate": 4.364057672502574e-08,
      "logits/chosen": -3.143749952316284,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -318.0,
      "loss": 0.5992,
      "rewards/accuracies": 0.6590477228164673,
      "rewards/chosen": 0.27556151151657104,
      "rewards/margins": 0.31757813692092896,
      "rewards/rejected": -0.0416259765625,
      "step": 3545
    },
    {
      "epoch": 0.9140061791967045,
      "grad_norm": 197.0,
      "learning_rate": 4.299691040164778e-08,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -3.153125047683716,
      "logps/chosen": -305.79998779296875,
      "logps/rejected": -303.0,
      "loss": 0.643,
      "rewards/accuracies": 0.6449999809265137,
      "rewards/chosen": 0.196533203125,
      "rewards/margins": 0.18593749403953552,
      "rewards/rejected": 0.01025390625,
      "step": 3550
    },
    {
      "epoch": 0.9152935118434603,
      "grad_norm": 229.0,
      "learning_rate": 4.235324407826982e-08,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -271.79998779296875,
      "loss": 0.5953,
      "rewards/accuracies": 0.633928656578064,
      "rewards/chosen": 0.19451904296875,
      "rewards/margins": 0.29374998807907104,
      "rewards/rejected": -0.09868164360523224,
      "step": 3555
    },
    {
      "epoch": 0.9165808444902163,
      "grad_norm": 274.0,
      "learning_rate": 4.170957775489186e-08,
      "logits/chosen": -3.1624999046325684,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -307.20001220703125,
      "logps/rejected": -247.60000610351562,
      "loss": 0.6031,
      "rewards/accuracies": 0.6035714149475098,
      "rewards/chosen": 0.12895508110523224,
      "rewards/margins": 0.324951171875,
      "rewards/rejected": -0.19599609076976776,
      "step": 3560
    },
    {
      "epoch": 0.9178681771369722,
      "grad_norm": 844.0,
      "learning_rate": 4.10659114315139e-08,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -2.890625,
      "logps/chosen": -260.6000061035156,
      "logps/rejected": -264.1000061035156,
      "loss": 0.6086,
      "rewards/accuracies": 0.6538095474243164,
      "rewards/chosen": 0.3667968809604645,
      "rewards/margins": 0.2899414002895355,
      "rewards/rejected": 0.07656250149011612,
      "step": 3565
    },
    {
      "epoch": 0.9191555097837281,
      "grad_norm": 428.0,
      "learning_rate": 4.042224510813594e-08,
      "logits/chosen": -3.0062499046325684,
      "logits/rejected": -2.7750000953674316,
      "logps/chosen": -268.0,
      "logps/rejected": -237.72500610351562,
      "loss": 0.5859,
      "rewards/accuracies": 0.645961582660675,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 0.3036132752895355,
      "rewards/rejected": -0.010498046875,
      "step": 3570
    },
    {
      "epoch": 0.920442842430484,
      "grad_norm": 268.0,
      "learning_rate": 3.977857878475798e-08,
      "logits/chosen": -3.153125047683716,
      "logits/rejected": -3.1624999046325684,
      "logps/chosen": -287.20001220703125,
      "logps/rejected": -205.8000030517578,
      "loss": 0.5711,
      "rewards/accuracies": 0.6647494435310364,
      "rewards/chosen": 0.1304931640625,
      "rewards/margins": 0.3672851622104645,
      "rewards/rejected": -0.236328125,
      "step": 3575
    },
    {
      "epoch": 0.92173017507724,
      "grad_norm": 264.0,
      "learning_rate": 3.9134912461380015e-08,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.21875,
      "logps/chosen": -267.20001220703125,
      "logps/rejected": -265.6000061035156,
      "loss": 0.6031,
      "rewards/accuracies": 0.6333683729171753,
      "rewards/chosen": 0.2787109315395355,
      "rewards/margins": 0.2837890684604645,
      "rewards/rejected": -0.004687500186264515,
      "step": 3580
    },
    {
      "epoch": 0.9230175077239959,
      "grad_norm": 272.0,
      "learning_rate": 3.849124613800206e-08,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.3125,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -227.6999969482422,
      "loss": 0.582,
      "rewards/accuracies": 0.65974360704422,
      "rewards/chosen": 0.2744140625,
      "rewards/margins": 0.33906251192092896,
      "rewards/rejected": -0.06562499701976776,
      "step": 3585
    },
    {
      "epoch": 0.9243048403707518,
      "grad_norm": 246.0,
      "learning_rate": 3.7847579814624094e-08,
      "logits/chosen": -3.1500000953674316,
      "logits/rejected": -2.901562452316284,
      "logps/chosen": -243.1999969482422,
      "logps/rejected": -213.8000030517578,
      "loss": 0.582,
      "rewards/accuracies": 0.5704761743545532,
      "rewards/chosen": 0.38818359375,
      "rewards/margins": 0.34843748807907104,
      "rewards/rejected": 0.04035644605755806,
      "step": 3590
    },
    {
      "epoch": 0.9255921730175077,
      "grad_norm": 211.0,
      "learning_rate": 3.720391349124614e-08,
      "logits/chosen": -3.2249999046325684,
      "logits/rejected": -3.0999999046325684,
      "logps/chosen": -250.8000030517578,
      "logps/rejected": -258.0,
      "loss": 0.5855,
      "rewards/accuracies": 0.6183333396911621,
      "rewards/chosen": 0.19951172173023224,
      "rewards/margins": 0.34648436307907104,
      "rewards/rejected": -0.14724120497703552,
      "step": 3595
    },
    {
      "epoch": 0.9268795056642637,
      "grad_norm": 192.0,
      "learning_rate": 3.656024716786817e-08,
      "logits/chosen": -3.140625,
      "logits/rejected": -3.0999999046325684,
      "logps/chosen": -240.60000610351562,
      "logps/rejected": -257.3999938964844,
      "loss": 0.5184,
      "rewards/accuracies": 0.7806746363639832,
      "rewards/chosen": 0.30244141817092896,
      "rewards/margins": 0.682812511920929,
      "rewards/rejected": -0.37968748807907104,
      "step": 3600
    },
    {
      "epoch": 0.9281668383110195,
      "grad_norm": 206.0,
      "learning_rate": 3.5916580844490216e-08,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -2.926562547683716,
      "logps/chosen": -258.20001220703125,
      "logps/rejected": -251.39999389648438,
      "loss": 0.6031,
      "rewards/accuracies": 0.6727380752563477,
      "rewards/chosen": 0.15434570610523224,
      "rewards/margins": 0.2767578065395355,
      "rewards/rejected": -0.12246093899011612,
      "step": 3605
    },
    {
      "epoch": 0.9294541709577755,
      "grad_norm": 272.0,
      "learning_rate": 3.527291452111225e-08,
      "logits/chosen": -3.21875,
      "logits/rejected": -3.234375,
      "logps/chosen": -288.3999938964844,
      "logps/rejected": -239.1999969482422,
      "loss": 0.618,
      "rewards/accuracies": 0.6047619581222534,
      "rewards/chosen": 0.05286865308880806,
      "rewards/margins": 0.26337891817092896,
      "rewards/rejected": -0.21049804985523224,
      "step": 3610
    },
    {
      "epoch": 0.9307415036045315,
      "grad_norm": 218.0,
      "learning_rate": 3.4629248197734294e-08,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.203125,
      "logps/chosen": -342.79998779296875,
      "logps/rejected": -298.6000061035156,
      "loss": 0.5977,
      "rewards/accuracies": 0.6428571939468384,
      "rewards/chosen": 0.3217529356479645,
      "rewards/margins": 0.33686524629592896,
      "rewards/rejected": -0.01513671875,
      "step": 3615
    },
    {
      "epoch": 0.9320288362512873,
      "grad_norm": 312.0,
      "learning_rate": 3.398558187435633e-08,
      "logits/chosen": -3.237499952316284,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -241.60000610351562,
      "logps/rejected": -198.8000030517578,
      "loss": 0.5703,
      "rewards/accuracies": 0.6737121343612671,
      "rewards/chosen": 0.2800976634025574,
      "rewards/margins": 0.35957032442092896,
      "rewards/rejected": -0.07919921725988388,
      "step": 3620
    },
    {
      "epoch": 0.9333161688980433,
      "grad_norm": 225.0,
      "learning_rate": 3.334191555097837e-08,
      "logits/chosen": -3.109375,
      "logits/rejected": -3.121875047683716,
      "logps/chosen": -272.6000061035156,
      "logps/rejected": -222.1999969482422,
      "loss": 0.5609,
      "rewards/accuracies": 0.6705050468444824,
      "rewards/chosen": 0.22978515923023224,
      "rewards/margins": 0.3833984434604645,
      "rewards/rejected": -0.15410156548023224,
      "step": 3625
    },
    {
      "epoch": 0.9346035015447992,
      "grad_norm": 209.0,
      "learning_rate": 3.269824922760041e-08,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.065624952316284,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -301.0,
      "loss": 0.6453,
      "rewards/accuracies": 0.6114219427108765,
      "rewards/chosen": 0.2701171934604645,
      "rewards/margins": 0.2333984375,
      "rewards/rejected": 0.0372314453125,
      "step": 3630
    },
    {
      "epoch": 0.935890834191555,
      "grad_norm": 213.0,
      "learning_rate": 3.205458290422245e-08,
      "logits/chosen": -3.1343750953674316,
      "logits/rejected": -3.1500000953674316,
      "logps/chosen": -275.0,
      "logps/rejected": -245.60000610351562,
      "loss": 0.6273,
      "rewards/accuracies": 0.6171079874038696,
      "rewards/chosen": 0.16708984971046448,
      "rewards/margins": 0.22434692084789276,
      "rewards/rejected": -0.057373046875,
      "step": 3635
    },
    {
      "epoch": 0.937178166838311,
      "grad_norm": 219.0,
      "learning_rate": 3.141091658084449e-08,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -282.0,
      "logps/rejected": -241.1999969482422,
      "loss": 0.5465,
      "rewards/accuracies": 0.7633334398269653,
      "rewards/chosen": 0.4800781309604645,
      "rewards/margins": 0.44707030057907104,
      "rewards/rejected": 0.03291015699505806,
      "step": 3640
    },
    {
      "epoch": 0.938465499485067,
      "grad_norm": 239.0,
      "learning_rate": 3.076725025746653e-08,
      "logits/chosen": -3.2593750953674316,
      "logits/rejected": -3.262500047683716,
      "logps/chosen": -356.79998779296875,
      "logps/rejected": -272.0,
      "loss": 0.6543,
      "rewards/accuracies": 0.4895237982273102,
      "rewards/chosen": 0.22275391221046448,
      "rewards/margins": 0.19882813096046448,
      "rewards/rejected": 0.02377929724752903,
      "step": 3645
    },
    {
      "epoch": 0.9397528321318228,
      "grad_norm": 334.0,
      "learning_rate": 3.0123583934088567e-08,
      "logits/chosen": -2.801562547683716,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -316.3999938964844,
      "logps/rejected": -264.3999938964844,
      "loss": 0.5977,
      "rewards/accuracies": 0.5980159044265747,
      "rewards/chosen": 0.3306640684604645,
      "rewards/margins": 0.31425780057907104,
      "rewards/rejected": 0.01702880859375,
      "step": 3650
    },
    {
      "epoch": 0.9410401647785788,
      "grad_norm": 227.0,
      "learning_rate": 2.9479917610710606e-08,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -272.79998779296875,
      "loss": 0.6035,
      "rewards/accuracies": 0.6281959414482117,
      "rewards/chosen": 0.2818359434604645,
      "rewards/margins": 0.2967773377895355,
      "rewards/rejected": -0.01495361328125,
      "step": 3655
    },
    {
      "epoch": 0.9423274974253347,
      "grad_norm": 278.0,
      "learning_rate": 2.8836251287332645e-08,
      "logits/chosen": -3.28125,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -303.79998779296875,
      "logps/rejected": -256.6000061035156,
      "loss": 0.6258,
      "rewards/accuracies": 0.6279761791229248,
      "rewards/chosen": 0.2740234434604645,
      "rewards/margins": 0.24628905951976776,
      "rewards/rejected": 0.02739257737994194,
      "step": 3660
    },
    {
      "epoch": 0.9436148300720907,
      "grad_norm": 204.0,
      "learning_rate": 2.8192584963954685e-08,
      "logits/chosen": -3.25,
      "logits/rejected": -3.331249952316284,
      "logps/chosen": -275.20001220703125,
      "logps/rejected": -254.60000610351562,
      "loss": 0.5781,
      "rewards/accuracies": 0.7235897779464722,
      "rewards/chosen": 0.281982421875,
      "rewards/margins": 0.37324219942092896,
      "rewards/rejected": -0.09140624850988388,
      "step": 3665
    },
    {
      "epoch": 0.9449021627188465,
      "grad_norm": 203.0,
      "learning_rate": 2.7548918640576724e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -3.1875,
      "logps/chosen": -242.8000030517578,
      "logps/rejected": -188.0,
      "loss": 0.6,
      "rewards/accuracies": 0.6888095140457153,
      "rewards/chosen": 0.24863281846046448,
      "rewards/margins": 0.264404296875,
      "rewards/rejected": -0.01519775390625,
      "step": 3670
    },
    {
      "epoch": 0.9461894953656025,
      "grad_norm": 316.0,
      "learning_rate": 2.6905252317198764e-08,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -3.1812500953674316,
      "logps/chosen": -230.1999969482422,
      "logps/rejected": -155.0,
      "loss": 0.6125,
      "rewards/accuracies": 0.7055556178092957,
      "rewards/chosen": 0.3560546934604645,
      "rewards/margins": 0.24687500298023224,
      "rewards/rejected": 0.10957030951976776,
      "step": 3675
    },
    {
      "epoch": 0.9474768280123584,
      "grad_norm": 260.0,
      "learning_rate": 2.6261585993820803e-08,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -2.9749999046325684,
      "logps/chosen": -235.3000030517578,
      "logps/rejected": -212.60000610351562,
      "loss": 0.6562,
      "rewards/accuracies": 0.65666663646698,
      "rewards/chosen": 0.212890625,
      "rewards/margins": 0.20937499403953552,
      "rewards/rejected": 0.0040527344681322575,
      "step": 3680
    },
    {
      "epoch": 0.9487641606591143,
      "grad_norm": 536.0,
      "learning_rate": 2.5617919670442842e-08,
      "logits/chosen": -3.0374999046325684,
      "logits/rejected": -2.9781250953674316,
      "logps/chosen": -246.89999389648438,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6195,
      "rewards/accuracies": 0.6250432729721069,
      "rewards/chosen": 0.30677491426467896,
      "rewards/margins": 0.2743164002895355,
      "rewards/rejected": 0.03310547024011612,
      "step": 3685
    },
    {
      "epoch": 0.9500514933058702,
      "grad_norm": 288.0,
      "learning_rate": 2.497425334706488e-08,
      "logits/chosen": -3.034374952316284,
      "logits/rejected": -3.234375,
      "logps/chosen": -301.3999938964844,
      "logps/rejected": -232.8000030517578,
      "loss": 0.598,
      "rewards/accuracies": 0.658608078956604,
      "rewards/chosen": 0.23139648139476776,
      "rewards/margins": 0.3070312440395355,
      "rewards/rejected": -0.07583007961511612,
      "step": 3690
    },
    {
      "epoch": 0.9513388259526262,
      "grad_norm": 294.0,
      "learning_rate": 2.433058702368692e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -293.6000061035156,
      "logps/rejected": -239.60000610351562,
      "loss": 0.6266,
      "rewards/accuracies": 0.5623077154159546,
      "rewards/chosen": 0.0049804686568677425,
      "rewards/margins": 0.265380859375,
      "rewards/rejected": -0.2602783143520355,
      "step": 3695
    },
    {
      "epoch": 0.952626158599382,
      "grad_norm": 246.0,
      "learning_rate": 2.368692070030896e-08,
      "logits/chosen": -3.059375047683716,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -289.70001220703125,
      "logps/rejected": -250.0,
      "loss": 0.568,
      "rewards/accuracies": 0.639653742313385,
      "rewards/chosen": 0.4873046875,
      "rewards/margins": 0.43134766817092896,
      "rewards/rejected": 0.05607910081744194,
      "step": 3700
    },
    {
      "epoch": 0.953913491246138,
      "grad_norm": 316.0,
      "learning_rate": 2.3043254376931e-08,
      "logits/chosen": -3.1875,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -213.60000610351562,
      "loss": 0.6453,
      "rewards/accuracies": 0.6354645490646362,
      "rewards/chosen": 0.21000976860523224,
      "rewards/margins": 0.23198242485523224,
      "rewards/rejected": -0.02226562425494194,
      "step": 3705
    },
    {
      "epoch": 0.955200823892894,
      "grad_norm": 290.0,
      "learning_rate": 2.239958805355304e-08,
      "logits/chosen": -3.018749952316284,
      "logits/rejected": -3.203125,
      "logps/chosen": -231.39999389648438,
      "logps/rejected": -190.6999969482422,
      "loss": 0.6586,
      "rewards/accuracies": 0.4617516100406647,
      "rewards/chosen": 0.16020508110523224,
      "rewards/margins": 0.1533203125,
      "rewards/rejected": 0.006884765811264515,
      "step": 3710
    },
    {
      "epoch": 0.9564881565396498,
      "grad_norm": 468.0,
      "learning_rate": 2.1755921730175075e-08,
      "logits/chosen": -3.0093750953674316,
      "logits/rejected": -3.1312499046325684,
      "logps/chosen": -240.60000610351562,
      "logps/rejected": -193.89999389648438,
      "loss": 0.6375,
      "rewards/accuracies": 0.621017336845398,
      "rewards/chosen": 0.20594482123851776,
      "rewards/margins": 0.18740233778953552,
      "rewards/rejected": 0.01845703087747097,
      "step": 3715
    },
    {
      "epoch": 0.9577754891864058,
      "grad_norm": 270.0,
      "learning_rate": 2.1112255406797115e-08,
      "logits/chosen": -3.159374952316284,
      "logits/rejected": -3.234375,
      "logps/chosen": -287.79998779296875,
      "logps/rejected": -222.39999389648438,
      "loss": 0.6242,
      "rewards/accuracies": 0.5992063283920288,
      "rewards/chosen": 0.16474609076976776,
      "rewards/margins": 0.2862792909145355,
      "rewards/rejected": -0.12158203125,
      "step": 3720
    },
    {
      "epoch": 0.9590628218331617,
      "grad_norm": 280.0,
      "learning_rate": 2.0468589083419154e-08,
      "logits/chosen": -3.206249952316284,
      "logits/rejected": -3.0062499046325684,
      "logps/chosen": -240.0,
      "logps/rejected": -221.3000030517578,
      "loss": 0.6516,
      "rewards/accuracies": 0.638888955116272,
      "rewards/chosen": -0.04121093824505806,
      "rewards/margins": 0.19941405951976776,
      "rewards/rejected": -0.24057617783546448,
      "step": 3725
    },
    {
      "epoch": 0.9603501544799176,
      "grad_norm": 228.0,
      "learning_rate": 1.9824922760041193e-08,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.1343750953674316,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -228.60000610351562,
      "loss": 0.6082,
      "rewards/accuracies": 0.6479653716087341,
      "rewards/chosen": 0.2757812440395355,
      "rewards/margins": 0.30659180879592896,
      "rewards/rejected": -0.03066406212747097,
      "step": 3730
    },
    {
      "epoch": 0.9616374871266735,
      "grad_norm": 200.0,
      "learning_rate": 1.9181256436663233e-08,
      "logits/chosen": -3.221874952316284,
      "logits/rejected": -3.1875,
      "logps/chosen": -277.0,
      "logps/rejected": -259.79998779296875,
      "loss": 0.5656,
      "rewards/accuracies": 0.670555591583252,
      "rewards/chosen": 0.3763671815395355,
      "rewards/margins": 0.36503905057907104,
      "rewards/rejected": 0.011425781063735485,
      "step": 3735
    },
    {
      "epoch": 0.9629248197734295,
      "grad_norm": 186.0,
      "learning_rate": 1.8537590113285272e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -3.049999952316284,
      "logps/chosen": -199.39999389648438,
      "logps/rejected": -200.39999389648438,
      "loss": 0.5328,
      "rewards/accuracies": 0.7481061220169067,
      "rewards/chosen": 0.21807861328125,
      "rewards/margins": 0.42890626192092896,
      "rewards/rejected": -0.21035155653953552,
      "step": 3740
    },
    {
      "epoch": 0.9642121524201854,
      "grad_norm": 197.0,
      "learning_rate": 1.789392378990731e-08,
      "logits/chosen": -3.096874952316284,
      "logits/rejected": -3.065624952316284,
      "logps/chosen": -267.6000061035156,
      "logps/rejected": -241.8000030517578,
      "loss": 0.5477,
      "rewards/accuracies": 0.6771429181098938,
      "rewards/chosen": 0.3525390625,
      "rewards/margins": 0.42851561307907104,
      "rewards/rejected": -0.07587890326976776,
      "step": 3745
    },
    {
      "epoch": 0.9654994850669413,
      "grad_norm": 203.0,
      "learning_rate": 1.725025746652935e-08,
      "logits/chosen": -2.971874952316284,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -189.60000610351562,
      "logps/rejected": -194.5,
      "loss": 0.6961,
      "rewards/accuracies": 0.41706353425979614,
      "rewards/chosen": 0.22382812201976776,
      "rewards/margins": 0.081298828125,
      "rewards/rejected": 0.14249268174171448,
      "step": 3750
    },
    {
      "epoch": 0.9667868177136972,
      "grad_norm": 221.0,
      "learning_rate": 1.660659114315139e-08,
      "logits/chosen": -3.1468749046325684,
      "logits/rejected": -3.2562499046325684,
      "logps/chosen": -255.0,
      "logps/rejected": -233.8000030517578,
      "loss": 0.6141,
      "rewards/accuracies": 0.7008333802223206,
      "rewards/chosen": 0.03837890550494194,
      "rewards/margins": 0.28251951932907104,
      "rewards/rejected": -0.24394531548023224,
      "step": 3755
    },
    {
      "epoch": 0.9680741503604532,
      "grad_norm": 227.0,
      "learning_rate": 1.596292481977343e-08,
      "logits/chosen": -3.2125000953674316,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -261.79998779296875,
      "logps/rejected": -230.0,
      "loss": 0.6082,
      "rewards/accuracies": 0.6325000524520874,
      "rewards/chosen": 0.004687500186264515,
      "rewards/margins": 0.2835937440395355,
      "rewards/rejected": -0.278564453125,
      "step": 3760
    },
    {
      "epoch": 0.969361483007209,
      "grad_norm": 524.0,
      "learning_rate": 1.531925849639547e-08,
      "logits/chosen": -3.0687499046325684,
      "logits/rejected": -2.965625047683716,
      "logps/chosen": -257.3999938964844,
      "logps/rejected": -216.0,
      "loss": 0.6398,
      "rewards/accuracies": 0.48777779936790466,
      "rewards/chosen": 0.13172607123851776,
      "rewards/margins": 0.22705078125,
      "rewards/rejected": -0.09501953423023224,
      "step": 3765
    },
    {
      "epoch": 0.970648815653965,
      "grad_norm": 231.0,
      "learning_rate": 1.4675592173017507e-08,
      "logits/chosen": -3.090625047683716,
      "logits/rejected": -2.7890625,
      "logps/chosen": -258.70001220703125,
      "logps/rejected": -255.39999389648438,
      "loss": 0.5867,
      "rewards/accuracies": 0.7288095355033875,
      "rewards/chosen": 0.2542480528354645,
      "rewards/margins": 0.32792967557907104,
      "rewards/rejected": -0.07314453274011612,
      "step": 3770
    },
    {
      "epoch": 0.971936148300721,
      "grad_norm": 306.0,
      "learning_rate": 1.4031925849639546e-08,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.140625,
      "logps/chosen": -302.0,
      "logps/rejected": -265.3999938964844,
      "loss": 0.5648,
      "rewards/accuracies": 0.7127381563186646,
      "rewards/chosen": 0.512890636920929,
      "rewards/margins": 0.38203126192092896,
      "rewards/rejected": 0.1309814453125,
      "step": 3775
    },
    {
      "epoch": 0.9732234809474768,
      "grad_norm": 194.0,
      "learning_rate": 1.3388259526261585e-08,
      "logits/chosen": -3.1187500953674316,
      "logits/rejected": -3.0625,
      "logps/chosen": -263.6000061035156,
      "logps/rejected": -253.8000030517578,
      "loss": 0.5637,
      "rewards/accuracies": 0.7410722970962524,
      "rewards/chosen": 0.27070313692092896,
      "rewards/margins": 0.33378905057907104,
      "rewards/rejected": -0.06316528469324112,
      "step": 3780
    },
    {
      "epoch": 0.9745108135942327,
      "grad_norm": 370.0,
      "learning_rate": 1.2744593202883625e-08,
      "logits/chosen": -3.0531249046325684,
      "logits/rejected": -3.1468749046325684,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -230.60000610351562,
      "loss": 0.6305,
      "rewards/accuracies": 0.5948051810264587,
      "rewards/chosen": 0.32709962129592896,
      "rewards/margins": 0.22792968153953552,
      "rewards/rejected": 0.09863968193531036,
      "step": 3785
    },
    {
      "epoch": 0.9757981462409887,
      "grad_norm": 255.0,
      "learning_rate": 1.2100926879505664e-08,
      "logits/chosen": -3.078125,
      "logits/rejected": -3.171875,
      "logps/chosen": -225.39999389648438,
      "logps/rejected": -225.4499969482422,
      "loss": 0.6,
      "rewards/accuracies": 0.6350000500679016,
      "rewards/chosen": 0.23515625298023224,
      "rewards/margins": 0.3207031190395355,
      "rewards/rejected": -0.08574219048023224,
      "step": 3790
    },
    {
      "epoch": 0.9770854788877446,
      "grad_norm": 216.0,
      "learning_rate": 1.1457260556127703e-08,
      "logits/chosen": -3.128124952316284,
      "logits/rejected": -3.253124952316284,
      "logps/chosen": -242.60000610351562,
      "logps/rejected": -255.1999969482422,
      "loss": 0.5938,
      "rewards/accuracies": 0.7084716558456421,
      "rewards/chosen": 0.12866821885108948,
      "rewards/margins": 0.29765623807907104,
      "rewards/rejected": -0.16871337592601776,
      "step": 3795
    },
    {
      "epoch": 0.9783728115345005,
      "grad_norm": 304.0,
      "learning_rate": 1.0813594232749741e-08,
      "logits/chosen": -3.2093749046325684,
      "logits/rejected": -3.1875,
      "logps/chosen": -297.0,
      "logps/rejected": -257.79998779296875,
      "loss": 0.6164,
      "rewards/accuracies": 0.6434524059295654,
      "rewards/chosen": 0.1171875,
      "rewards/margins": 0.26860350370407104,
      "rewards/rejected": -0.15185546875,
      "step": 3800
    },
    {
      "epoch": 0.9796601441812565,
      "grad_norm": 222.0,
      "learning_rate": 1.016992790937178e-08,
      "logits/chosen": -3.253124952316284,
      "logits/rejected": -3.221874952316284,
      "logps/chosen": -234.8000030517578,
      "logps/rejected": -246.0,
      "loss": 0.5945,
      "rewards/accuracies": 0.6779762506484985,
      "rewards/chosen": -0.03059997595846653,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.32304686307907104,
      "step": 3805
    },
    {
      "epoch": 0.9809474768280123,
      "grad_norm": 258.0,
      "learning_rate": 9.52626158599382e-09,
      "logits/chosen": -3.190624952316284,
      "logits/rejected": -3.2406249046325684,
      "logps/chosen": -236.39999389648438,
      "logps/rejected": -205.39999389648438,
      "loss": 0.6891,
      "rewards/accuracies": 0.5547619462013245,
      "rewards/chosen": 0.06083984300494194,
      "rewards/margins": 0.13901367783546448,
      "rewards/rejected": -0.078125,
      "step": 3810
    },
    {
      "epoch": 0.9822348094747683,
      "grad_norm": 203.0,
      "learning_rate": 8.88259526261586e-09,
      "logits/chosen": -3.1937499046325684,
      "logits/rejected": -3.246875047683716,
      "logps/chosen": -228.0,
      "logps/rejected": -226.6999969482422,
      "loss": 0.6336,
      "rewards/accuracies": 0.5626373887062073,
      "rewards/chosen": 0.22910156846046448,
      "rewards/margins": 0.22048339247703552,
      "rewards/rejected": 0.0084686279296875,
      "step": 3815
    },
    {
      "epoch": 0.9835221421215242,
      "grad_norm": 238.0,
      "learning_rate": 8.238928939237899e-09,
      "logits/chosen": -3.125,
      "logits/rejected": -3.0875000953674316,
      "logps/chosen": -254.0,
      "logps/rejected": -276.0,
      "loss": 0.6453,
      "rewards/accuracies": 0.6486905217170715,
      "rewards/chosen": 0.13974609971046448,
      "rewards/margins": 0.19804687798023224,
      "rewards/rejected": -0.05864257737994194,
      "step": 3820
    },
    {
      "epoch": 0.9848094747682801,
      "grad_norm": 300.0,
      "learning_rate": 7.595262615859938e-09,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.140625,
      "logps/chosen": -246.0,
      "logps/rejected": -271.6000061035156,
      "loss": 0.6992,
      "rewards/accuracies": 0.5317400097846985,
      "rewards/chosen": -0.0075927735306322575,
      "rewards/margins": 0.0814208984375,
      "rewards/rejected": -0.08867187798023224,
      "step": 3825
    },
    {
      "epoch": 0.986096807415036,
      "grad_norm": 348.0,
      "learning_rate": 6.951596292481977e-09,
      "logits/chosen": -3.137500047683716,
      "logits/rejected": -3.200000047683716,
      "logps/chosen": -323.20001220703125,
      "logps/rejected": -273.20001220703125,
      "loss": 0.6344,
      "rewards/accuracies": 0.634772777557373,
      "rewards/chosen": 0.1572265625,
      "rewards/margins": 0.275390625,
      "rewards/rejected": -0.11796875298023224,
      "step": 3830
    },
    {
      "epoch": 0.987384140061792,
      "grad_norm": 266.0,
      "learning_rate": 6.307929969104016e-09,
      "logits/chosen": -3.246875047683716,
      "logits/rejected": -3.0,
      "logps/chosen": -309.79998779296875,
      "logps/rejected": -293.20001220703125,
      "loss": 0.591,
      "rewards/accuracies": 0.663095235824585,
      "rewards/chosen": 0.05942382663488388,
      "rewards/margins": 0.3209472596645355,
      "rewards/rejected": -0.2616210877895355,
      "step": 3835
    },
    {
      "epoch": 0.9886714727085479,
      "grad_norm": 236.0,
      "learning_rate": 5.664263645726055e-09,
      "logits/chosen": -3.125,
      "logits/rejected": -3.2249999046325684,
      "logps/chosen": -321.6000061035156,
      "logps/rejected": -283.6000061035156,
      "loss": 0.6398,
      "rewards/accuracies": 0.6102564334869385,
      "rewards/chosen": 0.37861329317092896,
      "rewards/margins": 0.24802246689796448,
      "rewards/rejected": 0.13090820610523224,
      "step": 3840
    },
    {
      "epoch": 0.9899588053553038,
      "grad_norm": 195.0,
      "learning_rate": 5.020597322348095e-09,
      "logits/chosen": -3.106250047683716,
      "logits/rejected": -3.2874999046325684,
      "logps/chosen": -364.79998779296875,
      "logps/rejected": -269.3999938964844,
      "loss": 0.5344,
      "rewards/accuracies": 0.7478571534156799,
      "rewards/chosen": 0.3985839784145355,
      "rewards/margins": 0.462890625,
      "rewards/rejected": -0.06435547024011612,
      "step": 3845
    },
    {
      "epoch": 0.9912461380020597,
      "grad_norm": 266.0,
      "learning_rate": 4.376930998970134e-09,
      "logits/chosen": -3.2750000953674316,
      "logits/rejected": -3.237499952316284,
      "logps/chosen": -336.0,
      "logps/rejected": -253.1999969482422,
      "loss": 0.5668,
      "rewards/accuracies": 0.6520971059799194,
      "rewards/chosen": 0.3326171934604645,
      "rewards/margins": 0.3980468809604645,
      "rewards/rejected": -0.06573486328125,
      "step": 3850
    },
    {
      "epoch": 0.9925334706488157,
      "grad_norm": 184.0,
      "learning_rate": 3.733264675592173e-09,
      "logits/chosen": -3.043750047683716,
      "logits/rejected": -3.0687499046325684,
      "logps/chosen": -240.60000610351562,
      "logps/rejected": -264.20001220703125,
      "loss": 0.6031,
      "rewards/accuracies": 0.6941666603088379,
      "rewards/chosen": 0.15046386420726776,
      "rewards/margins": 0.38066405057907104,
      "rewards/rejected": -0.23037108778953552,
      "step": 3855
    },
    {
      "epoch": 0.9938208032955715,
      "grad_norm": 177.0,
      "learning_rate": 3.089598352214212e-09,
      "logits/chosen": -3.2281250953674316,
      "logits/rejected": -3.265625,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -274.0,
      "loss": 0.5969,
      "rewards/accuracies": 0.6410714387893677,
      "rewards/chosen": 0.17557373642921448,
      "rewards/margins": 0.2978515625,
      "rewards/rejected": -0.12258300930261612,
      "step": 3860
    },
    {
      "epoch": 0.9951081359423275,
      "grad_norm": 227.0,
      "learning_rate": 2.445932028836251e-09,
      "logits/chosen": -3.0999999046325684,
      "logits/rejected": -3.1656250953674316,
      "logps/chosen": -326.0,
      "logps/rejected": -294.0,
      "loss": 0.6086,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.23496094346046448,
      "rewards/margins": 0.3084960877895355,
      "rewards/rejected": -0.07343749701976776,
      "step": 3865
    },
    {
      "epoch": 0.9963954685890835,
      "grad_norm": 228.0,
      "learning_rate": 1.8022657054582903e-09,
      "logits/chosen": -3.2437500953674316,
      "logits/rejected": -3.184375047683716,
      "logps/chosen": -300.79998779296875,
      "logps/rejected": -284.3999938964844,
      "loss": 0.5766,
      "rewards/accuracies": 0.6576923131942749,
      "rewards/chosen": 0.15107421576976776,
      "rewards/margins": 0.3578124940395355,
      "rewards/rejected": -0.20668944716453552,
      "step": 3870
    },
    {
      "epoch": 0.9976828012358393,
      "grad_norm": 247.0,
      "learning_rate": 1.1585993820803295e-09,
      "logits/chosen": -3.1312499046325684,
      "logits/rejected": -3.2437500953674316,
      "logps/chosen": -303.6000061035156,
      "logps/rejected": -237.39999389648438,
      "loss": 0.632,
      "rewards/accuracies": 0.5641666650772095,
      "rewards/chosen": 0.27167969942092896,
      "rewards/margins": 0.25312501192092896,
      "rewards/rejected": 0.01823730394244194,
      "step": 3875
    },
    {
      "epoch": 0.9989701338825953,
      "grad_norm": 266.0,
      "learning_rate": 5.149330587023687e-10,
      "logits/chosen": -3.174999952316284,
      "logits/rejected": -3.078125,
      "logps/chosen": -276.3999938964844,
      "logps/rejected": -206.8000030517578,
      "loss": 0.6266,
      "rewards/accuracies": 0.6282251477241516,
      "rewards/chosen": 0.25634765625,
      "rewards/margins": 0.22988280653953552,
      "rewards/rejected": 0.02695312537252903,
      "step": 3880
    }
  ],
  "logging_steps": 5,
  "max_steps": 3884,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 3000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}