{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 50,
  "global_step": 3884,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0012873326467559218,
      "grad_norm": 139.0,
      "learning_rate": 4.99356333676622e-07,
      "logits/chosen": -0.27949219942092896,
      "logits/rejected": -0.25908201932907104,
      "logps/chosen": -257.3999938964844,
      "logps/rejected": -219.8000030517578,
      "loss": 0.6922,
      "rewards/accuracies": 0.23000001907348633,
      "rewards/chosen": 0.0010925292735919356,
      "rewards/margins": 0.0091552734375,
      "rewards/rejected": -0.008056640625,
      "step": 5
    },
    {
      "epoch": 0.0025746652935118436,
      "grad_norm": 91.0,
      "learning_rate": 4.987126673532441e-07,
      "logits/chosen": 0.0005371093866415322,
      "logits/rejected": -0.4136718809604645,
      "logps/chosen": -234.6999969482422,
      "logps/rejected": -179.89999389648438,
      "loss": 0.6836,
      "rewards/accuracies": 0.28844156861305237,
      "rewards/chosen": 0.01600341871380806,
      "rewards/margins": 0.01941223070025444,
      "rewards/rejected": -0.0033508301712572575,
      "step": 10
    },
    {
      "epoch": 0.0038619979402677654,
      "grad_norm": 107.5,
      "learning_rate": 4.980690010298661e-07,
      "logits/chosen": -0.2728515565395355,
      "logits/rejected": -0.2734375,
      "logps/chosen": -283.3999938964844,
      "logps/rejected": -264.6000061035156,
      "loss": 0.6891,
      "rewards/accuracies": 0.3425000309944153,
      "rewards/chosen": 0.01163330115377903,
      "rewards/margins": 0.015419716015458107,
      "rewards/rejected": -0.0037673949263989925,
      "step": 15
    },
    {
      "epoch": 0.005149330587023687,
      "grad_norm": 101.5,
      "learning_rate": 4.974253347064881e-07,
      "logits/chosen": -0.31132811307907104,
      "logits/rejected": -0.29326170682907104,
      "logps/chosen": -291.3999938964844,
      "logps/rejected": -241.60000610351562,
      "loss": 0.6852,
      "rewards/accuracies": 0.2958333492279053,
      "rewards/chosen": 0.02814025804400444,
      "rewards/margins": 0.02118835411965847,
      "rewards/rejected": 0.00694961566478014,
      "step": 20
    },
    {
      "epoch": 0.006436663233779609,
      "grad_norm": 93.0,
      "learning_rate": 4.967816683831102e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.3623046875,
      "logps/chosen": -285.0,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6844,
      "rewards/accuracies": 0.351666659116745,
      "rewards/chosen": 0.00882720947265625,
      "rewards/margins": 0.01831665076315403,
      "rewards/rejected": -0.009488677605986595,
      "step": 25
    },
    {
      "epoch": 0.007723995880535531,
      "grad_norm": 117.5,
      "learning_rate": 4.961380020597322e-07,
      "logits/chosen": -0.32177734375,
      "logits/rejected": -0.2998046875,
      "logps/chosen": -259.20001220703125,
      "logps/rejected": -235.1999969482422,
      "loss": 0.6922,
      "rewards/accuracies": 0.2574999928474426,
      "rewards/chosen": 0.00812377966940403,
      "rewards/margins": 0.0039459229446947575,
      "rewards/rejected": 0.004180908203125,
      "step": 30
    },
    {
      "epoch": 0.009011328527291453,
      "grad_norm": 115.0,
      "learning_rate": 4.954943357363543e-07,
      "logits/chosen": -0.22817382216453552,
      "logits/rejected": -0.17641600966453552,
      "logps/chosen": -230.1999969482422,
      "logps/rejected": -212.10000610351562,
      "loss": 0.6883,
      "rewards/accuracies": 0.3698076903820038,
      "rewards/chosen": 0.01279296912252903,
      "rewards/margins": 0.01370849646627903,
      "rewards/rejected": -0.0009277343633584678,
      "step": 35
    },
    {
      "epoch": 0.010298661174047374,
      "grad_norm": 108.0,
      "learning_rate": 4.948506694129763e-07,
      "logits/chosen": -0.2730468809604645,
      "logits/rejected": -0.3773437440395355,
      "logps/chosen": -294.0,
      "logps/rejected": -269.79998779296875,
      "loss": 0.6945,
      "rewards/accuracies": 0.24000000953674316,
      "rewards/chosen": 0.011561584658920765,
      "rewards/margins": -0.0020996094681322575,
      "rewards/rejected": 0.013623046688735485,
      "step": 40
    },
    {
      "epoch": 0.011585993820803296,
      "grad_norm": 132.0,
      "learning_rate": 4.942070030895984e-07,
      "logits/chosen": -0.38007813692092896,
      "logits/rejected": -0.31132811307907104,
      "logps/chosen": -298.0,
      "logps/rejected": -269.0,
      "loss": 0.6828,
      "rewards/accuracies": 0.33500000834465027,
      "rewards/chosen": 0.02399902418255806,
      "rewards/margins": 0.02048950269818306,
      "rewards/rejected": 0.0034988403785973787,
      "step": 45
    },
    {
      "epoch": 0.012873326467559218,
      "grad_norm": 131.0,
      "learning_rate": 4.935633367662204e-07,
      "logits/chosen": -0.20849609375,
      "logits/rejected": -0.20449218153953552,
      "logps/chosen": -274.20001220703125,
      "logps/rejected": -238.39999389648438,
      "loss": 0.675,
      "rewards/accuracies": 0.2977272868156433,
      "rewards/chosen": 0.02877197228372097,
      "rewards/margins": 0.04194030910730362,
      "rewards/rejected": -0.01318359375,
      "step": 50
    },
    {
      "epoch": 0.01416065911431514,
      "grad_norm": 182.0,
      "learning_rate": 4.929196704428423e-07,
      "logits/chosen": -0.26123046875,
      "logits/rejected": -0.3114257752895355,
      "logps/chosen": -236.39999389648438,
      "logps/rejected": -213.0,
      "loss": 0.7031,
      "rewards/accuracies": 0.2857142984867096,
      "rewards/chosen": -0.0054412842728197575,
      "rewards/margins": -0.01875000074505806,
      "rewards/rejected": 0.013336181640625,
      "step": 55
    },
    {
      "epoch": 0.015447991761071062,
      "grad_norm": 105.5,
      "learning_rate": 4.922760041194645e-07,
      "logits/chosen": -0.3619628846645355,
      "logits/rejected": -0.2982421815395355,
      "logps/chosen": -231.1999969482422,
      "logps/rejected": -217.60000610351562,
      "loss": 0.7031,
      "rewards/accuracies": 0.23250000178813934,
      "rewards/chosen": 0.0009033203241415322,
      "rewards/margins": -0.01523437537252903,
      "rewards/rejected": 0.016204833984375,
      "step": 60
    },
    {
      "epoch": 0.016735324407826983,
      "grad_norm": 116.5,
      "learning_rate": 4.916323377960865e-07,
      "logits/chosen": -0.29902344942092896,
      "logits/rejected": -0.31621092557907104,
      "logps/chosen": -262.20001220703125,
      "logps/rejected": -242.1999969482422,
      "loss": 0.6797,
      "rewards/accuracies": 0.34166666865348816,
      "rewards/chosen": 0.01579437218606472,
      "rewards/margins": 0.02775878831744194,
      "rewards/rejected": -0.011981201358139515,
      "step": 65
    },
    {
      "epoch": 0.018022657054582905,
      "grad_norm": 146.0,
      "learning_rate": 4.909886714727085e-07,
      "logits/chosen": -0.21943359076976776,
      "logits/rejected": -0.13200683891773224,
      "logps/chosen": -311.3999938964844,
      "logps/rejected": -252.60000610351562,
      "loss": 0.682,
      "rewards/accuracies": 0.3121969699859619,
      "rewards/chosen": 0.029541015625,
      "rewards/margins": 0.020922088995575905,
      "rewards/rejected": 0.008618163876235485,
      "step": 70
    },
    {
      "epoch": 0.019309989701338827,
      "grad_norm": 137.0,
      "learning_rate": 4.903450051493306e-07,
      "logits/chosen": -0.25932615995407104,
      "logits/rejected": -0.25390625,
      "logps/chosen": -332.0,
      "logps/rejected": -267.0,
      "loss": 0.6867,
      "rewards/accuracies": 0.3050000071525574,
      "rewards/chosen": 0.012713623233139515,
      "rewards/margins": 0.0150146484375,
      "rewards/rejected": -0.0023193359375,
      "step": 75
    },
    {
      "epoch": 0.02059732234809475,
      "grad_norm": 130.0,
      "learning_rate": 4.897013388259526e-07,
      "logits/chosen": -0.2547851502895355,
      "logits/rejected": -0.21196289360523224,
      "logps/chosen": -356.0,
      "logps/rejected": -336.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.28214284777641296,
      "rewards/chosen": 0.04510498046875,
      "rewards/margins": 0.0071655274368822575,
      "rewards/rejected": 0.03797607496380806,
      "step": 80
    },
    {
      "epoch": 0.02188465499485067,
      "grad_norm": 129.0,
      "learning_rate": 4.890576725025746e-07,
      "logits/chosen": -0.44999998807907104,
      "logits/rejected": -0.4117187559604645,
      "logps/chosen": -272.79998779296875,
      "logps/rejected": -287.3999938964844,
      "loss": 0.6875,
      "rewards/accuracies": 0.2775000035762787,
      "rewards/chosen": 0.014880371280014515,
      "rewards/margins": 0.011883544735610485,
      "rewards/rejected": 0.0029724121559411287,
      "step": 85
    },
    {
      "epoch": 0.023171987641606592,
      "grad_norm": 104.0,
      "learning_rate": 4.884140061791967e-07,
      "logits/chosen": -0.4287109375,
      "logits/rejected": -0.2978515625,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -246.0,
      "loss": 0.6695,
      "rewards/accuracies": 0.4466666579246521,
      "rewards/chosen": 0.05021972581744194,
      "rewards/margins": 0.0462646484375,
      "rewards/rejected": 0.003979492001235485,
      "step": 90
    },
    {
      "epoch": 0.024459320288362514,
      "grad_norm": 255.0,
      "learning_rate": 4.877703398558187e-07,
      "logits/chosen": -0.23916015028953552,
      "logits/rejected": -0.294921875,
      "logps/chosen": -306.79998779296875,
      "logps/rejected": -254.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.31969699263572693,
      "rewards/chosen": 0.01606445387005806,
      "rewards/margins": 0.0068115233443677425,
      "rewards/rejected": 0.009204101748764515,
      "step": 95
    },
    {
      "epoch": 0.025746652935118436,
      "grad_norm": 116.0,
      "learning_rate": 4.871266735324407e-07,
      "logits/chosen": -0.4189453125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -310.3999938964844,
      "logps/rejected": -268.0,
      "loss": 0.6852,
      "rewards/accuracies": 0.3475000262260437,
      "rewards/chosen": 0.032534025609493256,
      "rewards/margins": 0.02002258226275444,
      "rewards/rejected": 0.012500000186264515,
      "step": 100
    },
    {
      "epoch": 0.027033985581874358,
      "grad_norm": 398.0,
      "learning_rate": 4.864830072090629e-07,
      "logits/chosen": -0.29619139432907104,
      "logits/rejected": -0.375,
      "logps/chosen": -290.0,
      "logps/rejected": -255.1999969482422,
      "loss": 0.6977,
      "rewards/accuracies": 0.41333335638046265,
      "rewards/chosen": 0.018310546875,
      "rewards/margins": 0.00870361365377903,
      "rewards/rejected": 0.009631347842514515,
      "step": 105
    },
    {
      "epoch": 0.02832131822863028,
      "grad_norm": 111.5,
      "learning_rate": 4.858393408856848e-07,
      "logits/chosen": -0.34257811307907104,
      "logits/rejected": -0.3475585877895355,
      "logps/chosen": -293.79998779296875,
      "logps/rejected": -271.79998779296875,
      "loss": 0.6961,
      "rewards/accuracies": 0.351666659116745,
      "rewards/chosen": 0.04102172702550888,
      "rewards/margins": -0.0027099610306322575,
      "rewards/rejected": 0.043701171875,
      "step": 110
    },
    {
      "epoch": 0.0296086508753862,
      "grad_norm": 178.0,
      "learning_rate": 4.851956745623069e-07,
      "logits/chosen": -0.3543945252895355,
      "logits/rejected": -0.28828126192092896,
      "logps/chosen": -296.79998779296875,
      "logps/rejected": -278.0,
      "loss": 0.6789,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.04083862155675888,
      "rewards/margins": 0.03194580227136612,
      "rewards/rejected": 0.008831786923110485,
      "step": 115
    },
    {
      "epoch": 0.030895983522142123,
      "grad_norm": 125.0,
      "learning_rate": 4.845520082389289e-07,
      "logits/chosen": -0.4339843690395355,
      "logits/rejected": -0.4078125059604645,
      "logps/chosen": -319.3999938964844,
      "logps/rejected": -271.79998779296875,
      "loss": 0.6844,
      "rewards/accuracies": 0.4050000309944153,
      "rewards/chosen": 0.05195312574505806,
      "rewards/margins": 0.02604980394244194,
      "rewards/rejected": 0.025984954088926315,
      "step": 120
    },
    {
      "epoch": 0.032183316168898045,
      "grad_norm": 170.0,
      "learning_rate": 4.839083419155509e-07,
      "logits/chosen": -0.30683594942092896,
      "logits/rejected": -0.162109375,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -239.8000030517578,
      "loss": 0.6781,
      "rewards/accuracies": 0.3977777659893036,
      "rewards/chosen": 0.05373535305261612,
      "rewards/margins": 0.03863678127527237,
      "rewards/rejected": 0.01505126990377903,
      "step": 125
    },
    {
      "epoch": 0.03347064881565397,
      "grad_norm": 125.0,
      "learning_rate": 4.83264675592173e-07,
      "logits/chosen": -0.3218750059604645,
      "logits/rejected": -0.29296875,
      "logps/chosen": -258.0,
      "logps/rejected": -241.8000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.4482142925262451,
      "rewards/chosen": 0.05534667894244194,
      "rewards/margins": 0.04257812350988388,
      "rewards/rejected": 0.012707519344985485,
      "step": 130
    },
    {
      "epoch": 0.03475798146240989,
      "grad_norm": 122.5,
      "learning_rate": 4.82621009268795e-07,
      "logits/chosen": -0.37763673067092896,
      "logits/rejected": -0.2506347596645355,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -227.60000610351562,
      "loss": 0.6953,
      "rewards/accuracies": 0.40498918294906616,
      "rewards/chosen": 0.02900390699505806,
      "rewards/margins": 0.00860595703125,
      "rewards/rejected": 0.0205078125,
      "step": 135
    },
    {
      "epoch": 0.03604531410916581,
      "grad_norm": 192.0,
      "learning_rate": 4.819773429454171e-07,
      "logits/chosen": -0.28276365995407104,
      "logits/rejected": -0.23664550483226776,
      "logps/chosen": -287.79998779296875,
      "logps/rejected": -222.1999969482422,
      "loss": 0.6953,
      "rewards/accuracies": 0.3323718011379242,
      "rewards/chosen": 0.01629638671875,
      "rewards/margins": 0.005297851748764515,
      "rewards/rejected": 0.01100910920649767,
      "step": 140
    },
    {
      "epoch": 0.03733264675592173,
      "grad_norm": 208.0,
      "learning_rate": 4.813336766220391e-07,
      "logits/chosen": -0.3755859434604645,
      "logits/rejected": -0.4078125059604645,
      "logps/chosen": -281.20001220703125,
      "logps/rejected": -215.3000030517578,
      "loss": 0.6805,
      "rewards/accuracies": 0.38154762983322144,
      "rewards/chosen": 0.05917968600988388,
      "rewards/margins": 0.03508300706744194,
      "rewards/rejected": 0.02404480054974556,
      "step": 145
    },
    {
      "epoch": 0.038619979402677654,
      "grad_norm": 211.0,
      "learning_rate": 4.806900102986612e-07,
      "logits/chosen": -0.3433593809604645,
      "logits/rejected": -0.35761719942092896,
      "logps/chosen": -299.0,
      "logps/rejected": -248.39999389648438,
      "loss": 0.6742,
      "rewards/accuracies": 0.42000001668930054,
      "rewards/chosen": 0.06776733696460724,
      "rewards/margins": 0.04287109524011612,
      "rewards/rejected": 0.02485351637005806,
      "step": 150
    },
    {
      "epoch": 0.039907312049433576,
      "grad_norm": 135.0,
      "learning_rate": 4.800463439752832e-07,
      "logits/chosen": -0.23027344048023224,
      "logits/rejected": -0.3988281190395355,
      "logps/chosen": -171.0,
      "logps/rejected": -152.39999389648438,
      "loss": 0.6852,
      "rewards/accuracies": 0.3839285969734192,
      "rewards/chosen": 0.035736083984375,
      "rewards/margins": 0.01593933068215847,
      "rewards/rejected": 0.01990356482565403,
      "step": 155
    },
    {
      "epoch": 0.0411946446961895,
      "grad_norm": 290.0,
      "learning_rate": 4.794026776519052e-07,
      "logits/chosen": -0.126708984375,
      "logits/rejected": -0.3541015684604645,
      "logps/chosen": -231.8000030517578,
      "logps/rejected": -222.8000030517578,
      "loss": 0.6758,
      "rewards/accuracies": 0.3784615397453308,
      "rewards/chosen": 0.03879394382238388,
      "rewards/margins": 0.04508667066693306,
      "rewards/rejected": -0.0063018798828125,
      "step": 160
    },
    {
      "epoch": 0.04248197734294542,
      "grad_norm": 91.0,
      "learning_rate": 4.787590113285273e-07,
      "logits/chosen": -0.30244141817092896,
      "logits/rejected": -0.32499998807907104,
      "logps/chosen": -247.8000030517578,
      "logps/rejected": -254.60000610351562,
      "loss": 0.682,
      "rewards/accuracies": 0.34095239639282227,
      "rewards/chosen": 0.043182373046875,
      "rewards/margins": 0.0220947265625,
      "rewards/rejected": 0.0211334228515625,
      "step": 165
    },
    {
      "epoch": 0.04376930998970134,
      "grad_norm": 172.0,
      "learning_rate": 4.781153450051493e-07,
      "logits/chosen": -0.2515625059604645,
      "logits/rejected": -0.27543944120407104,
      "logps/chosen": -295.6000061035156,
      "logps/rejected": -293.6000061035156,
      "loss": 0.6937,
      "rewards/accuracies": 0.29942309856414795,
      "rewards/chosen": 0.02937011793255806,
      "rewards/margins": 0.0047973631881177425,
      "rewards/rejected": 0.02455444261431694,
      "step": 170
    },
    {
      "epoch": 0.04505664263645726,
      "grad_norm": 116.0,
      "learning_rate": 4.774716786817714e-07,
      "logits/chosen": -0.24423828721046448,
      "logits/rejected": -0.16337890923023224,
      "logps/chosen": -255.0,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6805,
      "rewards/accuracies": 0.39797621965408325,
      "rewards/chosen": 0.05039062350988388,
      "rewards/margins": 0.03078613243997097,
      "rewards/rejected": 0.01967773400247097,
      "step": 175
    },
    {
      "epoch": 0.046343975283213185,
      "grad_norm": 139.0,
      "learning_rate": 4.7682801235839336e-07,
      "logits/chosen": -0.2582031190395355,
      "logits/rejected": -0.27099609375,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -266.3999938964844,
      "loss": 0.6797,
      "rewards/accuracies": 0.4489285945892334,
      "rewards/chosen": 0.023285675793886185,
      "rewards/margins": 0.03536377102136612,
      "rewards/rejected": -0.0120849609375,
      "step": 180
    },
    {
      "epoch": 0.047631307929969106,
      "grad_norm": 126.5,
      "learning_rate": 4.7618434603501545e-07,
      "logits/chosen": -0.3628906309604645,
      "logits/rejected": -0.28828126192092896,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -281.20001220703125,
      "loss": 0.693,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.04859619215130806,
      "rewards/margins": 0.01395263709127903,
      "rewards/rejected": 0.03452148288488388,
      "step": 185
    },
    {
      "epoch": 0.04891864057672503,
      "grad_norm": 162.0,
      "learning_rate": 4.755406797116375e-07,
      "logits/chosen": -0.43867188692092896,
      "logits/rejected": -0.2738281190395355,
      "logps/chosen": -240.60000610351562,
      "logps/rejected": -265.6000061035156,
      "loss": 0.682,
      "rewards/accuracies": 0.44282054901123047,
      "rewards/chosen": 0.06024169921875,
      "rewards/margins": 0.04194336012005806,
      "rewards/rejected": 0.01833190955221653,
      "step": 190
    },
    {
      "epoch": 0.05020597322348095,
      "grad_norm": 101.0,
      "learning_rate": 4.748970133882595e-07,
      "logits/chosen": -0.3460937440395355,
      "logits/rejected": -0.3578124940395355,
      "logps/chosen": -247.39999389648438,
      "logps/rejected": -218.60000610351562,
      "loss": 0.7031,
      "rewards/accuracies": 0.39416664838790894,
      "rewards/chosen": 0.07011719048023224,
      "rewards/margins": -0.00955810584127903,
      "rewards/rejected": 0.0794677734375,
      "step": 195
    },
    {
      "epoch": 0.05149330587023687,
      "grad_norm": 130.0,
      "learning_rate": 4.742533470648816e-07,
      "logits/chosen": -0.2676757872104645,
      "logits/rejected": -0.35078126192092896,
      "logps/chosen": -252.1999969482422,
      "logps/rejected": -263.6000061035156,
      "loss": 0.6883,
      "rewards/accuracies": 0.381923109292984,
      "rewards/chosen": 0.017916107550263405,
      "rewards/margins": 0.01697998121380806,
      "rewards/rejected": 0.0009277343633584678,
      "step": 200
    },
    {
      "epoch": 0.052780638516992794,
      "grad_norm": 147.0,
      "learning_rate": 4.7360968074150357e-07,
      "logits/chosen": -0.51171875,
      "logits/rejected": -0.4496093690395355,
      "logps/chosen": -268.79998779296875,
      "logps/rejected": -272.6000061035156,
      "loss": 0.6828,
      "rewards/accuracies": 0.5182142853736877,
      "rewards/chosen": 0.06253661960363388,
      "rewards/margins": 0.03233947604894638,
      "rewards/rejected": 0.03020019456744194,
      "step": 205
    },
    {
      "epoch": 0.054067971163748715,
      "grad_norm": 254.0,
      "learning_rate": 4.729660144181256e-07,
      "logits/chosen": -0.3841796815395355,
      "logits/rejected": -0.3890624940395355,
      "logps/chosen": -318.0,
      "logps/rejected": -287.3999938964844,
      "loss": 0.6813,
      "rewards/accuracies": 0.4032692313194275,
      "rewards/chosen": 0.05887908861041069,
      "rewards/margins": 0.02734375,
      "rewards/rejected": 0.03162536770105362,
      "step": 210
    },
    {
      "epoch": 0.05535530381050464,
      "grad_norm": 181.0,
      "learning_rate": 4.7232234809474765e-07,
      "logits/chosen": -0.34296876192092896,
      "logits/rejected": -0.3902343809604645,
      "logps/chosen": -298.0,
      "logps/rejected": -244.60000610351562,
      "loss": 0.6656,
      "rewards/accuracies": 0.429166704416275,
      "rewards/chosen": 0.07065429538488388,
      "rewards/margins": 0.05650634691119194,
      "rewards/rejected": 0.01422119140625,
      "step": 215
    },
    {
      "epoch": 0.05664263645726056,
      "grad_norm": 153.0,
      "learning_rate": 4.716786817713697e-07,
      "logits/chosen": -0.45234376192092896,
      "logits/rejected": -0.439453125,
      "logps/chosen": -314.3999938964844,
      "logps/rejected": -256.6000061035156,
      "loss": 0.6641,
      "rewards/accuracies": 0.40416663885116577,
      "rewards/chosen": 0.08349609375,
      "rewards/margins": 0.06516113132238388,
      "rewards/rejected": 0.01843872107565403,
      "step": 220
    },
    {
      "epoch": 0.05792996910401648,
      "grad_norm": 140.0,
      "learning_rate": 4.7103501544799174e-07,
      "logits/chosen": -0.2572265565395355,
      "logits/rejected": -0.2650390565395355,
      "logps/chosen": -306.0,
      "logps/rejected": -284.6000061035156,
      "loss": 0.693,
      "rewards/accuracies": 0.36000004410743713,
      "rewards/chosen": 0.03232421725988388,
      "rewards/margins": 0.010723876766860485,
      "rewards/rejected": 0.0216217041015625,
      "step": 225
    },
    {
      "epoch": 0.0592173017507724,
      "grad_norm": 278.0,
      "learning_rate": 4.703913491246138e-07,
      "logits/chosen": -0.3013671934604645,
      "logits/rejected": -0.3319335877895355,
      "logps/chosen": -318.20001220703125,
      "logps/rejected": -299.3999938964844,
      "loss": 0.6922,
      "rewards/accuracies": 0.3422619104385376,
      "rewards/chosen": 0.03037109412252903,
      "rewards/margins": 0.009791565127670765,
      "rewards/rejected": 0.02055663987994194,
      "step": 230
    },
    {
      "epoch": 0.060504634397528324,
      "grad_norm": 149.0,
      "learning_rate": 4.697476828012358e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -344.79998779296875,
      "logps/rejected": -320.20001220703125,
      "loss": 0.6813,
      "rewards/accuracies": 0.3857143223285675,
      "rewards/chosen": 0.05219726637005806,
      "rewards/margins": 0.020111083984375,
      "rewards/rejected": 0.03214111179113388,
      "step": 235
    },
    {
      "epoch": 0.061791967044284246,
      "grad_norm": 656.0,
      "learning_rate": 4.6910401647785787e-07,
      "logits/chosen": -0.40410155057907104,
      "logits/rejected": -0.4076171815395355,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -261.3999938964844,
      "loss": 0.6742,
      "rewards/accuracies": 0.4468589723110199,
      "rewards/chosen": 0.08696289360523224,
      "rewards/margins": 0.04173583909869194,
      "rewards/rejected": 0.0452728271484375,
      "step": 240
    },
    {
      "epoch": 0.06307929969104016,
      "grad_norm": 158.0,
      "learning_rate": 4.6846035015447986e-07,
      "logits/chosen": -0.16752929985523224,
      "logits/rejected": -0.19163818657398224,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -268.0,
      "loss": 0.675,
      "rewards/accuracies": 0.40583333373069763,
      "rewards/chosen": 0.05635528638958931,
      "rewards/margins": 0.04343261569738388,
      "rewards/rejected": 0.013012695126235485,
      "step": 245
    },
    {
      "epoch": 0.06436663233779609,
      "grad_norm": 306.0,
      "learning_rate": 4.6781668383110195e-07,
      "logits/chosen": -0.29736328125,
      "logits/rejected": -0.2964843809604645,
      "logps/chosen": -249.39999389648438,
      "logps/rejected": -283.79998779296875,
      "loss": 0.6805,
      "rewards/accuracies": 0.3725000023841858,
      "rewards/chosen": 0.06855468451976776,
      "rewards/margins": 0.03720702975988388,
      "rewards/rejected": 0.03137817233800888,
      "step": 250
    },
    {
      "epoch": 0.065653964984552,
      "grad_norm": 155.0,
      "learning_rate": 4.67173017507724e-07,
      "logits/chosen": -0.20640258491039276,
      "logits/rejected": -0.20017090439796448,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -281.0,
      "loss": 0.7,
      "rewards/accuracies": 0.4580952525138855,
      "rewards/chosen": 0.0218505859375,
      "rewards/margins": 0.009838867001235485,
      "rewards/rejected": 0.011981201358139515,
      "step": 255
    },
    {
      "epoch": 0.06694129763130793,
      "grad_norm": 148.0,
      "learning_rate": 4.66529351184346e-07,
      "logits/chosen": -0.32988280057907104,
      "logits/rejected": -0.30839842557907104,
      "logps/chosen": -283.3999938964844,
      "logps/rejected": -266.20001220703125,
      "loss": 0.6727,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.09843750298023224,
      "rewards/margins": 0.06748046725988388,
      "rewards/rejected": 0.0308837890625,
      "step": 260
    },
    {
      "epoch": 0.06822863027806385,
      "grad_norm": 174.0,
      "learning_rate": 4.658856848609681e-07,
      "logits/chosen": -0.42304688692092896,
      "logits/rejected": -0.38164061307907104,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -221.0,
      "loss": 0.6773,
      "rewards/accuracies": 0.47333335876464844,
      "rewards/chosen": 0.08969726413488388,
      "rewards/margins": 0.03272705152630806,
      "rewards/rejected": 0.05686035007238388,
      "step": 265
    },
    {
      "epoch": 0.06951596292481978,
      "grad_norm": 147.0,
      "learning_rate": 4.652420185375901e-07,
      "logits/chosen": -0.3578124940395355,
      "logits/rejected": -0.24346923828125,
      "logps/chosen": -330.20001220703125,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6734,
      "rewards/accuracies": 0.45749998092651367,
      "rewards/chosen": 0.09907226264476776,
      "rewards/margins": 0.04402618482708931,
      "rewards/rejected": 0.05515136569738388,
      "step": 270
    },
    {
      "epoch": 0.07080329557157569,
      "grad_norm": 116.0,
      "learning_rate": 4.645983522142121e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": -0.4507812559604645,
      "logps/chosen": -296.20001220703125,
      "logps/rejected": -243.1999969482422,
      "loss": 0.6711,
      "rewards/accuracies": 0.4415384829044342,
      "rewards/chosen": 0.09282226860523224,
      "rewards/margins": 0.06800536811351776,
      "rewards/rejected": 0.02486267127096653,
      "step": 275
    },
    {
      "epoch": 0.07209062821833162,
      "grad_norm": 270.0,
      "learning_rate": 4.639546858908342e-07,
      "logits/chosen": -0.3128906190395355,
      "logits/rejected": -0.3628906309604645,
      "logps/chosen": -306.79998779296875,
      "logps/rejected": -234.8000030517578,
      "loss": 0.6758,
      "rewards/accuracies": 0.48249998688697815,
      "rewards/chosen": 0.09648437798023224,
      "rewards/margins": 0.04770507663488388,
      "rewards/rejected": 0.04887695237994194,
      "step": 280
    },
    {
      "epoch": 0.07337796086508754,
      "grad_norm": 113.5,
      "learning_rate": 4.633110195674562e-07,
      "logits/chosen": -0.15664061903953552,
      "logits/rejected": -0.14321288466453552,
      "logps/chosen": -265.0,
      "logps/rejected": -232.8000030517578,
      "loss": 0.6914,
      "rewards/accuracies": 0.4090476632118225,
      "rewards/chosen": 0.05852050706744194,
      "rewards/margins": 0.008898925967514515,
      "rewards/rejected": 0.04962768405675888,
      "step": 285
    },
    {
      "epoch": 0.07466529351184346,
      "grad_norm": 110.5,
      "learning_rate": 4.6266735324407824e-07,
      "logits/chosen": -0.29863280057907104,
      "logits/rejected": -0.27167969942092896,
      "logps/chosen": -287.6000061035156,
      "logps/rejected": -285.3999938964844,
      "loss": 0.7,
      "rewards/accuracies": 0.32833331823349,
      "rewards/chosen": 0.05145873874425888,
      "rewards/margins": -0.0070434571243822575,
      "rewards/rejected": 0.058441162109375,
      "step": 290
    },
    {
      "epoch": 0.07595262615859938,
      "grad_norm": 137.0,
      "learning_rate": 4.620236869207003e-07,
      "logits/chosen": -0.37617188692092896,
      "logits/rejected": -0.3236328065395355,
      "logps/chosen": -318.79998779296875,
      "logps/rejected": -309.3999938964844,
      "loss": 0.65,
      "rewards/accuracies": 0.5894047617912292,
      "rewards/chosen": 0.14462891221046448,
      "rewards/margins": 0.09453125298023224,
      "rewards/rejected": 0.05019531399011612,
      "step": 295
    },
    {
      "epoch": 0.07723995880535531,
      "grad_norm": 110.0,
      "learning_rate": 4.613800205973223e-07,
      "logits/chosen": -0.32320863008499146,
      "logits/rejected": -0.24521484971046448,
      "logps/chosen": -286.79998779296875,
      "logps/rejected": -244.8000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.43695059418678284,
      "rewards/chosen": 0.09755859524011612,
      "rewards/margins": 0.039763640612363815,
      "rewards/rejected": 0.05781250074505806,
      "step": 300
    },
    {
      "epoch": 0.07852729145211122,
      "grad_norm": 117.0,
      "learning_rate": 4.6073635427394437e-07,
      "logits/chosen": -0.30195313692092896,
      "logits/rejected": -0.30888670682907104,
      "logps/chosen": -290.20001220703125,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6633,
      "rewards/accuracies": 0.44333332777023315,
      "rewards/chosen": 0.13457031548023224,
      "rewards/margins": 0.06778564304113388,
      "rewards/rejected": 0.06689453125,
      "step": 305
    },
    {
      "epoch": 0.07981462409886715,
      "grad_norm": 130.0,
      "learning_rate": 4.6009268795056636e-07,
      "logits/chosen": -0.28303223848342896,
      "logits/rejected": -0.31269532442092896,
      "logps/chosen": -282.3999938964844,
      "logps/rejected": -254.39999389648438,
      "loss": 0.675,
      "rewards/accuracies": 0.4665476381778717,
      "rewards/chosen": 0.10649414360523224,
      "rewards/margins": 0.04677734524011612,
      "rewards/rejected": 0.05970459058880806,
      "step": 310
    },
    {
      "epoch": 0.08110195674562307,
      "grad_norm": 193.0,
      "learning_rate": 4.5944902162718845e-07,
      "logits/chosen": -0.3783203065395355,
      "logits/rejected": -0.392578125,
      "logps/chosen": -241.60000610351562,
      "logps/rejected": -263.79998779296875,
      "loss": 0.6898,
      "rewards/accuracies": 0.395641028881073,
      "rewards/chosen": 0.07685546576976776,
      "rewards/margins": 0.0076660155318677425,
      "rewards/rejected": 0.06928710639476776,
      "step": 315
    },
    {
      "epoch": 0.082389289392379,
      "grad_norm": 116.5,
      "learning_rate": 4.588053553038105e-07,
      "logits/chosen": -0.36054688692092896,
      "logits/rejected": -0.3188548982143402,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -229.4499969482422,
      "loss": 0.6711,
      "rewards/accuracies": 0.40857142210006714,
      "rewards/chosen": 0.10114745795726776,
      "rewards/margins": 0.04750976711511612,
      "rewards/rejected": 0.05380859225988388,
      "step": 320
    },
    {
      "epoch": 0.08367662203913491,
      "grad_norm": 876.0,
      "learning_rate": 4.581616889804325e-07,
      "logits/chosen": -0.3716796934604645,
      "logits/rejected": -0.39335936307907104,
      "logps/chosen": -316.79998779296875,
      "logps/rejected": -286.20001220703125,
      "loss": 0.6906,
      "rewards/accuracies": 0.3819444477558136,
      "rewards/chosen": 0.04985351487994194,
      "rewards/margins": 0.01528320275247097,
      "rewards/rejected": 0.03452148288488388,
      "step": 325
    },
    {
      "epoch": 0.08496395468589084,
      "grad_norm": 150.0,
      "learning_rate": 4.575180226570546e-07,
      "logits/chosen": -0.37910157442092896,
      "logits/rejected": -0.353515625,
      "logps/chosen": -254.0,
      "logps/rejected": -233.39999389648438,
      "loss": 0.682,
      "rewards/accuracies": 0.47738099098205566,
      "rewards/chosen": 0.09931640326976776,
      "rewards/margins": 0.02673644945025444,
      "rewards/rejected": 0.07246093451976776,
      "step": 330
    },
    {
      "epoch": 0.08625128733264675,
      "grad_norm": 129.0,
      "learning_rate": 4.568743563336766e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.26459962129592896,
      "logps/chosen": -240.1999969482422,
      "logps/rejected": -178.1999969482422,
      "loss": 0.6781,
      "rewards/accuracies": 0.4568939805030823,
      "rewards/chosen": 0.1014404296875,
      "rewards/margins": 0.04270629957318306,
      "rewards/rejected": 0.05860137939453125,
      "step": 335
    },
    {
      "epoch": 0.08753861997940268,
      "grad_norm": 127.5,
      "learning_rate": 4.562306900102986e-07,
      "logits/chosen": -0.33125001192092896,
      "logits/rejected": -0.4312500059604645,
      "logps/chosen": -242.89999389648438,
      "logps/rejected": -236.3000030517578,
      "loss": 0.6836,
      "rewards/accuracies": 0.43714290857315063,
      "rewards/chosen": 0.09030761569738388,
      "rewards/margins": 0.02492675743997097,
      "rewards/rejected": 0.06524620205163956,
      "step": 340
    },
    {
      "epoch": 0.0888259526261586,
      "grad_norm": 129.0,
      "learning_rate": 4.555870236869207e-07,
      "logits/chosen": -0.44843751192092896,
      "logits/rejected": -0.21821288764476776,
      "logps/chosen": -173.0,
      "logps/rejected": -194.6999969482422,
      "loss": 0.6836,
      "rewards/accuracies": 0.41575756669044495,
      "rewards/chosen": 0.08901367336511612,
      "rewards/margins": 0.02823486365377903,
      "rewards/rejected": 0.06066284328699112,
      "step": 345
    },
    {
      "epoch": 0.09011328527291453,
      "grad_norm": 132.0,
      "learning_rate": 4.549433573635427e-07,
      "logits/chosen": -0.30018919706344604,
      "logits/rejected": -0.31196290254592896,
      "logps/chosen": -306.0,
      "logps/rejected": -246.1999969482422,
      "loss": 0.682,
      "rewards/accuracies": 0.4426373839378357,
      "rewards/chosen": 0.09794922173023224,
      "rewards/margins": 0.03073730506002903,
      "rewards/rejected": 0.06718750298023224,
      "step": 350
    },
    {
      "epoch": 0.09140061791967044,
      "grad_norm": 138.0,
      "learning_rate": 4.5429969104016474e-07,
      "logits/chosen": -0.23876953125,
      "logits/rejected": -0.20588378608226776,
      "logps/chosen": -282.0,
      "logps/rejected": -229.60000610351562,
      "loss": 0.6797,
      "rewards/accuracies": 0.4340476095676422,
      "rewards/chosen": 0.09998778998851776,
      "rewards/margins": 0.03602294996380806,
      "rewards/rejected": 0.06406249850988388,
      "step": 355
    },
    {
      "epoch": 0.09268795056642637,
      "grad_norm": 95.5,
      "learning_rate": 4.5365602471678684e-07,
      "logits/chosen": -0.47187501192092896,
      "logits/rejected": -0.39204102754592896,
      "logps/chosen": -311.20001220703125,
      "logps/rejected": -281.0,
      "loss": 0.6641,
      "rewards/accuracies": 0.41702383756637573,
      "rewards/chosen": 0.11994628608226776,
      "rewards/margins": 0.06040038913488388,
      "rewards/rejected": 0.05971984937787056,
      "step": 360
    },
    {
      "epoch": 0.09397528321318228,
      "grad_norm": 124.0,
      "learning_rate": 4.5301235839340883e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.4595703184604645,
      "logps/chosen": -345.6000061035156,
      "logps/rejected": -278.20001220703125,
      "loss": 0.6617,
      "rewards/accuracies": 0.52775639295578,
      "rewards/chosen": 0.12353515625,
      "rewards/margins": 0.07631530612707138,
      "rewards/rejected": 0.04737548902630806,
      "step": 365
    },
    {
      "epoch": 0.09526261585993821,
      "grad_norm": 148.0,
      "learning_rate": 4.5236869207003087e-07,
      "logits/chosen": -0.3232421875,
      "logits/rejected": -0.2992187440395355,
      "logps/chosen": -266.6000061035156,
      "logps/rejected": -241.60000610351562,
      "loss": 0.7023,
      "rewards/accuracies": 0.40772730112075806,
      "rewards/chosen": 0.02348632737994194,
      "rewards/margins": -0.013256835751235485,
      "rewards/rejected": 0.03648681566119194,
      "step": 370
    },
    {
      "epoch": 0.09654994850669413,
      "grad_norm": 105.5,
      "learning_rate": 4.517250257466529e-07,
      "logits/chosen": -0.34326171875,
      "logits/rejected": -0.31953126192092896,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -264.6000061035156,
      "loss": 0.6695,
      "rewards/accuracies": 0.40738096833229065,
      "rewards/chosen": 0.17900390923023224,
      "rewards/margins": 0.04904785007238388,
      "rewards/rejected": 0.1298828125,
      "step": 375
    },
    {
      "epoch": 0.09783728115345006,
      "grad_norm": 270.0,
      "learning_rate": 4.5108135942327496e-07,
      "logits/chosen": -0.4007812440395355,
      "logits/rejected": -0.34111326932907104,
      "logps/chosen": -280.20001220703125,
      "logps/rejected": -212.39999389648438,
      "loss": 0.6766,
      "rewards/accuracies": 0.5061111450195312,
      "rewards/chosen": 0.12363281100988388,
      "rewards/margins": 0.04926757887005806,
      "rewards/rejected": 0.07436523586511612,
      "step": 380
    },
    {
      "epoch": 0.09912461380020597,
      "grad_norm": 114.0,
      "learning_rate": 4.50437693099897e-07,
      "logits/chosen": -0.2958984375,
      "logits/rejected": -0.3265624940395355,
      "logps/chosen": -307.79998779296875,
      "logps/rejected": -239.6999969482422,
      "loss": 0.6711,
      "rewards/accuracies": 0.3800000250339508,
      "rewards/chosen": 0.09072265774011612,
      "rewards/margins": 0.0465087890625,
      "rewards/rejected": 0.044142257422208786,
      "step": 385
    },
    {
      "epoch": 0.1004119464469619,
      "grad_norm": 106.5,
      "learning_rate": 4.49794026776519e-07,
      "logits/chosen": -0.3597656190395355,
      "logits/rejected": -0.4136718809604645,
      "logps/chosen": -294.20001220703125,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6789,
      "rewards/accuracies": 0.5041667222976685,
      "rewards/chosen": 0.173828125,
      "rewards/margins": 0.04057617112994194,
      "rewards/rejected": 0.13359375298023224,
      "step": 390
    },
    {
      "epoch": 0.10169927909371781,
      "grad_norm": 153.0,
      "learning_rate": 4.491503604531411e-07,
      "logits/chosen": -0.245361328125,
      "logits/rejected": -0.19101563096046448,
      "logps/chosen": -291.0,
      "logps/rejected": -208.8000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.38333335518836975,
      "rewards/chosen": 0.125244140625,
      "rewards/margins": 0.04741210862994194,
      "rewards/rejected": 0.07755737006664276,
      "step": 395
    },
    {
      "epoch": 0.10298661174047374,
      "grad_norm": 130.0,
      "learning_rate": 4.4850669412976313e-07,
      "logits/chosen": -0.13212890923023224,
      "logits/rejected": -0.14082030951976776,
      "logps/chosen": -253.8000030517578,
      "logps/rejected": -251.1999969482422,
      "loss": 0.6773,
      "rewards/accuracies": 0.5379945039749146,
      "rewards/chosen": 0.0941162109375,
      "rewards/margins": 0.0386962890625,
      "rewards/rejected": 0.05556640774011612,
      "step": 400
    },
    {
      "epoch": 0.10427394438722966,
      "grad_norm": 118.0,
      "learning_rate": 4.478630278063851e-07,
      "logits/chosen": -0.296630859375,
      "logits/rejected": -0.3086914122104645,
      "logps/chosen": -280.79998779296875,
      "logps/rejected": -265.20001220703125,
      "loss": 0.682,
      "rewards/accuracies": 0.40430402755737305,
      "rewards/chosen": 0.13750000298023224,
      "rewards/margins": 0.028318310156464577,
      "rewards/rejected": 0.10947265475988388,
      "step": 405
    },
    {
      "epoch": 0.10556127703398559,
      "grad_norm": 195.0,
      "learning_rate": 4.472193614830072e-07,
      "logits/chosen": -0.3490234315395355,
      "logits/rejected": -0.36250001192092896,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -236.60000610351562,
      "loss": 0.6594,
      "rewards/accuracies": 0.5233333706855774,
      "rewards/chosen": 0.126423642039299,
      "rewards/margins": 0.06816406548023224,
      "rewards/rejected": 0.05830078199505806,
      "step": 410
    },
    {
      "epoch": 0.1068486096807415,
      "grad_norm": 129.0,
      "learning_rate": 4.4657569515962926e-07,
      "logits/chosen": -0.22622069716453552,
      "logits/rejected": -0.3119140565395355,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -257.79998779296875,
      "loss": 0.6609,
      "rewards/accuracies": 0.503333330154419,
      "rewards/chosen": 0.10873260349035263,
      "rewards/margins": 0.0811767578125,
      "rewards/rejected": 0.02741699293255806,
      "step": 415
    },
    {
      "epoch": 0.10813594232749743,
      "grad_norm": 119.5,
      "learning_rate": 4.4593202883625124e-07,
      "logits/chosen": -0.2835937440395355,
      "logits/rejected": -0.3447265625,
      "logps/chosen": -267.79998779296875,
      "logps/rejected": -242.1999969482422,
      "loss": 0.6773,
      "rewards/accuracies": 0.5398809313774109,
      "rewards/chosen": 0.15517577528953552,
      "rewards/margins": 0.0408935546875,
      "rewards/rejected": 0.11445312201976776,
      "step": 420
    },
    {
      "epoch": 0.10942327497425335,
      "grad_norm": 133.0,
      "learning_rate": 4.4528836251287334e-07,
      "logits/chosen": -0.28266602754592896,
      "logits/rejected": -0.37250977754592896,
      "logps/chosen": -272.20001220703125,
      "logps/rejected": -275.0,
      "loss": 0.6781,
      "rewards/accuracies": 0.3938094973564148,
      "rewards/chosen": 0.10258789360523224,
      "rewards/margins": 0.04365234449505806,
      "rewards/rejected": 0.05888671800494194,
      "step": 425
    },
    {
      "epoch": 0.11071060762100927,
      "grad_norm": 220.0,
      "learning_rate": 4.4464469618949533e-07,
      "logits/chosen": -0.35859376192092896,
      "logits/rejected": -0.36054688692092896,
      "logps/chosen": -239.1999969482422,
      "logps/rejected": -245.0,
      "loss": 0.6602,
      "rewards/accuracies": 0.5527380704879761,
      "rewards/chosen": 0.15625,
      "rewards/margins": 0.07259521633386612,
      "rewards/rejected": 0.08378906548023224,
      "step": 430
    },
    {
      "epoch": 0.11199794026776519,
      "grad_norm": 118.0,
      "learning_rate": 4.4400102986611737e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.3890624940395355,
      "logps/chosen": -304.0,
      "logps/rejected": -281.20001220703125,
      "loss": 0.6633,
      "rewards/accuracies": 0.4933333396911621,
      "rewards/chosen": 0.14462891221046448,
      "rewards/margins": 0.07026366889476776,
      "rewards/rejected": 0.074249267578125,
      "step": 435
    },
    {
      "epoch": 0.11328527291452112,
      "grad_norm": 162.0,
      "learning_rate": 4.4335736354273947e-07,
      "logits/chosen": -0.30546873807907104,
      "logits/rejected": -0.3716796934604645,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -272.79998779296875,
      "loss": 0.6984,
      "rewards/accuracies": 0.4079762101173401,
      "rewards/chosen": 0.11923827975988388,
      "rewards/margins": 0.0006240844959393144,
      "rewards/rejected": 0.11870117485523224,
      "step": 440
    },
    {
      "epoch": 0.11457260556127703,
      "grad_norm": 103.0,
      "learning_rate": 4.4271369721936146e-07,
      "logits/chosen": -0.29082030057907104,
      "logits/rejected": -0.29462891817092896,
      "logps/chosen": -244.0,
      "logps/rejected": -222.60000610351562,
      "loss": 0.6664,
      "rewards/accuracies": 0.5573717951774597,
      "rewards/chosen": 0.14169922471046448,
      "rewards/margins": 0.06517334282398224,
      "rewards/rejected": 0.076629638671875,
      "step": 445
    },
    {
      "epoch": 0.11585993820803296,
      "grad_norm": 101.5,
      "learning_rate": 4.420700308959835e-07,
      "logits/chosen": -0.32304686307907104,
      "logits/rejected": -0.39375001192092896,
      "logps/chosen": -278.79998779296875,
      "logps/rejected": -298.6000061035156,
      "loss": 0.6781,
      "rewards/accuracies": 0.40583333373069763,
      "rewards/chosen": 0.1217041015625,
      "rewards/margins": 0.03781170770525932,
      "rewards/rejected": 0.08381347358226776,
      "step": 450
    },
    {
      "epoch": 0.11714727085478888,
      "grad_norm": 101.0,
      "learning_rate": 4.414263645726055e-07,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.43632811307907104,
      "logps/chosen": -267.3999938964844,
      "logps/rejected": -250.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.14199218153953552,
      "rewards/margins": 0.08610840141773224,
      "rewards/rejected": 0.05574951320886612,
      "step": 455
    },
    {
      "epoch": 0.1184346035015448,
      "grad_norm": 117.5,
      "learning_rate": 4.407826982492276e-07,
      "logits/chosen": -0.3349609375,
      "logits/rejected": -0.3360351622104645,
      "logps/chosen": -261.3999938964844,
      "logps/rejected": -251.1999969482422,
      "loss": 0.6766,
      "rewards/accuracies": 0.5158333778381348,
      "rewards/chosen": 0.10458984225988388,
      "rewards/margins": 0.04282226413488388,
      "rewards/rejected": 0.061767578125,
      "step": 460
    },
    {
      "epoch": 0.11972193614830072,
      "grad_norm": 116.0,
      "learning_rate": 4.4013903192584963e-07,
      "logits/chosen": -0.2945312559604645,
      "logits/rejected": -0.17324218153953552,
      "logps/chosen": -241.1999969482422,
      "logps/rejected": -243.1999969482422,
      "loss": 0.6648,
      "rewards/accuracies": 0.48257574439048767,
      "rewards/chosen": 0.13331298530101776,
      "rewards/margins": 0.05538330227136612,
      "rewards/rejected": 0.07792969048023224,
      "step": 465
    },
    {
      "epoch": 0.12100926879505665,
      "grad_norm": 111.0,
      "learning_rate": 4.394953656024716e-07,
      "logits/chosen": -0.3369140625,
      "logits/rejected": -0.2525390684604645,
      "logps/chosen": -298.0,
      "logps/rejected": -253.60000610351562,
      "loss": 0.675,
      "rewards/accuracies": 0.3754761815071106,
      "rewards/chosen": 0.117431640625,
      "rewards/margins": 0.04523925855755806,
      "rewards/rejected": 0.07229004055261612,
      "step": 470
    },
    {
      "epoch": 0.12229660144181256,
      "grad_norm": 79.0,
      "learning_rate": 4.388516992790937e-07,
      "logits/chosen": -0.3910156190395355,
      "logits/rejected": -0.3910156190395355,
      "logps/chosen": -231.1999969482422,
      "logps/rejected": -260.6000061035156,
      "loss": 0.6766,
      "rewards/accuracies": 0.5225000381469727,
      "rewards/chosen": 0.13876953721046448,
      "rewards/margins": 0.04416503757238388,
      "rewards/rejected": 0.09458007663488388,
      "step": 475
    },
    {
      "epoch": 0.12358393408856849,
      "grad_norm": 155.0,
      "learning_rate": 4.3820803295571576e-07,
      "logits/chosen": -0.24013671278953552,
      "logits/rejected": -0.3184570372104645,
      "logps/chosen": -291.70001220703125,
      "logps/rejected": -243.8000030517578,
      "loss": 0.6516,
      "rewards/accuracies": 0.536309540271759,
      "rewards/chosen": 0.15625,
      "rewards/margins": 0.09111328423023224,
      "rewards/rejected": 0.06496582180261612,
      "step": 480
    },
    {
      "epoch": 0.12487126673532441,
      "grad_norm": 121.5,
      "learning_rate": 4.3756436663233775e-07,
      "logits/chosen": -0.3466796875,
      "logits/rejected": -0.3921875059604645,
      "logps/chosen": -292.0,
      "logps/rejected": -300.79998779296875,
      "loss": 0.6813,
      "rewards/accuracies": 0.463203489780426,
      "rewards/chosen": 0.13657227158546448,
      "rewards/margins": 0.041015625,
      "rewards/rejected": 0.09580077975988388,
      "step": 485
    },
    {
      "epoch": 0.12615859938208032,
      "grad_norm": 112.5,
      "learning_rate": 4.3692070030895984e-07,
      "logits/chosen": -0.24404296278953552,
      "logits/rejected": -0.2998046875,
      "logps/chosen": -222.60000610351562,
      "logps/rejected": -286.0,
      "loss": 0.6859,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.10292968899011612,
      "rewards/margins": 0.02664794959127903,
      "rewards/rejected": 0.07622070610523224,
      "step": 490
    },
    {
      "epoch": 0.12744593202883625,
      "grad_norm": 106.0,
      "learning_rate": 4.3627703398558183e-07,
      "logits/chosen": -0.2515625059604645,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -238.1999969482422,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6508,
      "rewards/accuracies": 0.6241666674613953,
      "rewards/chosen": 0.15048828721046448,
      "rewards/margins": 0.09477539360523224,
      "rewards/rejected": 0.0558319091796875,
      "step": 495
    },
    {
      "epoch": 0.12873326467559218,
      "grad_norm": 103.0,
      "learning_rate": 4.356333676622039e-07,
      "logits/chosen": -0.204833984375,
      "logits/rejected": -0.08637695014476776,
      "logps/chosen": -257.79998779296875,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6625,
      "rewards/accuracies": 0.5307143330574036,
      "rewards/chosen": 0.141845703125,
      "rewards/margins": 0.06557617336511612,
      "rewards/rejected": 0.07624511420726776,
      "step": 500
    },
    {
      "epoch": 0.1300205973223481,
      "grad_norm": 89.5,
      "learning_rate": 4.3498970133882597e-07,
      "logits/chosen": -0.3343749940395355,
      "logits/rejected": -0.3896484375,
      "logps/chosen": -273.3999938964844,
      "logps/rejected": -234.0,
      "loss": 0.6734,
      "rewards/accuracies": 0.4425000250339508,
      "rewards/chosen": 0.10122070461511612,
      "rewards/margins": 0.03066406212747097,
      "rewards/rejected": 0.070556640625,
      "step": 505
    },
    {
      "epoch": 0.131307929969104,
      "grad_norm": 111.0,
      "learning_rate": 4.3434603501544796e-07,
      "logits/chosen": -0.4468750059604645,
      "logits/rejected": -0.42656248807907104,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -232.39999389648438,
      "loss": 0.6687,
      "rewards/accuracies": 0.5601190328598022,
      "rewards/chosen": 0.23281249403953552,
      "rewards/margins": 0.063018798828125,
      "rewards/rejected": 0.16933593153953552,
      "step": 510
    },
    {
      "epoch": 0.13259526261585994,
      "grad_norm": 110.5,
      "learning_rate": 4.3370236869207e-07,
      "logits/chosen": -0.3232421875,
      "logits/rejected": -0.30097657442092896,
      "logps/chosen": -253.60000610351562,
      "logps/rejected": -235.1999969482422,
      "loss": 0.6602,
      "rewards/accuracies": 0.46087661385536194,
      "rewards/chosen": 0.135009765625,
      "rewards/margins": 0.08115234225988388,
      "rewards/rejected": 0.0538330078125,
      "step": 515
    },
    {
      "epoch": 0.13388259526261587,
      "grad_norm": 134.0,
      "learning_rate": 4.330587023686921e-07,
      "logits/chosen": -0.25175780057907104,
      "logits/rejected": -0.38691407442092896,
      "logps/chosen": -266.0,
      "logps/rejected": -284.6000061035156,
      "loss": 0.6578,
      "rewards/accuracies": 0.4739743769168854,
      "rewards/chosen": 0.15839843451976776,
      "rewards/margins": 0.07583007961511612,
      "rewards/rejected": 0.08266601711511612,
      "step": 520
    },
    {
      "epoch": 0.1351699279093718,
      "grad_norm": 153.0,
      "learning_rate": 4.324150360453141e-07,
      "logits/chosen": -0.3954101502895355,
      "logits/rejected": -0.36308592557907104,
      "logps/chosen": -329.0,
      "logps/rejected": -248.60000610351562,
      "loss": 0.6953,
      "rewards/accuracies": 0.40714287757873535,
      "rewards/chosen": 0.12017822265625,
      "rewards/margins": 0.01132812537252903,
      "rewards/rejected": 0.10888671875,
      "step": 525
    },
    {
      "epoch": 0.1364572605561277,
      "grad_norm": 191.0,
      "learning_rate": 4.3177136972193613e-07,
      "logits/chosen": -0.37646484375,
      "logits/rejected": -0.3157714903354645,
      "logps/chosen": -268.0,
      "logps/rejected": -245.0,
      "loss": 0.6703,
      "rewards/accuracies": 0.4258333146572113,
      "rewards/chosen": 0.15595702826976776,
      "rewards/margins": 0.06674804538488388,
      "rewards/rejected": 0.08939208835363388,
      "step": 530
    },
    {
      "epoch": 0.13774459320288363,
      "grad_norm": 136.0,
      "learning_rate": 4.311277033985581e-07,
      "logits/chosen": -0.29035645723342896,
      "logits/rejected": -0.28203123807907104,
      "logps/chosen": -276.3999938964844,
      "logps/rejected": -290.20001220703125,
      "loss": 0.6781,
      "rewards/accuracies": 0.47083336114883423,
      "rewards/chosen": 0.10344238579273224,
      "rewards/margins": 0.03367919847369194,
      "rewards/rejected": 0.06997070461511612,
      "step": 535
    },
    {
      "epoch": 0.13903192584963955,
      "grad_norm": 102.5,
      "learning_rate": 4.304840370751802e-07,
      "logits/chosen": -0.18828125298023224,
      "logits/rejected": -0.00937500037252903,
      "logps/chosen": -240.64999389648438,
      "logps/rejected": -258.6000061035156,
      "loss": 0.6852,
      "rewards/accuracies": 0.37833333015441895,
      "rewards/chosen": 0.14628906548023224,
      "rewards/margins": 0.0044799805618822575,
      "rewards/rejected": 0.14174804091453552,
      "step": 540
    },
    {
      "epoch": 0.14031925849639545,
      "grad_norm": 101.0,
      "learning_rate": 4.2984037075180226e-07,
      "logits/chosen": -0.22578124701976776,
      "logits/rejected": -0.23643799126148224,
      "logps/chosen": -263.20001220703125,
      "logps/rejected": -243.1999969482422,
      "loss": 0.6633,
      "rewards/accuracies": 0.4399999976158142,
      "rewards/chosen": 0.12636718153953552,
      "rewards/margins": 0.06252441555261612,
      "rewards/rejected": 0.06374511867761612,
      "step": 545
    },
    {
      "epoch": 0.14160659114315138,
      "grad_norm": 129.0,
      "learning_rate": 4.2919670442842425e-07,
      "logits/chosen": -0.4937500059604645,
      "logits/rejected": -0.528124988079071,
      "logps/chosen": -292.20001220703125,
      "logps/rejected": -237.8000030517578,
      "loss": 0.6758,
      "rewards/accuracies": 0.48357143998146057,
      "rewards/chosen": 0.17993775010108948,
      "rewards/margins": 0.04658203199505806,
      "rewards/rejected": 0.13349609076976776,
      "step": 550
    },
    {
      "epoch": 0.1428939237899073,
      "grad_norm": 119.0,
      "learning_rate": 4.2855303810504634e-07,
      "logits/chosen": -0.3779296875,
      "logits/rejected": -0.419921875,
      "logps/chosen": -238.0,
      "logps/rejected": -228.60000610351562,
      "loss": 0.6875,
      "rewards/accuracies": 0.45654764771461487,
      "rewards/chosen": 0.0877685546875,
      "rewards/margins": 0.01163330115377903,
      "rewards/rejected": 0.07619629055261612,
      "step": 555
    },
    {
      "epoch": 0.14418125643666324,
      "grad_norm": 114.0,
      "learning_rate": 4.2790937178166833e-07,
      "logits/chosen": -0.31367188692092896,
      "logits/rejected": -0.27851563692092896,
      "logps/chosen": -253.8000030517578,
      "logps/rejected": -237.5,
      "loss": 0.6477,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": 0.16621093451976776,
      "rewards/margins": 0.10917969048023224,
      "rewards/rejected": 0.05687255784869194,
      "step": 560
    },
    {
      "epoch": 0.14546858908341914,
      "grad_norm": 242.0,
      "learning_rate": 4.272657054582904e-07,
      "logits/chosen": -0.3031249940395355,
      "logits/rejected": -0.25859373807907104,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -248.8000030517578,
      "loss": 0.6508,
      "rewards/accuracies": 0.6150641441345215,
      "rewards/chosen": 0.16328124701976776,
      "rewards/margins": 0.08270263671875,
      "rewards/rejected": 0.08071289211511612,
      "step": 565
    },
    {
      "epoch": 0.14675592173017507,
      "grad_norm": 155.0,
      "learning_rate": 4.2662203913491247e-07,
      "logits/chosen": -0.2660156190395355,
      "logits/rejected": -0.11984863132238388,
      "logps/chosen": -255.60000610351562,
      "logps/rejected": -226.1999969482422,
      "loss": 0.6602,
      "rewards/accuracies": 0.47198718786239624,
      "rewards/chosen": 0.14913025498390198,
      "rewards/margins": 0.08122558891773224,
      "rewards/rejected": 0.06791992485523224,
      "step": 570
    },
    {
      "epoch": 0.148043254376931,
      "grad_norm": 141.0,
      "learning_rate": 4.2597837281153446e-07,
      "logits/chosen": -0.32246094942092896,
      "logits/rejected": -0.3486328125,
      "logps/chosen": -237.8000030517578,
      "logps/rejected": -283.6000061035156,
      "loss": 0.682,
      "rewards/accuracies": 0.5458333492279053,
      "rewards/chosen": 0.14199218153953552,
      "rewards/margins": 0.039031982421875,
      "rewards/rejected": 0.10268554836511612,
      "step": 575
    },
    {
      "epoch": 0.14933058702368693,
      "grad_norm": 125.5,
      "learning_rate": 4.253347064881565e-07,
      "logits/chosen": -0.48945313692092896,
      "logits/rejected": -0.34980469942092896,
      "logps/chosen": -304.0,
      "logps/rejected": -290.20001220703125,
      "loss": 0.6484,
      "rewards/accuracies": 0.5191666483879089,
      "rewards/chosen": 0.1611328125,
      "rewards/margins": 0.10463867336511612,
      "rewards/rejected": 0.05644531175494194,
      "step": 580
    },
    {
      "epoch": 0.15061791967044283,
      "grad_norm": 119.0,
      "learning_rate": 4.246910401647786e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.34765625,
      "logps/chosen": -317.6000061035156,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6609,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.16982421278953552,
      "rewards/margins": 0.08895263820886612,
      "rewards/rejected": 0.081298828125,
      "step": 585
    },
    {
      "epoch": 0.15190525231719876,
      "grad_norm": 154.0,
      "learning_rate": 4.240473738414006e-07,
      "logits/chosen": -0.3501953184604645,
      "logits/rejected": -0.34980469942092896,
      "logps/chosen": -326.0,
      "logps/rejected": -282.0,
      "loss": 0.6891,
      "rewards/accuracies": 0.34583336114883423,
      "rewards/chosen": 0.10225830227136612,
      "rewards/margins": 0.013934326358139515,
      "rewards/rejected": 0.08832397311925888,
      "step": 590
    },
    {
      "epoch": 0.1531925849639547,
      "grad_norm": 128.0,
      "learning_rate": 4.2340370751802263e-07,
      "logits/chosen": -0.38232421875,
      "logits/rejected": -0.4652343690395355,
      "logps/chosen": -312.20001220703125,
      "logps/rejected": -255.1999969482422,
      "loss": 0.6555,
      "rewards/accuracies": 0.5378205180168152,
      "rewards/chosen": 0.15214844048023224,
      "rewards/margins": 0.08737792819738388,
      "rewards/rejected": 0.06495361030101776,
      "step": 595
    },
    {
      "epoch": 0.15447991761071062,
      "grad_norm": 100.0,
      "learning_rate": 4.227600411946447e-07,
      "logits/chosen": -0.482421875,
      "logits/rejected": -0.50390625,
      "logps/chosen": -265.0,
      "logps/rejected": -199.5,
      "loss": 0.6672,
      "rewards/accuracies": 0.46916666626930237,
      "rewards/chosen": 0.16562500596046448,
      "rewards/margins": 0.05397949367761612,
      "rewards/rejected": 0.11186523735523224,
      "step": 600
    },
    {
      "epoch": 0.15576725025746652,
      "grad_norm": 99.0,
      "learning_rate": 4.221163748712667e-07,
      "logits/chosen": -0.4203124940395355,
      "logits/rejected": -0.3714843690395355,
      "logps/chosen": -353.6000061035156,
      "logps/rejected": -294.79998779296875,
      "loss": 0.6406,
      "rewards/accuracies": 0.5460714101791382,
      "rewards/chosen": 0.228515625,
      "rewards/margins": 0.12099609524011612,
      "rewards/rejected": 0.10795898735523224,
      "step": 605
    },
    {
      "epoch": 0.15705458290422245,
      "grad_norm": 129.0,
      "learning_rate": 4.2147270854788876e-07,
      "logits/chosen": -0.24912109971046448,
      "logits/rejected": -0.23222656548023224,
      "logps/chosen": -244.6999969482422,
      "logps/rejected": -221.39999389648438,
      "loss": 0.668,
      "rewards/accuracies": 0.5949999690055847,
      "rewards/chosen": 0.11704101413488388,
      "rewards/margins": 0.06101074069738388,
      "rewards/rejected": 0.055908203125,
      "step": 610
    },
    {
      "epoch": 0.15834191555097837,
      "grad_norm": 221.0,
      "learning_rate": 4.208290422245108e-07,
      "logits/chosen": -0.43085938692092896,
      "logits/rejected": -0.3628906309604645,
      "logps/chosen": -292.3999938964844,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6633,
      "rewards/accuracies": 0.522857129573822,
      "rewards/chosen": 0.19423827528953552,
      "rewards/margins": 0.08218993991613388,
      "rewards/rejected": 0.11196289211511612,
      "step": 615
    },
    {
      "epoch": 0.1596292481977343,
      "grad_norm": 110.0,
      "learning_rate": 4.2018537590113285e-07,
      "logits/chosen": -0.25927734375,
      "logits/rejected": -0.39140623807907104,
      "logps/chosen": -230.1999969482422,
      "logps/rejected": -286.3999938964844,
      "loss": 0.6656,
      "rewards/accuracies": 0.5276282429695129,
      "rewards/chosen": 0.15732422471046448,
      "rewards/margins": 0.05512695387005806,
      "rewards/rejected": 0.10205078125,
      "step": 620
    },
    {
      "epoch": 0.1609165808444902,
      "grad_norm": 96.5,
      "learning_rate": 4.195417095777549e-07,
      "logits/chosen": -0.35468751192092896,
      "logits/rejected": -0.29218751192092896,
      "logps/chosen": -272.0,
      "logps/rejected": -239.60000610351562,
      "loss": 0.6648,
      "rewards/accuracies": 0.4357143044471741,
      "rewards/chosen": 0.15302734076976776,
      "rewards/margins": 0.06376953423023224,
      "rewards/rejected": 0.08974609524011612,
      "step": 625
    },
    {
      "epoch": 0.16220391349124613,
      "grad_norm": 120.5,
      "learning_rate": 4.188980432543769e-07,
      "logits/chosen": -0.3050781190395355,
      "logits/rejected": -0.24277344346046448,
      "logps/chosen": -245.1999969482422,
      "logps/rejected": -190.39999389648438,
      "loss": 0.6594,
      "rewards/accuracies": 0.4795454442501068,
      "rewards/chosen": 0.1572265625,
      "rewards/margins": 0.0869140625,
      "rewards/rejected": 0.0704345703125,
      "step": 630
    },
    {
      "epoch": 0.16349124613800206,
      "grad_norm": 120.5,
      "learning_rate": 4.18254376930999e-07,
      "logits/chosen": -0.3193359375,
      "logits/rejected": -0.2962402403354645,
      "logps/chosen": -286.3999938964844,
      "logps/rejected": -270.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.47083336114883423,
      "rewards/chosen": 0.16132812201976776,
      "rewards/margins": 0.0609130859375,
      "rewards/rejected": 0.10029296576976776,
      "step": 635
    },
    {
      "epoch": 0.164778578784758,
      "grad_norm": 174.0,
      "learning_rate": 4.1761071060762096e-07,
      "logits/chosen": -0.37480467557907104,
      "logits/rejected": -0.30976563692092896,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -250.39999389648438,
      "loss": 0.6727,
      "rewards/accuracies": 0.45826926827430725,
      "rewards/chosen": 0.15034179389476776,
      "rewards/margins": 0.05227050930261612,
      "rewards/rejected": 0.09825439751148224,
      "step": 640
    },
    {
      "epoch": 0.1660659114315139,
      "grad_norm": 201.0,
      "learning_rate": 4.16967044284243e-07,
      "logits/chosen": -0.23974609375,
      "logits/rejected": -0.21904297173023224,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -261.0,
      "loss": 0.6813,
      "rewards/accuracies": 0.5214285850524902,
      "rewards/chosen": 0.16428223252296448,
      "rewards/margins": 0.0397491455078125,
      "rewards/rejected": 0.12453613430261612,
      "step": 645
    },
    {
      "epoch": 0.16735324407826982,
      "grad_norm": 133.0,
      "learning_rate": 4.163233779608651e-07,
      "logits/chosen": -0.29523926973342896,
      "logits/rejected": -0.44316405057907104,
      "logps/chosen": -297.6000061035156,
      "logps/rejected": -285.79998779296875,
      "loss": 0.6578,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.12954100966453552,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": 0.04465331882238388,
      "step": 650
    },
    {
      "epoch": 0.16864057672502575,
      "grad_norm": 129.0,
      "learning_rate": 4.156797116374871e-07,
      "logits/chosen": -0.24628905951976776,
      "logits/rejected": -0.19365234673023224,
      "logps/chosen": -214.3000030517578,
      "logps/rejected": -212.3000030517578,
      "loss": 0.6937,
      "rewards/accuracies": 0.44641607999801636,
      "rewards/chosen": 0.03612976148724556,
      "rewards/margins": 0.004467773251235485,
      "rewards/rejected": 0.03159179538488388,
      "step": 655
    },
    {
      "epoch": 0.16992790937178168,
      "grad_norm": 147.0,
      "learning_rate": 4.1503604531410913e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.4222656190395355,
      "logps/chosen": -325.79998779296875,
      "logps/rejected": -261.3999938964844,
      "loss": 0.6617,
      "rewards/accuracies": 0.5057142972946167,
      "rewards/chosen": 0.18359375,
      "rewards/margins": 0.09085693210363388,
      "rewards/rejected": 0.09290771186351776,
      "step": 660
    },
    {
      "epoch": 0.17121524201853758,
      "grad_norm": 103.0,
      "learning_rate": 4.1439237899073123e-07,
      "logits/chosen": -0.16132812201976776,
      "logits/rejected": -0.2544921934604645,
      "logps/chosen": -256.20001220703125,
      "logps/rejected": -290.20001220703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.4960714280605316,
      "rewards/chosen": 0.20644530653953552,
      "rewards/margins": 0.05502929538488388,
      "rewards/rejected": 0.1513671875,
      "step": 665
    },
    {
      "epoch": 0.1725025746652935,
      "grad_norm": 744.0,
      "learning_rate": 4.137487126673532e-07,
      "logits/chosen": -0.3648437559604645,
      "logits/rejected": -0.39765626192092896,
      "logps/chosen": -281.0,
      "logps/rejected": -266.0,
      "loss": 0.675,
      "rewards/accuracies": 0.5649999976158142,
      "rewards/chosen": 0.0962982177734375,
      "rewards/margins": 0.05128173902630806,
      "rewards/rejected": 0.044921875,
      "step": 670
    },
    {
      "epoch": 0.17378990731204944,
      "grad_norm": 168.0,
      "learning_rate": 4.1310504634397526e-07,
      "logits/chosen": -0.3289550840854645,
      "logits/rejected": -0.3736328184604645,
      "logps/chosen": -325.20001220703125,
      "logps/rejected": -237.1999969482422,
      "loss": 0.6508,
      "rewards/accuracies": 0.540238082408905,
      "rewards/chosen": 0.17128905653953552,
      "rewards/margins": 0.09218750149011612,
      "rewards/rejected": 0.07900390774011612,
      "step": 675
    },
    {
      "epoch": 0.17507723995880536,
      "grad_norm": 220.0,
      "learning_rate": 4.124613800205973e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.287109375,
      "logps/chosen": -312.6000061035156,
      "logps/rejected": -329.20001220703125,
      "loss": 0.6492,
      "rewards/accuracies": 0.5910714268684387,
      "rewards/chosen": 0.21904297173023224,
      "rewards/margins": 0.11318359524011612,
      "rewards/rejected": 0.10612182319164276,
      "step": 680
    },
    {
      "epoch": 0.17636457260556127,
      "grad_norm": 488.0,
      "learning_rate": 4.1181771369721935e-07,
      "logits/chosen": -0.30546873807907104,
      "logits/rejected": -0.24223633110523224,
      "logps/chosen": -305.3999938964844,
      "logps/rejected": -240.1999969482422,
      "loss": 0.6492,
      "rewards/accuracies": 0.5900000333786011,
      "rewards/chosen": 0.21552734076976776,
      "rewards/margins": 0.1005859375,
      "rewards/rejected": 0.11513672024011612,
      "step": 685
    },
    {
      "epoch": 0.1776519052523172,
      "grad_norm": 116.5,
      "learning_rate": 4.111740473738414e-07,
      "logits/chosen": -0.4369140565395355,
      "logits/rejected": -0.38330078125,
      "logps/chosen": -297.0,
      "logps/rejected": -314.0,
      "loss": 0.6891,
      "rewards/accuracies": 0.41333335638046265,
      "rewards/chosen": 0.10854492336511612,
      "rewards/margins": 0.01640624925494194,
      "rewards/rejected": 0.09233398735523224,
      "step": 690
    },
    {
      "epoch": 0.17893923789907312,
      "grad_norm": 170.0,
      "learning_rate": 4.1053038105046343e-07,
      "logits/chosen": -0.4144531190395355,
      "logits/rejected": -0.39667969942092896,
      "logps/chosen": -340.0,
      "logps/rejected": -288.20001220703125,
      "loss": 0.6586,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.16054686903953552,
      "rewards/margins": 0.08491210639476776,
      "rewards/rejected": 0.07558593899011612,
      "step": 695
    },
    {
      "epoch": 0.18022657054582905,
      "grad_norm": 208.0,
      "learning_rate": 4.098867147270855e-07,
      "logits/chosen": -0.32539063692092896,
      "logits/rejected": -0.443359375,
      "logps/chosen": -311.0,
      "logps/rejected": -280.20001220703125,
      "loss": 0.6859,
      "rewards/accuracies": 0.4219047427177429,
      "rewards/chosen": 0.10305175930261612,
      "rewards/margins": 0.01636657677590847,
      "rewards/rejected": 0.08671875298023224,
      "step": 700
    },
    {
      "epoch": 0.18151390319258495,
      "grad_norm": 125.0,
      "learning_rate": 4.0924304840370747e-07,
      "logits/chosen": -0.3331054747104645,
      "logits/rejected": -0.3743652403354645,
      "logps/chosen": -284.3999938964844,
      "logps/rejected": -280.79998779296875,
      "loss": 0.6648,
      "rewards/accuracies": 0.5049999952316284,
      "rewards/chosen": 0.18193359673023224,
      "rewards/margins": 0.06832275539636612,
      "rewards/rejected": 0.11362304538488388,
      "step": 705
    },
    {
      "epoch": 0.18280123583934088,
      "grad_norm": 185.0,
      "learning_rate": 4.085993820803295e-07,
      "logits/chosen": -0.3492187559604645,
      "logits/rejected": -0.39335936307907104,
      "logps/chosen": -300.6000061035156,
      "logps/rejected": -280.0,
      "loss": 0.6828,
      "rewards/accuracies": 0.4446428716182709,
      "rewards/chosen": 0.12045898288488388,
      "rewards/margins": 0.03580932691693306,
      "rewards/rejected": 0.08464355766773224,
      "step": 710
    },
    {
      "epoch": 0.1840885684860968,
      "grad_norm": 205.0,
      "learning_rate": 4.079557157569516e-07,
      "logits/chosen": -0.2769531309604645,
      "logits/rejected": -0.419921875,
      "logps/chosen": -219.9499969482422,
      "logps/rejected": -228.60000610351562,
      "loss": 0.6742,
      "rewards/accuracies": 0.46071428060531616,
      "rewards/chosen": 0.08412475883960724,
      "rewards/margins": 0.06931152194738388,
      "rewards/rejected": 0.014941406436264515,
      "step": 715
    },
    {
      "epoch": 0.18537590113285274,
      "grad_norm": 167.0,
      "learning_rate": 4.073120494335736e-07,
      "logits/chosen": -0.544921875,
      "logits/rejected": -0.545703113079071,
      "logps/chosen": -298.79998779296875,
      "logps/rejected": -257.3999938964844,
      "loss": 0.6547,
      "rewards/accuracies": 0.5575000047683716,
      "rewards/chosen": 0.18378905951976776,
      "rewards/margins": 0.09663085639476776,
      "rewards/rejected": 0.08697509765625,
      "step": 720
    },
    {
      "epoch": 0.18666323377960864,
      "grad_norm": 282.0,
      "learning_rate": 4.0666838311019564e-07,
      "logits/chosen": -0.21870116889476776,
      "logits/rejected": -0.13134765625,
      "logps/chosen": -274.79998779296875,
      "logps/rejected": -236.39999389648438,
      "loss": 0.6422,
      "rewards/accuracies": 0.6043590307235718,
      "rewards/chosen": 0.14316406846046448,
      "rewards/margins": 0.11789550632238388,
      "rewards/rejected": 0.02501220628619194,
      "step": 725
    },
    {
      "epoch": 0.18795056642636457,
      "grad_norm": 208.0,
      "learning_rate": 4.0602471678681773e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": -0.17197266221046448,
      "logps/chosen": -273.0,
      "logps/rejected": -279.3999938964844,
      "loss": 0.6641,
      "rewards/accuracies": 0.5508333444595337,
      "rewards/chosen": 0.18417969346046448,
      "rewards/margins": 0.07102050632238388,
      "rewards/rejected": 0.11293945461511612,
      "step": 730
    },
    {
      "epoch": 0.1892378990731205,
      "grad_norm": 102.5,
      "learning_rate": 4.053810504634397e-07,
      "logits/chosen": -0.3365722596645355,
      "logits/rejected": -0.39765626192092896,
      "logps/chosen": -314.0,
      "logps/rejected": -318.6000061035156,
      "loss": 0.7078,
      "rewards/accuracies": 0.39971446990966797,
      "rewards/chosen": 0.10227050632238388,
      "rewards/margins": -0.01618347130715847,
      "rewards/rejected": 0.11845703423023224,
      "step": 735
    },
    {
      "epoch": 0.19052523171987643,
      "grad_norm": 124.0,
      "learning_rate": 4.0473738414006176e-07,
      "logits/chosen": -0.3255859315395355,
      "logits/rejected": -0.3833984434604645,
      "logps/chosen": -267.0,
      "logps/rejected": -259.20001220703125,
      "loss": 0.6703,
      "rewards/accuracies": 0.44583335518836975,
      "rewards/chosen": 0.13511963188648224,
      "rewards/margins": 0.067138671875,
      "rewards/rejected": 0.06804199516773224,
      "step": 740
    },
    {
      "epoch": 0.19181256436663233,
      "grad_norm": 103.0,
      "learning_rate": 4.040937178166838e-07,
      "logits/chosen": -0.45075684785842896,
      "logits/rejected": -0.3345703184604645,
      "logps/chosen": -311.3999938964844,
      "logps/rejected": -243.39999389648438,
      "loss": 0.6438,
      "rewards/accuracies": 0.6355952620506287,
      "rewards/chosen": 0.21425780653953552,
      "rewards/margins": 0.12578125298023224,
      "rewards/rejected": 0.08828125149011612,
      "step": 745
    },
    {
      "epoch": 0.19309989701338826,
      "grad_norm": 108.5,
      "learning_rate": 4.0345005149330585e-07,
      "logits/chosen": -0.31132811307907104,
      "logits/rejected": -0.3363281190395355,
      "logps/chosen": -264.79998779296875,
      "logps/rejected": -243.60000610351562,
      "loss": 0.6594,
      "rewards/accuracies": 0.46833333373069763,
      "rewards/chosen": 0.16196289658546448,
      "rewards/margins": 0.08964844048023224,
      "rewards/rejected": 0.07231445610523224,
      "step": 750
    },
    {
      "epoch": 0.19438722966014418,
      "grad_norm": 117.5,
      "learning_rate": 4.028063851699279e-07,
      "logits/chosen": -0.3565429747104645,
      "logits/rejected": -0.33476561307907104,
      "logps/chosen": -312.20001220703125,
      "logps/rejected": -284.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.5560714602470398,
      "rewards/chosen": 0.13059082627296448,
      "rewards/margins": 0.07700195163488388,
      "rewards/rejected": 0.05381927639245987,
      "step": 755
    },
    {
      "epoch": 0.1956745623069001,
      "grad_norm": 112.5,
      "learning_rate": 4.0216271884654994e-07,
      "logits/chosen": -0.27617186307907104,
      "logits/rejected": -0.32646483182907104,
      "logps/chosen": -255.1999969482422,
      "logps/rejected": -258.79998779296875,
      "loss": 0.6734,
      "rewards/accuracies": 0.5442857146263123,
      "rewards/chosen": 0.14072266221046448,
      "rewards/margins": 0.05683593824505806,
      "rewards/rejected": 0.08387450873851776,
      "step": 760
    },
    {
      "epoch": 0.196961894953656,
      "grad_norm": 219.0,
      "learning_rate": 4.01519052523172e-07,
      "logits/chosen": -0.27473753690719604,
      "logits/rejected": -0.3109374940395355,
      "logps/chosen": -257.0,
      "logps/rejected": -212.8000030517578,
      "loss": 0.6781,
      "rewards/accuracies": 0.4920346140861511,
      "rewards/chosen": 0.10410156100988388,
      "rewards/margins": 0.04863281175494194,
      "rewards/rejected": 0.05559997633099556,
      "step": 765
    },
    {
      "epoch": 0.19824922760041194,
      "grad_norm": 227.0,
      "learning_rate": 4.00875386199794e-07,
      "logits/chosen": -0.3115234375,
      "logits/rejected": -0.39179688692092896,
      "logps/chosen": -301.0,
      "logps/rejected": -260.3999938964844,
      "loss": 0.6531,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.16562500596046448,
      "rewards/margins": 0.10302734375,
      "rewards/rejected": 0.06272277981042862,
      "step": 770
    },
    {
      "epoch": 0.19953656024716787,
      "grad_norm": 108.5,
      "learning_rate": 4.0023171987641606e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.4312500059604645,
      "logps/chosen": -276.0,
      "logps/rejected": -252.1999969482422,
      "loss": 0.6555,
      "rewards/accuracies": 0.5785714387893677,
      "rewards/chosen": 0.15756836533546448,
      "rewards/margins": 0.08461914211511612,
      "rewards/rejected": 0.07292480766773224,
      "step": 775
    },
    {
      "epoch": 0.2008238928939238,
      "grad_norm": 131.0,
      "learning_rate": 3.995880535530381e-07,
      "logits/chosen": -0.3138671815395355,
      "logits/rejected": -0.38203126192092896,
      "logps/chosen": -275.20001220703125,
      "logps/rejected": -271.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.4072222113609314,
      "rewards/chosen": 0.0591888427734375,
      "rewards/margins": 0.02890625037252903,
      "rewards/rejected": 0.03024902381002903,
      "step": 780
    },
    {
      "epoch": 0.2021112255406797,
      "grad_norm": 229.0,
      "learning_rate": 3.989443872296601e-07,
      "logits/chosen": -0.3263183534145355,
      "logits/rejected": -0.3064941465854645,
      "logps/chosen": -243.0,
      "logps/rejected": -215.39999389648438,
      "loss": 0.6828,
      "rewards/accuracies": 0.5483683347702026,
      "rewards/chosen": 0.11812744289636612,
      "rewards/margins": 0.0418701171875,
      "rewards/rejected": 0.076171875,
      "step": 785
    },
    {
      "epoch": 0.20339855818743563,
      "grad_norm": 125.0,
      "learning_rate": 3.9830072090628214e-07,
      "logits/chosen": -0.35600584745407104,
      "logits/rejected": -0.47216796875,
      "logps/chosen": -256.79998779296875,
      "logps/rejected": -225.1999969482422,
      "loss": 0.6523,
      "rewards/accuracies": 0.5405952334403992,
      "rewards/chosen": 0.14541015028953552,
      "rewards/margins": 0.09492187201976776,
      "rewards/rejected": 0.05052490159869194,
      "step": 790
    },
    {
      "epoch": 0.20468589083419156,
      "grad_norm": 160.0,
      "learning_rate": 3.9765705458290423e-07,
      "logits/chosen": -0.36835938692092896,
      "logits/rejected": -0.35371094942092896,
      "logps/chosen": -301.0,
      "logps/rejected": -267.6000061035156,
      "loss": 0.6547,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.17460937798023224,
      "rewards/margins": 0.09633789211511612,
      "rewards/rejected": 0.078069306910038,
      "step": 795
    },
    {
      "epoch": 0.2059732234809475,
      "grad_norm": 97.5,
      "learning_rate": 3.970133882595262e-07,
      "logits/chosen": -0.4486328065395355,
      "logits/rejected": -0.3667968809604645,
      "logps/chosen": -265.6000061035156,
      "logps/rejected": -247.8000030517578,
      "loss": 0.6602,
      "rewards/accuracies": 0.5224999785423279,
      "rewards/chosen": 0.18837890028953552,
      "rewards/margins": 0.07456054538488388,
      "rewards/rejected": 0.11328125,
      "step": 800
    },
    {
      "epoch": 0.2072605561277034,
      "grad_norm": 88.0,
      "learning_rate": 3.9636972193614827e-07,
      "logits/chosen": -0.34882813692092896,
      "logits/rejected": -0.353515625,
      "logps/chosen": -304.0,
      "logps/rejected": -263.79998779296875,
      "loss": 0.6828,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.134033203125,
      "rewards/margins": 0.02780761756002903,
      "rewards/rejected": 0.10625000298023224,
      "step": 805
    },
    {
      "epoch": 0.20854788877445932,
      "grad_norm": 112.5,
      "learning_rate": 3.9572605561277036e-07,
      "logits/chosen": -0.3919921815395355,
      "logits/rejected": -0.3343749940395355,
      "logps/chosen": -257.6000061035156,
      "logps/rejected": -250.1999969482422,
      "loss": 0.6469,
      "rewards/accuracies": 0.5752381086349487,
      "rewards/chosen": 0.23476561903953552,
      "rewards/margins": 0.11416015774011612,
      "rewards/rejected": 0.12050781399011612,
      "step": 810
    },
    {
      "epoch": 0.20983522142121525,
      "grad_norm": 125.5,
      "learning_rate": 3.9508238928939235e-07,
      "logits/chosen": -0.504687488079071,
      "logits/rejected": -0.3028320372104645,
      "logps/chosen": -285.79998779296875,
      "logps/rejected": -254.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.4608333706855774,
      "rewards/chosen": 0.09086914360523224,
      "rewards/margins": 0.03265075758099556,
      "rewards/rejected": 0.05832519382238388,
      "step": 815
    },
    {
      "epoch": 0.21112255406797117,
      "grad_norm": 118.5,
      "learning_rate": 3.944387229660144e-07,
      "logits/chosen": -0.4410156309604645,
      "logits/rejected": -0.37578123807907104,
      "logps/chosen": -238.1999969482422,
      "logps/rejected": -184.39999389648438,
      "loss": 0.6539,
      "rewards/accuracies": 0.5969444513320923,
      "rewards/chosen": 0.17148438096046448,
      "rewards/margins": 0.10239257663488388,
      "rewards/rejected": 0.068673275411129,
      "step": 820
    },
    {
      "epoch": 0.21240988671472708,
      "grad_norm": 134.0,
      "learning_rate": 3.9379505664263644e-07,
      "logits/chosen": -0.374755859375,
      "logits/rejected": -0.45039063692092896,
      "logps/chosen": -327.6000061035156,
      "logps/rejected": -302.3999938964844,
      "loss": 0.6703,
      "rewards/accuracies": 0.3884523808956146,
      "rewards/chosen": 0.1474609375,
      "rewards/margins": 0.05511169508099556,
      "rewards/rejected": 0.09223632514476776,
      "step": 825
    },
    {
      "epoch": 0.213697219361483,
      "grad_norm": 113.5,
      "learning_rate": 3.931513903192585e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.3199218809604645,
      "logps/chosen": -304.79998779296875,
      "logps/rejected": -269.79998779296875,
      "loss": 0.6578,
      "rewards/accuracies": 0.5605219602584839,
      "rewards/chosen": 0.16230468451976776,
      "rewards/margins": 0.09977416694164276,
      "rewards/rejected": 0.06252441555261612,
      "step": 830
    },
    {
      "epoch": 0.21498455200823893,
      "grad_norm": 135.0,
      "learning_rate": 3.925077239958805e-07,
      "logits/chosen": -0.3692871034145355,
      "logits/rejected": -0.4457031190395355,
      "logps/chosen": -295.79998779296875,
      "logps/rejected": -266.0,
      "loss": 0.6734,
      "rewards/accuracies": 0.49813857674598694,
      "rewards/chosen": 0.125,
      "rewards/margins": 0.04941863939166069,
      "rewards/rejected": 0.07570800930261612,
      "step": 835
    },
    {
      "epoch": 0.21627188465499486,
      "grad_norm": 139.0,
      "learning_rate": 3.9186405767250257e-07,
      "logits/chosen": -0.3232421875,
      "logits/rejected": -0.2972656190395355,
      "logps/chosen": -308.6000061035156,
      "logps/rejected": -235.39999389648438,
      "loss": 0.6453,
      "rewards/accuracies": 0.59333336353302,
      "rewards/chosen": 0.19492188096046448,
      "rewards/margins": 0.11694641411304474,
      "rewards/rejected": 0.07807616889476776,
      "step": 840
    },
    {
      "epoch": 0.21755921730175076,
      "grad_norm": 384.0,
      "learning_rate": 3.912203913491246e-07,
      "logits/chosen": -0.43964844942092896,
      "logits/rejected": -0.4164062440395355,
      "logps/chosen": -238.0,
      "logps/rejected": -221.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.18710938096046448,
      "rewards/margins": 0.096923828125,
      "rewards/rejected": 0.08994140475988388,
      "step": 845
    },
    {
      "epoch": 0.2188465499485067,
      "grad_norm": 132.0,
      "learning_rate": 3.905767250257466e-07,
      "logits/chosen": -0.2750000059604645,
      "logits/rejected": -0.32246094942092896,
      "logps/chosen": -260.0,
      "logps/rejected": -240.0,
      "loss": 0.643,
      "rewards/accuracies": 0.5800758004188538,
      "rewards/chosen": 0.13445433974266052,
      "rewards/margins": 0.11635742336511612,
      "rewards/rejected": 0.018310546875,
      "step": 850
    },
    {
      "epoch": 0.22013388259526262,
      "grad_norm": 508.0,
      "learning_rate": 3.899330587023687e-07,
      "logits/chosen": -0.2502807676792145,
      "logits/rejected": -0.25483399629592896,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -256.3999938964844,
      "loss": 0.6398,
      "rewards/accuracies": 0.5365384817123413,
      "rewards/chosen": 0.191650390625,
      "rewards/margins": 0.125732421875,
      "rewards/rejected": 0.06591796875,
      "step": 855
    },
    {
      "epoch": 0.22142121524201855,
      "grad_norm": 114.5,
      "learning_rate": 3.8928939237899074e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": -0.38398438692092896,
      "logps/chosen": -326.20001220703125,
      "logps/rejected": -262.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.44333332777023315,
      "rewards/chosen": 0.11894531548023224,
      "rewards/margins": 0.03791503980755806,
      "rewards/rejected": 0.08122558891773224,
      "step": 860
    },
    {
      "epoch": 0.22270854788877445,
      "grad_norm": 185.0,
      "learning_rate": 3.886457260556127e-07,
      "logits/chosen": -0.31328123807907104,
      "logits/rejected": -0.2889648377895355,
      "logps/chosen": -245.39999389648438,
      "logps/rejected": -205.39999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.5616758465766907,
      "rewards/chosen": 0.157470703125,
      "rewards/margins": 0.10585937649011612,
      "rewards/rejected": 0.05153198167681694,
      "step": 865
    },
    {
      "epoch": 0.22399588053553038,
      "grad_norm": 198.0,
      "learning_rate": 3.8800205973223477e-07,
      "logits/chosen": -0.2919921875,
      "logits/rejected": -0.19882813096046448,
      "logps/chosen": -239.39999389648438,
      "logps/rejected": -276.0,
      "loss": 0.7055,
      "rewards/accuracies": 0.4350990653038025,
      "rewards/chosen": 0.11748047173023224,
      "rewards/margins": -0.00931396521627903,
      "rewards/rejected": 0.12675781548023224,
      "step": 870
    },
    {
      "epoch": 0.2252832131822863,
      "grad_norm": 112.0,
      "learning_rate": 3.8735839340885686e-07,
      "logits/chosen": -0.3958984315395355,
      "logits/rejected": -0.3658203184604645,
      "logps/chosen": -257.0,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6281,
      "rewards/accuracies": 0.6757143139839172,
      "rewards/chosen": 0.19033202528953552,
      "rewards/margins": 0.15390625596046448,
      "rewards/rejected": 0.03635253757238388,
      "step": 875
    },
    {
      "epoch": 0.22657054582904224,
      "grad_norm": 108.0,
      "learning_rate": 3.8671472708547885e-07,
      "logits/chosen": -0.4371093809604645,
      "logits/rejected": -0.3515625,
      "logps/chosen": -303.6000061035156,
      "logps/rejected": -243.39999389648438,
      "loss": 0.6586,
      "rewards/accuracies": 0.5508333444595337,
      "rewards/chosen": 0.1796875,
      "rewards/margins": 0.07871093600988388,
      "rewards/rejected": 0.10102538764476776,
      "step": 880
    },
    {
      "epoch": 0.22785787847579814,
      "grad_norm": 350.0,
      "learning_rate": 3.860710607621009e-07,
      "logits/chosen": -0.2919921875,
      "logits/rejected": -0.21494141221046448,
      "logps/chosen": -293.6000061035156,
      "logps/rejected": -268.79998779296875,
      "loss": 0.6445,
      "rewards/accuracies": 0.5658333897590637,
      "rewards/chosen": 0.16640624403953552,
      "rewards/margins": 0.11469726264476776,
      "rewards/rejected": 0.05155639722943306,
      "step": 885
    },
    {
      "epoch": 0.22914521112255407,
      "grad_norm": 183.0,
      "learning_rate": 3.8542739443872294e-07,
      "logits/chosen": -0.29023438692092896,
      "logits/rejected": -0.30302733182907104,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -238.3000030517578,
      "loss": 0.675,
      "rewards/accuracies": 0.5358333587646484,
      "rewards/chosen": 0.15883788466453552,
      "rewards/margins": 0.05240478366613388,
      "rewards/rejected": 0.10649414360523224,
      "step": 890
    },
    {
      "epoch": 0.23043254376931,
      "grad_norm": 264.0,
      "learning_rate": 3.84783728115345e-07,
      "logits/chosen": -0.26777344942092896,
      "logits/rejected": -0.3451171815395355,
      "logps/chosen": -245.0,
      "logps/rejected": -235.0,
      "loss": 0.6594,
      "rewards/accuracies": 0.4972619116306305,
      "rewards/chosen": 0.09901122748851776,
      "rewards/margins": 0.08837890625,
      "rewards/rejected": 0.01059570349752903,
      "step": 895
    },
    {
      "epoch": 0.23171987641606592,
      "grad_norm": 190.0,
      "learning_rate": 3.84140061791967e-07,
      "logits/chosen": -0.3169921934604645,
      "logits/rejected": -0.25834959745407104,
      "logps/chosen": -279.6000061035156,
      "logps/rejected": -212.0,
      "loss": 0.6289,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": 0.22597655653953552,
      "rewards/margins": 0.15166015923023224,
      "rewards/rejected": 0.07441405951976776,
      "step": 900
    },
    {
      "epoch": 0.23300720906282182,
      "grad_norm": 128.0,
      "learning_rate": 3.8349639546858907e-07,
      "logits/chosen": -0.4390625059604645,
      "logits/rejected": -0.4859375059604645,
      "logps/chosen": -306.20001220703125,
      "logps/rejected": -219.39999389648438,
      "loss": 0.6555,
      "rewards/accuracies": 0.46427488327026367,
      "rewards/chosen": 0.1689453125,
      "rewards/margins": 0.07846679538488388,
      "rewards/rejected": 0.09042968600988388,
      "step": 905
    },
    {
      "epoch": 0.23429454170957775,
      "grad_norm": 122.0,
      "learning_rate": 3.828527291452111e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.4613281190395355,
      "logps/chosen": -351.20001220703125,
      "logps/rejected": -355.20001220703125,
      "loss": 0.6797,
      "rewards/accuracies": 0.5010714530944824,
      "rewards/chosen": 0.11928711086511612,
      "rewards/margins": 0.05036621168255806,
      "rewards/rejected": 0.06890258938074112,
      "step": 910
    },
    {
      "epoch": 0.23558187435633368,
      "grad_norm": 130.0,
      "learning_rate": 3.8220906282183315e-07,
      "logits/chosen": -0.20185546576976776,
      "logits/rejected": -0.14384765923023224,
      "logps/chosen": -216.8000030517578,
      "logps/rejected": -267.3999938964844,
      "loss": 0.6469,
      "rewards/accuracies": 0.5033333897590637,
      "rewards/chosen": 0.17470702528953552,
      "rewards/margins": 0.10180053859949112,
      "rewards/rejected": 0.07304687798023224,
      "step": 915
    },
    {
      "epoch": 0.2368692070030896,
      "grad_norm": 198.0,
      "learning_rate": 3.815653964984552e-07,
      "logits/chosen": -0.3677734434604645,
      "logits/rejected": -0.341796875,
      "logps/chosen": -260.20001220703125,
      "logps/rejected": -210.8000030517578,
      "loss": 0.6625,
      "rewards/accuracies": 0.5430952310562134,
      "rewards/chosen": 0.13751220703125,
      "rewards/margins": 0.073974609375,
      "rewards/rejected": 0.0633544921875,
      "step": 920
    },
    {
      "epoch": 0.2381565396498455,
      "grad_norm": 188.0,
      "learning_rate": 3.8092173017507724e-07,
      "logits/chosen": -0.43085938692092896,
      "logits/rejected": -0.3773437440395355,
      "logps/chosen": -250.60000610351562,
      "logps/rejected": -251.8000030517578,
      "loss": 0.6844,
      "rewards/accuracies": 0.4766666889190674,
      "rewards/chosen": 0.15971679985523224,
      "rewards/margins": 0.03887939453125,
      "rewards/rejected": 0.12126465141773224,
      "step": 925
    },
    {
      "epoch": 0.23944387229660144,
      "grad_norm": 454.0,
      "learning_rate": 3.8027806385169923e-07,
      "logits/chosen": -0.4253906309604645,
      "logits/rejected": -0.4574218690395355,
      "logps/chosen": -235.0,
      "logps/rejected": -214.5500030517578,
      "loss": 0.675,
      "rewards/accuracies": 0.4697619378566742,
      "rewards/chosen": 0.19658203423023224,
      "rewards/margins": 0.05571288987994194,
      "rewards/rejected": 0.14112548530101776,
      "step": 930
    },
    {
      "epoch": 0.24073120494335737,
      "grad_norm": 106.0,
      "learning_rate": 3.796343975283213e-07,
      "logits/chosen": -0.3375000059604645,
      "logits/rejected": -0.39375001192092896,
      "logps/chosen": -244.0,
      "logps/rejected": -248.6999969482422,
      "loss": 0.6484,
      "rewards/accuracies": 0.5316666960716248,
      "rewards/chosen": 0.15322265028953552,
      "rewards/margins": 0.09489135444164276,
      "rewards/rejected": 0.05839843675494194,
      "step": 935
    },
    {
      "epoch": 0.2420185375901133,
      "grad_norm": 107.0,
      "learning_rate": 3.7899073120494337e-07,
      "logits/chosen": -0.17465820908546448,
      "logits/rejected": -0.22963866591453552,
      "logps/chosen": -265.20001220703125,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6492,
      "rewards/accuracies": 0.6222402453422546,
      "rewards/chosen": 0.17499999701976776,
      "rewards/margins": 0.10765381157398224,
      "rewards/rejected": 0.06711425632238388,
      "step": 940
    },
    {
      "epoch": 0.2433058702368692,
      "grad_norm": 119.5,
      "learning_rate": 3.7834706488156536e-07,
      "logits/chosen": -0.4371093809604645,
      "logits/rejected": -0.41484373807907104,
      "logps/chosen": -304.0,
      "logps/rejected": -273.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.43968868255615234,
      "rewards/chosen": 0.11713866889476776,
      "rewards/margins": 0.07486114650964737,
      "rewards/rejected": 0.04254150390625,
      "step": 945
    },
    {
      "epoch": 0.24459320288362513,
      "grad_norm": 198.0,
      "learning_rate": 3.777033985581874e-07,
      "logits/chosen": -0.39935302734375,
      "logits/rejected": -0.38139647245407104,
      "logps/chosen": -260.0,
      "logps/rejected": -257.3999938964844,
      "loss": 0.6867,
      "rewards/accuracies": 0.371153861284256,
      "rewards/chosen": 0.10034789890050888,
      "rewards/margins": 0.02895507775247097,
      "rewards/rejected": 0.07143554836511612,
      "step": 950
    },
    {
      "epoch": 0.24588053553038106,
      "grad_norm": 99.5,
      "learning_rate": 3.770597322348095e-07,
      "logits/chosen": -0.25578612089157104,
      "logits/rejected": -0.24257811903953552,
      "logps/chosen": -230.8000030517578,
      "logps/rejected": -222.0,
      "loss": 0.6586,
      "rewards/accuracies": 0.5525000095367432,
      "rewards/chosen": 0.16767577826976776,
      "rewards/margins": 0.08486328274011612,
      "rewards/rejected": 0.082763671875,
      "step": 955
    },
    {
      "epoch": 0.24716786817713698,
      "grad_norm": 126.5,
      "learning_rate": 3.764160659114315e-07,
      "logits/chosen": -0.36640626192092896,
      "logits/rejected": -0.2603515684604645,
      "logps/chosen": -336.3999938964844,
      "logps/rejected": -286.3999938964844,
      "loss": 0.6344,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.23613281548023224,
      "rewards/margins": 0.14677734673023224,
      "rewards/rejected": 0.08964844048023224,
      "step": 960
    },
    {
      "epoch": 0.24845520082389289,
      "grad_norm": 173.0,
      "learning_rate": 3.7577239958805353e-07,
      "logits/chosen": -0.3277343809604645,
      "logits/rejected": -0.33818358182907104,
      "logps/chosen": -285.6000061035156,
      "logps/rejected": -248.8000030517578,
      "loss": 0.6273,
      "rewards/accuracies": 0.6519047617912292,
      "rewards/chosen": 0.17119140923023224,
      "rewards/margins": 0.144287109375,
      "rewards/rejected": 0.02731170691549778,
      "step": 965
    },
    {
      "epoch": 0.24974253347064881,
      "grad_norm": 90.0,
      "learning_rate": 3.7512873326467557e-07,
      "logits/chosen": -0.40507811307907104,
      "logits/rejected": -0.443359375,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -268.3999938964844,
      "loss": 0.668,
      "rewards/accuracies": 0.5275000333786011,
      "rewards/chosen": 0.18574218451976776,
      "rewards/margins": 0.06972656399011612,
      "rewards/rejected": 0.11640624701976776,
      "step": 970
    },
    {
      "epoch": 0.25102986611740474,
      "grad_norm": 98.0,
      "learning_rate": 3.744850669412976e-07,
      "logits/chosen": -0.4085937440395355,
      "logits/rejected": -0.37773436307907104,
      "logps/chosen": -323.79998779296875,
      "logps/rejected": -249.1999969482422,
      "loss": 0.6391,
      "rewards/accuracies": 0.5841666460037231,
      "rewards/chosen": 0.13930663466453552,
      "rewards/margins": 0.12968750298023224,
      "rewards/rejected": 0.009776306338608265,
      "step": 975
    },
    {
      "epoch": 0.25231719876416064,
      "grad_norm": 256.0,
      "learning_rate": 3.7384140061791965e-07,
      "logits/chosen": -0.31035155057907104,
      "logits/rejected": -0.3154296875,
      "logps/chosen": -291.20001220703125,
      "logps/rejected": -248.0,
      "loss": 0.6742,
      "rewards/accuracies": 0.5158333778381348,
      "rewards/chosen": 0.14956054091453552,
      "rewards/margins": 0.06325378268957138,
      "rewards/rejected": 0.08616943657398224,
      "step": 980
    },
    {
      "epoch": 0.2536045314109166,
      "grad_norm": 556.0,
      "learning_rate": 3.731977342945417e-07,
      "logits/chosen": -0.4781250059604645,
      "logits/rejected": -0.4351562559604645,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -288.79998779296875,
      "loss": 0.6461,
      "rewards/accuracies": 0.556859016418457,
      "rewards/chosen": 0.20273438096046448,
      "rewards/margins": 0.119873046875,
      "rewards/rejected": 0.08283691108226776,
      "step": 985
    },
    {
      "epoch": 0.2548918640576725,
      "grad_norm": 157.0,
      "learning_rate": 3.7255406797116374e-07,
      "logits/chosen": -0.3539062440395355,
      "logits/rejected": -0.3871093690395355,
      "logps/chosen": -278.6000061035156,
      "logps/rejected": -254.39999389648438,
      "loss": 0.6469,
      "rewards/accuracies": 0.6361905336380005,
      "rewards/chosen": 0.24257811903953552,
      "rewards/margins": 0.11713866889476776,
      "rewards/rejected": 0.12539061903953552,
      "step": 990
    },
    {
      "epoch": 0.2561791967044284,
      "grad_norm": 133.0,
      "learning_rate": 3.7191040164778573e-07,
      "logits/chosen": -0.3607421815395355,
      "logits/rejected": -0.33125001192092896,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -232.8000030517578,
      "loss": 0.682,
      "rewards/accuracies": 0.4816666543483734,
      "rewards/chosen": 0.17597655951976776,
      "rewards/margins": 0.04665222018957138,
      "rewards/rejected": 0.12929686903953552,
      "step": 995
    },
    {
      "epoch": 0.25746652935118436,
      "grad_norm": 128.0,
      "learning_rate": 3.712667353244078e-07,
      "logits/chosen": -0.44843751192092896,
      "logits/rejected": -0.3949218690395355,
      "logps/chosen": -259.20001220703125,
      "logps/rejected": -248.1999969482422,
      "loss": 0.643,
      "rewards/accuracies": 0.5467857122421265,
      "rewards/chosen": 0.15791015326976776,
      "rewards/margins": 0.11181640625,
      "rewards/rejected": 0.04619140550494194,
      "step": 1000
    },
    {
      "epoch": 0.25875386199794026,
      "grad_norm": 112.0,
      "learning_rate": 3.7062306900102987e-07,
      "logits/chosen": -0.44843751192092896,
      "logits/rejected": -0.41874998807907104,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -259.79998779296875,
      "loss": 0.6602,
      "rewards/accuracies": 0.49724358320236206,
      "rewards/chosen": 0.15400390326976776,
      "rewards/margins": 0.0849609375,
      "rewards/rejected": 0.06905517727136612,
      "step": 1005
    },
    {
      "epoch": 0.2600411946446962,
      "grad_norm": 110.5,
      "learning_rate": 3.6997940267765186e-07,
      "logits/chosen": -0.41523438692092896,
      "logits/rejected": -0.40253907442092896,
      "logps/chosen": -328.79998779296875,
      "logps/rejected": -267.3999938964844,
      "loss": 0.6445,
      "rewards/accuracies": 0.6241666674613953,
      "rewards/chosen": 0.22099609673023224,
      "rewards/margins": 0.11870117485523224,
      "rewards/rejected": 0.10224609076976776,
      "step": 1010
    },
    {
      "epoch": 0.2613285272914521,
      "grad_norm": 129.0,
      "learning_rate": 3.6933573635427395e-07,
      "logits/chosen": -0.38323974609375,
      "logits/rejected": -0.31953126192092896,
      "logps/chosen": -341.79998779296875,
      "logps/rejected": -272.3999938964844,
      "loss": 0.6664,
      "rewards/accuracies": 0.5112121105194092,
      "rewards/chosen": 0.14536133408546448,
      "rewards/margins": 0.04853515699505806,
      "rewards/rejected": 0.09682617336511612,
      "step": 1015
    },
    {
      "epoch": 0.262615859938208,
      "grad_norm": 168.0,
      "learning_rate": 3.68692070030896e-07,
      "logits/chosen": -0.37089842557907104,
      "logits/rejected": -0.41826170682907104,
      "logps/chosen": -262.20001220703125,
      "logps/rejected": -266.20001220703125,
      "loss": 0.6578,
      "rewards/accuracies": 0.5283333659172058,
      "rewards/chosen": 0.19511719048023224,
      "rewards/margins": 0.08442382514476776,
      "rewards/rejected": 0.11054687201976776,
      "step": 1020
    },
    {
      "epoch": 0.263903192584964,
      "grad_norm": 362.0,
      "learning_rate": 3.68048403707518e-07,
      "logits/chosen": -0.27460938692092896,
      "logits/rejected": -0.26777344942092896,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -210.8000030517578,
      "loss": 0.6922,
      "rewards/accuracies": 0.45746031403541565,
      "rewards/chosen": 0.04936523362994194,
      "rewards/margins": 0.011035156436264515,
      "rewards/rejected": 0.03850097581744194,
      "step": 1025
    },
    {
      "epoch": 0.2651905252317199,
      "grad_norm": 172.0,
      "learning_rate": 3.6740473738414003e-07,
      "logits/chosen": -0.38554686307907104,
      "logits/rejected": -0.34794920682907104,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6781,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.1680908203125,
      "rewards/margins": 0.03632812574505806,
      "rewards/rejected": 0.13150635361671448,
      "step": 1030
    },
    {
      "epoch": 0.2664778578784758,
      "grad_norm": 159.0,
      "learning_rate": 3.6676107106076207e-07,
      "logits/chosen": -0.2939453125,
      "logits/rejected": -0.3070312440395355,
      "logps/chosen": -271.0,
      "logps/rejected": -263.0,
      "loss": 0.6711,
      "rewards/accuracies": 0.4407051205635071,
      "rewards/chosen": 0.14753417670726776,
      "rewards/margins": 0.06368102878332138,
      "rewards/rejected": 0.08376464992761612,
      "step": 1035
    },
    {
      "epoch": 0.26776519052523173,
      "grad_norm": 141.0,
      "learning_rate": 3.661174047373841e-07,
      "logits/chosen": -0.3818359375,
      "logits/rejected": -0.3602050840854645,
      "logps/chosen": -265.3999938964844,
      "logps/rejected": -244.0,
      "loss": 0.6586,
      "rewards/accuracies": 0.49464282393455505,
      "rewards/chosen": 0.24521484971046448,
      "rewards/margins": 0.09421386569738388,
      "rewards/rejected": 0.15078124403953552,
      "step": 1040
    },
    {
      "epoch": 0.26905252317198763,
      "grad_norm": 456.0,
      "learning_rate": 3.6547373841400616e-07,
      "logits/chosen": -0.3866210877895355,
      "logits/rejected": -0.3123535215854645,
      "logps/chosen": -356.6000061035156,
      "logps/rejected": -281.20001220703125,
      "loss": 0.6602,
      "rewards/accuracies": 0.4464285969734192,
      "rewards/chosen": 0.19355468451976776,
      "rewards/margins": 0.07968749850988388,
      "rewards/rejected": 0.11396484076976776,
      "step": 1045
    },
    {
      "epoch": 0.2703398558187436,
      "grad_norm": 198.0,
      "learning_rate": 3.648300720906282e-07,
      "logits/chosen": -0.24599608778953552,
      "logits/rejected": -0.24433593451976776,
      "logps/chosen": -231.60000610351562,
      "logps/rejected": -221.39999389648438,
      "loss": 0.7172,
      "rewards/accuracies": 0.48845237493515015,
      "rewards/chosen": 0.17243652045726776,
      "rewards/margins": -0.00616455078125,
      "rewards/rejected": 0.17840576171875,
      "step": 1050
    },
    {
      "epoch": 0.2716271884654995,
      "grad_norm": 390.0,
      "learning_rate": 3.6418640576725024e-07,
      "logits/chosen": -0.41523438692092896,
      "logits/rejected": -0.38691407442092896,
      "logps/chosen": -269.0,
      "logps/rejected": -296.0,
      "loss": 0.7023,
      "rewards/accuracies": 0.48178571462631226,
      "rewards/chosen": 0.13616943359375,
      "rewards/margins": -0.004864501766860485,
      "rewards/rejected": 0.1407470703125,
      "step": 1055
    },
    {
      "epoch": 0.2729145211122554,
      "grad_norm": 162.0,
      "learning_rate": 3.635427394438723e-07,
      "logits/chosen": -0.4847656190395355,
      "logits/rejected": -0.47187501192092896,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -267.6000061035156,
      "loss": 0.6836,
      "rewards/accuracies": 0.5000757575035095,
      "rewards/chosen": 0.140625,
      "rewards/margins": 0.03188476711511612,
      "rewards/rejected": 0.10874023288488388,
      "step": 1060
    },
    {
      "epoch": 0.27420185375901135,
      "grad_norm": 109.5,
      "learning_rate": 3.6289907312049433e-07,
      "logits/chosen": -0.4208984375,
      "logits/rejected": -0.3871093690395355,
      "logps/chosen": -293.20001220703125,
      "logps/rejected": -287.6000061035156,
      "loss": 0.6836,
      "rewards/accuracies": 0.5460714101791382,
      "rewards/chosen": 0.11403808742761612,
      "rewards/margins": 0.041748046875,
      "rewards/rejected": 0.07207031548023224,
      "step": 1065
    },
    {
      "epoch": 0.27548918640576725,
      "grad_norm": 274.0,
      "learning_rate": 3.6225540679711637e-07,
      "logits/chosen": -0.38945311307907104,
      "logits/rejected": -0.306640625,
      "logps/chosen": -251.0,
      "logps/rejected": -215.6999969482422,
      "loss": 0.668,
      "rewards/accuracies": 0.46964287757873535,
      "rewards/chosen": 0.15253905951976776,
      "rewards/margins": 0.06894531100988388,
      "rewards/rejected": 0.08365478366613388,
      "step": 1070
    },
    {
      "epoch": 0.27677651905252315,
      "grad_norm": 118.5,
      "learning_rate": 3.6161174047373836e-07,
      "logits/chosen": -0.4105468690395355,
      "logits/rejected": -0.330078125,
      "logps/chosen": -285.20001220703125,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6727,
      "rewards/accuracies": 0.5541666746139526,
      "rewards/chosen": 0.18808594346046448,
      "rewards/margins": 0.05917968600988388,
      "rewards/rejected": 0.12900391221046448,
      "step": 1075
    },
    {
      "epoch": 0.2780638516992791,
      "grad_norm": 150.0,
      "learning_rate": 3.6096807415036046e-07,
      "logits/chosen": -0.26513671875,
      "logits/rejected": -0.15449218451976776,
      "logps/chosen": -240.8000030517578,
      "logps/rejected": -220.0,
      "loss": 0.643,
      "rewards/accuracies": 0.5352564454078674,
      "rewards/chosen": 0.19941405951976776,
      "rewards/margins": 0.11992187798023224,
      "rewards/rejected": 0.07961425930261612,
      "step": 1080
    },
    {
      "epoch": 0.279351184346035,
      "grad_norm": 96.0,
      "learning_rate": 3.603244078269825e-07,
      "logits/chosen": -0.38984376192092896,
      "logits/rejected": -0.39238280057907104,
      "logps/chosen": -365.20001220703125,
      "logps/rejected": -271.79998779296875,
      "loss": 0.6258,
      "rewards/accuracies": 0.5516666769981384,
      "rewards/chosen": 0.2822265625,
      "rewards/margins": 0.16738280653953552,
      "rewards/rejected": 0.11513672024011612,
      "step": 1085
    },
    {
      "epoch": 0.2806385169927909,
      "grad_norm": 221.0,
      "learning_rate": 3.596807415036045e-07,
      "logits/chosen": -0.3570312559604645,
      "logits/rejected": -0.36542969942092896,
      "logps/chosen": -323.0,
      "logps/rejected": -294.6000061035156,
      "loss": 0.6547,
      "rewards/accuracies": 0.5576099157333374,
      "rewards/chosen": 0.15078124403953552,
      "rewards/margins": 0.0899658203125,
      "rewards/rejected": 0.0609130859375,
      "step": 1090
    },
    {
      "epoch": 0.28192584963954687,
      "grad_norm": 123.5,
      "learning_rate": 3.590370751802266e-07,
      "logits/chosen": -0.3124023377895355,
      "logits/rejected": -0.34199219942092896,
      "logps/chosen": -271.0,
      "logps/rejected": -232.39999389648438,
      "loss": 0.6625,
      "rewards/accuracies": 0.6158334016799927,
      "rewards/chosen": 0.16835936903953552,
      "rewards/margins": 0.08364257961511612,
      "rewards/rejected": 0.08502502739429474,
      "step": 1095
    },
    {
      "epoch": 0.28321318228630277,
      "grad_norm": 115.5,
      "learning_rate": 3.583934088568486e-07,
      "logits/chosen": -0.3544921875,
      "logits/rejected": -0.3369140625,
      "logps/chosen": -311.20001220703125,
      "logps/rejected": -267.0,
      "loss": 0.6687,
      "rewards/accuracies": 0.5214285850524902,
      "rewards/chosen": 0.18085937201976776,
      "rewards/margins": 0.074462890625,
      "rewards/rejected": 0.10656128078699112,
      "step": 1100
    },
    {
      "epoch": 0.2845005149330587,
      "grad_norm": 92.5,
      "learning_rate": 3.577497425334706e-07,
      "logits/chosen": -0.3038085997104645,
      "logits/rejected": -0.251953125,
      "logps/chosen": -251.0,
      "logps/rejected": -240.39999389648438,
      "loss": 0.6523,
      "rewards/accuracies": 0.5844047665596008,
      "rewards/chosen": 0.20859375596046448,
      "rewards/margins": 0.09284667670726776,
      "rewards/rejected": 0.11638183891773224,
      "step": 1105
    },
    {
      "epoch": 0.2857878475798146,
      "grad_norm": 260.0,
      "learning_rate": 3.571060762100927e-07,
      "logits/chosen": -0.39765626192092896,
      "logits/rejected": -0.46210938692092896,
      "logps/chosen": -280.20001220703125,
      "logps/rejected": -213.0,
      "loss": 0.6734,
      "rewards/accuracies": 0.5060714483261108,
      "rewards/chosen": 0.16474609076976776,
      "rewards/margins": 0.05460204929113388,
      "rewards/rejected": 0.10986328125,
      "step": 1110
    },
    {
      "epoch": 0.2870751802265705,
      "grad_norm": 127.5,
      "learning_rate": 3.564624098867147e-07,
      "logits/chosen": -0.18183593451976776,
      "logits/rejected": -0.22988280653953552,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -267.6000061035156,
      "loss": 0.6742,
      "rewards/accuracies": 0.47040295600891113,
      "rewards/chosen": 0.08063964545726776,
      "rewards/margins": 0.05742187425494194,
      "rewards/rejected": 0.02321777306497097,
      "step": 1115
    },
    {
      "epoch": 0.2883625128733265,
      "grad_norm": 148.0,
      "learning_rate": 3.5581874356333674e-07,
      "logits/chosen": -0.3988281190395355,
      "logits/rejected": -0.3125,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -263.79998779296875,
      "loss": 0.6672,
      "rewards/accuracies": 0.5571428537368774,
      "rewards/chosen": 0.1455078125,
      "rewards/margins": 0.06919467449188232,
      "rewards/rejected": 0.07622070610523224,
      "step": 1120
    },
    {
      "epoch": 0.2896498455200824,
      "grad_norm": 660.0,
      "learning_rate": 3.551750772399588e-07,
      "logits/chosen": -0.22324219346046448,
      "logits/rejected": -0.32746583223342896,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -228.8000030517578,
      "loss": 0.675,
      "rewards/accuracies": 0.6088286638259888,
      "rewards/chosen": 0.12592773139476776,
      "rewards/margins": 0.05666504055261612,
      "rewards/rejected": 0.06935425102710724,
      "step": 1125
    },
    {
      "epoch": 0.2909371781668383,
      "grad_norm": 474.0,
      "learning_rate": 3.5453141091658083e-07,
      "logits/chosen": -0.4203124940395355,
      "logits/rejected": -0.462890625,
      "logps/chosen": -282.3999938964844,
      "logps/rejected": -230.60000610351562,
      "loss": 0.6617,
      "rewards/accuracies": 0.596666693687439,
      "rewards/chosen": 0.18369141221046448,
      "rewards/margins": 0.07900390774011612,
      "rewards/rejected": 0.10463867336511612,
      "step": 1130
    },
    {
      "epoch": 0.29222451081359424,
      "grad_norm": 288.0,
      "learning_rate": 3.5388774459320287e-07,
      "logits/chosen": -0.27587890625,
      "logits/rejected": -0.22285155951976776,
      "logps/chosen": -261.79998779296875,
      "logps/rejected": -280.6000061035156,
      "loss": 0.65,
      "rewards/accuracies": 0.5791667103767395,
      "rewards/chosen": 0.20947265625,
      "rewards/margins": 0.10800781100988388,
      "rewards/rejected": 0.10129394382238388,
      "step": 1135
    },
    {
      "epoch": 0.29351184346035014,
      "grad_norm": 114.0,
      "learning_rate": 3.5324407826982486e-07,
      "logits/chosen": -0.19169922173023224,
      "logits/rejected": -0.21201172471046448,
      "logps/chosen": -286.0,
      "logps/rejected": -233.1999969482422,
      "loss": 0.6398,
      "rewards/accuracies": 0.5272619128227234,
      "rewards/chosen": 0.16962890326976776,
      "rewards/margins": 0.11745605617761612,
      "rewards/rejected": 0.05211181566119194,
      "step": 1140
    },
    {
      "epoch": 0.2947991761071061,
      "grad_norm": 126.0,
      "learning_rate": 3.5260041194644696e-07,
      "logits/chosen": -0.4066406190395355,
      "logits/rejected": -0.37226563692092896,
      "logps/chosen": -282.0,
      "logps/rejected": -266.79998779296875,
      "loss": 0.6727,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.21308593451976776,
      "rewards/margins": 0.054443359375,
      "rewards/rejected": 0.15854492783546448,
      "step": 1145
    },
    {
      "epoch": 0.296086508753862,
      "grad_norm": 142.0,
      "learning_rate": 3.51956745623069e-07,
      "logits/chosen": -0.16522522270679474,
      "logits/rejected": -0.07441405951976776,
      "logps/chosen": -306.6000061035156,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6578,
      "rewards/accuracies": 0.5605769157409668,
      "rewards/chosen": 0.15996094048023224,
      "rewards/margins": 0.09589843451976776,
      "rewards/rejected": 0.06412353366613388,
      "step": 1150
    },
    {
      "epoch": 0.2973738414006179,
      "grad_norm": 117.5,
      "learning_rate": 3.51313079299691e-07,
      "logits/chosen": -0.38945311307907104,
      "logits/rejected": -0.37382811307907104,
      "logps/chosen": -279.0,
      "logps/rejected": -248.1999969482422,
      "loss": 0.6656,
      "rewards/accuracies": 0.5342857241630554,
      "rewards/chosen": 0.15715332329273224,
      "rewards/margins": 0.07331542670726776,
      "rewards/rejected": 0.08399047702550888,
      "step": 1155
    },
    {
      "epoch": 0.29866117404737386,
      "grad_norm": 252.0,
      "learning_rate": 3.506694129763131e-07,
      "logits/chosen": -0.2973388731479645,
      "logits/rejected": -0.271728515625,
      "logps/chosen": -277.3999938964844,
      "logps/rejected": -226.1999969482422,
      "loss": 0.6469,
      "rewards/accuracies": 0.67166668176651,
      "rewards/chosen": 0.22431640326976776,
      "rewards/margins": 0.11406250298023224,
      "rewards/rejected": 0.11015625298023224,
      "step": 1160
    },
    {
      "epoch": 0.29994850669412976,
      "grad_norm": 123.5,
      "learning_rate": 3.5002574665293513e-07,
      "logits/chosen": -0.46601563692092896,
      "logits/rejected": -0.5009765625,
      "logps/chosen": -327.6000061035156,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6656,
      "rewards/accuracies": 0.5525000095367432,
      "rewards/chosen": 0.21923828125,
      "rewards/margins": 0.06915283203125,
      "rewards/rejected": 0.15036621689796448,
      "step": 1165
    },
    {
      "epoch": 0.30123583934088566,
      "grad_norm": 108.0,
      "learning_rate": 3.493820803295571e-07,
      "logits/chosen": -0.36689454317092896,
      "logits/rejected": -0.36601561307907104,
      "logps/chosen": -301.0,
      "logps/rejected": -304.0,
      "loss": 0.6648,
      "rewards/accuracies": 0.53083336353302,
      "rewards/chosen": 0.21562500298023224,
      "rewards/margins": 0.07980956882238388,
      "rewards/rejected": 0.13564452528953552,
      "step": 1170
    },
    {
      "epoch": 0.3025231719876416,
      "grad_norm": 185.0,
      "learning_rate": 3.487384140061792e-07,
      "logits/chosen": -0.23994140326976776,
      "logits/rejected": -0.18105468153953552,
      "logps/chosen": -293.20001220703125,
      "logps/rejected": -276.0,
      "loss": 0.6703,
      "rewards/accuracies": 0.5383333563804626,
      "rewards/chosen": 0.13671875,
      "rewards/margins": 0.05628204345703125,
      "rewards/rejected": 0.08059082180261612,
      "step": 1175
    },
    {
      "epoch": 0.3038105046343975,
      "grad_norm": 192.0,
      "learning_rate": 3.480947476828012e-07,
      "logits/chosen": -0.40703123807907104,
      "logits/rejected": -0.4625000059604645,
      "logps/chosen": -307.79998779296875,
      "logps/rejected": -238.60000610351562,
      "loss": 0.6547,
      "rewards/accuracies": 0.6102380752563477,
      "rewards/chosen": 0.17441406846046448,
      "rewards/margins": 0.09208984673023224,
      "rewards/rejected": 0.08236084133386612,
      "step": 1180
    },
    {
      "epoch": 0.30509783728115347,
      "grad_norm": 106.0,
      "learning_rate": 3.4745108135942325e-07,
      "logits/chosen": -0.23652343451976776,
      "logits/rejected": -0.26220703125,
      "logps/chosen": -339.3999938964844,
      "logps/rejected": -269.6000061035156,
      "loss": 0.6484,
      "rewards/accuracies": 0.6590476632118225,
      "rewards/chosen": 0.232666015625,
      "rewards/margins": 0.11542968451976776,
      "rewards/rejected": 0.11731567233800888,
      "step": 1185
    },
    {
      "epoch": 0.3063851699279094,
      "grad_norm": 116.5,
      "learning_rate": 3.4680741503604534e-07,
      "logits/chosen": -0.42109376192092896,
      "logits/rejected": -0.3388671875,
      "logps/chosen": -308.20001220703125,
      "logps/rejected": -297.79998779296875,
      "loss": 0.6727,
      "rewards/accuracies": 0.44410258531570435,
      "rewards/chosen": 0.19389648735523224,
      "rewards/margins": 0.06782226264476776,
      "rewards/rejected": 0.12626953423023224,
      "step": 1190
    },
    {
      "epoch": 0.3076725025746653,
      "grad_norm": 107.5,
      "learning_rate": 3.4616374871266733e-07,
      "logits/chosen": -0.33027344942092896,
      "logits/rejected": -0.2720703184604645,
      "logps/chosen": -260.6000061035156,
      "logps/rejected": -245.60000610351562,
      "loss": 0.6789,
      "rewards/accuracies": 0.4763828217983246,
      "rewards/chosen": 0.13247069716453552,
      "rewards/margins": 0.03798828274011612,
      "rewards/rejected": 0.09428711235523224,
      "step": 1195
    },
    {
      "epoch": 0.30895983522142123,
      "grad_norm": 113.0,
      "learning_rate": 3.455200823892894e-07,
      "logits/chosen": -0.4164062440395355,
      "logits/rejected": -0.2752441465854645,
      "logps/chosen": -321.20001220703125,
      "logps/rejected": -238.1999969482422,
      "loss": 0.632,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.23886719346046448,
      "rewards/margins": 0.155517578125,
      "rewards/rejected": 0.0826416015625,
      "step": 1200
    },
    {
      "epoch": 0.31024716786817713,
      "grad_norm": 434.0,
      "learning_rate": 3.448764160659114e-07,
      "logits/chosen": -0.4476562440395355,
      "logits/rejected": -0.46210938692092896,
      "logps/chosen": -295.20001220703125,
      "logps/rejected": -291.20001220703125,
      "loss": 0.6523,
      "rewards/accuracies": 0.4675000309944153,
      "rewards/chosen": 0.19031372666358948,
      "rewards/margins": 0.10302734375,
      "rewards/rejected": 0.08779297024011612,
      "step": 1205
    },
    {
      "epoch": 0.31153450051493303,
      "grad_norm": 144.0,
      "learning_rate": 3.4423274974253346e-07,
      "logits/chosen": -0.42351073026657104,
      "logits/rejected": -0.42036134004592896,
      "logps/chosen": -340.79998779296875,
      "logps/rejected": -269.79998779296875,
      "loss": 0.6297,
      "rewards/accuracies": 0.6327564120292664,
      "rewards/chosen": 0.2503906190395355,
      "rewards/margins": 0.16142578423023224,
      "rewards/rejected": 0.08901934325695038,
      "step": 1210
    },
    {
      "epoch": 0.312821833161689,
      "grad_norm": 102.0,
      "learning_rate": 3.435890834191555e-07,
      "logits/chosen": -0.41484373807907104,
      "logits/rejected": -0.34101563692092896,
      "logps/chosen": -264.0,
      "logps/rejected": -226.0,
      "loss": 0.6461,
      "rewards/accuracies": 0.6482143402099609,
      "rewards/chosen": 0.2275390625,
      "rewards/margins": 0.11943359673023224,
      "rewards/rejected": 0.108154296875,
      "step": 1215
    },
    {
      "epoch": 0.3141091658084449,
      "grad_norm": 161.0,
      "learning_rate": 3.429454170957775e-07,
      "logits/chosen": -0.33125001192092896,
      "logits/rejected": -0.39179688692092896,
      "logps/chosen": -291.20001220703125,
      "logps/rejected": -271.0,
      "loss": 0.6188,
      "rewards/accuracies": 0.6274999976158142,
      "rewards/chosen": 0.2685546875,
      "rewards/margins": 0.189453125,
      "rewards/rejected": 0.07871093600988388,
      "step": 1220
    },
    {
      "epoch": 0.31539649845520085,
      "grad_norm": 129.0,
      "learning_rate": 3.423017507723996e-07,
      "logits/chosen": -0.33867186307907104,
      "logits/rejected": -0.3921875059604645,
      "logps/chosen": -243.0,
      "logps/rejected": -249.39999389648438,
      "loss": 0.6547,
      "rewards/accuracies": 0.5449999570846558,
      "rewards/chosen": 0.19990234076976776,
      "rewards/margins": 0.09384765475988388,
      "rewards/rejected": 0.10625000298023224,
      "step": 1225
    },
    {
      "epoch": 0.31668383110195675,
      "grad_norm": 167.0,
      "learning_rate": 3.4165808444902163e-07,
      "logits/chosen": -0.3173828125,
      "logits/rejected": -0.2724609375,
      "logps/chosen": -282.20001220703125,
      "logps/rejected": -254.60000610351562,
      "loss": 0.6312,
      "rewards/accuracies": 0.6201923489570618,
      "rewards/chosen": 0.24301758408546448,
      "rewards/margins": 0.14824219048023224,
      "rewards/rejected": 0.09487304836511612,
      "step": 1230
    },
    {
      "epoch": 0.31797116374871265,
      "grad_norm": 90.5,
      "learning_rate": 3.410144181256436e-07,
      "logits/chosen": -0.225128173828125,
      "logits/rejected": -0.2586303651332855,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -264.6000061035156,
      "loss": 0.6742,
      "rewards/accuracies": 0.4074999690055847,
      "rewards/chosen": 0.10125732421875,
      "rewards/margins": 0.04434814304113388,
      "rewards/rejected": 0.05669403076171875,
      "step": 1235
    },
    {
      "epoch": 0.3192584963954686,
      "grad_norm": 133.0,
      "learning_rate": 3.403707518022657e-07,
      "logits/chosen": -0.353515625,
      "logits/rejected": -0.4789062440395355,
      "logps/chosen": -283.3999938964844,
      "logps/rejected": -257.6000061035156,
      "loss": 0.6633,
      "rewards/accuracies": 0.5174242258071899,
      "rewards/chosen": 0.16269531846046448,
      "rewards/margins": 0.09670410305261612,
      "rewards/rejected": 0.06580810248851776,
      "step": 1240
    },
    {
      "epoch": 0.3205458290422245,
      "grad_norm": 105.0,
      "learning_rate": 3.3972708547888776e-07,
      "logits/chosen": -0.3375000059604645,
      "logits/rejected": -0.3421874940395355,
      "logps/chosen": -272.79998779296875,
      "logps/rejected": -229.89999389648438,
      "loss": 0.6805,
      "rewards/accuracies": 0.41781750321388245,
      "rewards/chosen": 0.12620849907398224,
      "rewards/margins": 0.05583496019244194,
      "rewards/rejected": 0.07041015475988388,
      "step": 1245
    },
    {
      "epoch": 0.3218331616889804,
      "grad_norm": 119.0,
      "learning_rate": 3.3908341915550975e-07,
      "logits/chosen": -0.37187498807907104,
      "logits/rejected": -0.3343749940395355,
      "logps/chosen": -259.20001220703125,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6453,
      "rewards/accuracies": 0.6058333516120911,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 0.12138672173023224,
      "rewards/rejected": 0.13442382216453552,
      "step": 1250
    },
    {
      "epoch": 0.32312049433573636,
      "grad_norm": 163.0,
      "learning_rate": 3.3843975283213184e-07,
      "logits/chosen": -0.34003907442092896,
      "logits/rejected": -0.20078125596046448,
      "logps/chosen": -298.79998779296875,
      "logps/rejected": -314.6000061035156,
      "loss": 0.6672,
      "rewards/accuracies": 0.5366666913032532,
      "rewards/chosen": 0.14047852158546448,
      "rewards/margins": 0.08444824069738388,
      "rewards/rejected": 0.05589599534869194,
      "step": 1255
    },
    {
      "epoch": 0.32440782698249226,
      "grad_norm": 118.0,
      "learning_rate": 3.3779608650875383e-07,
      "logits/chosen": -0.4169921875,
      "logits/rejected": -0.4058593809604645,
      "logps/chosen": -278.79998779296875,
      "logps/rejected": -227.10000610351562,
      "loss": 0.675,
      "rewards/accuracies": 0.498452365398407,
      "rewards/chosen": 0.14213867485523224,
      "rewards/margins": 0.07265625149011612,
      "rewards/rejected": 0.069580078125,
      "step": 1260
    },
    {
      "epoch": 0.3256951596292482,
      "grad_norm": 302.0,
      "learning_rate": 3.371524201853759e-07,
      "logits/chosen": -0.5171874761581421,
      "logits/rejected": -0.40478515625,
      "logps/chosen": -251.8000030517578,
      "logps/rejected": -215.89999389648438,
      "loss": 0.632,
      "rewards/accuracies": 0.663690447807312,
      "rewards/chosen": 0.22080078721046448,
      "rewards/margins": 0.145751953125,
      "rewards/rejected": 0.07503204047679901,
      "step": 1265
    },
    {
      "epoch": 0.3269824922760041,
      "grad_norm": 183.0,
      "learning_rate": 3.3650875386199797e-07,
      "logits/chosen": -0.36213380098342896,
      "logits/rejected": -0.36982423067092896,
      "logps/chosen": -273.0,
      "logps/rejected": -259.3999938964844,
      "loss": 0.6453,
      "rewards/accuracies": 0.6616666913032532,
      "rewards/chosen": 0.188232421875,
      "rewards/margins": 0.11699219048023224,
      "rewards/rejected": 0.07167968899011612,
      "step": 1270
    },
    {
      "epoch": 0.32826982492276,
      "grad_norm": 100.5,
      "learning_rate": 3.3586508753861996e-07,
      "logits/chosen": -0.39238280057907104,
      "logits/rejected": -0.388671875,
      "logps/chosen": -325.79998779296875,
      "logps/rejected": -285.3999938964844,
      "loss": 0.6711,
      "rewards/accuracies": 0.47583335638046265,
      "rewards/chosen": 0.17133788764476776,
      "rewards/margins": 0.06977538764476776,
      "rewards/rejected": 0.10161133110523224,
      "step": 1275
    },
    {
      "epoch": 0.329557157569516,
      "grad_norm": 93.5,
      "learning_rate": 3.35221421215242e-07,
      "logits/chosen": -0.40742188692092896,
      "logits/rejected": -0.4332031309604645,
      "logps/chosen": -300.20001220703125,
      "logps/rejected": -294.79998779296875,
      "loss": 0.6711,
      "rewards/accuracies": 0.6074999570846558,
      "rewards/chosen": 0.20859375596046448,
      "rewards/margins": 0.05986328050494194,
      "rewards/rejected": 0.14892578125,
      "step": 1280
    },
    {
      "epoch": 0.3308444902162719,
      "grad_norm": 124.5,
      "learning_rate": 3.34577754891864e-07,
      "logits/chosen": -0.33574217557907104,
      "logits/rejected": -0.537890613079071,
      "logps/chosen": -300.6000061035156,
      "logps/rejected": -218.8000030517578,
      "loss": 0.6578,
      "rewards/accuracies": 0.4816666543483734,
      "rewards/chosen": 0.15107421576976776,
      "rewards/margins": 0.09340820461511612,
      "rewards/rejected": 0.05750121921300888,
      "step": 1285
    },
    {
      "epoch": 0.3321318228630278,
      "grad_norm": 135.0,
      "learning_rate": 3.339340885684861e-07,
      "logits/chosen": -0.2982421815395355,
      "logits/rejected": -0.2986389100551605,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -247.1999969482422,
      "loss": 0.6594,
      "rewards/accuracies": 0.504807710647583,
      "rewards/chosen": 0.17539063096046448,
      "rewards/margins": 0.08310546725988388,
      "rewards/rejected": 0.09234619140625,
      "step": 1290
    },
    {
      "epoch": 0.33341915550978374,
      "grad_norm": 84.0,
      "learning_rate": 3.3329042224510813e-07,
      "logits/chosen": -0.44843751192092896,
      "logits/rejected": -0.3956054747104645,
      "logps/chosen": -273.0,
      "logps/rejected": -269.20001220703125,
      "loss": 0.6477,
      "rewards/accuracies": 0.64000004529953,
      "rewards/chosen": 0.2513671815395355,
      "rewards/margins": 0.11667480319738388,
      "rewards/rejected": 0.13469238579273224,
      "step": 1295
    },
    {
      "epoch": 0.33470648815653964,
      "grad_norm": 164.0,
      "learning_rate": 3.326467559217301e-07,
      "logits/chosen": -0.3473144471645355,
      "logits/rejected": -0.3519531190395355,
      "logps/chosen": -352.79998779296875,
      "logps/rejected": -328.79998779296875,
      "loss": 0.6562,
      "rewards/accuracies": 0.4883333146572113,
      "rewards/chosen": 0.21855469048023224,
      "rewards/margins": 0.08737792819738388,
      "rewards/rejected": 0.13144531846046448,
      "step": 1300
    },
    {
      "epoch": 0.3359938208032956,
      "grad_norm": 132.0,
      "learning_rate": 3.320030895983522e-07,
      "logits/chosen": -0.38203126192092896,
      "logits/rejected": -0.4078125059604645,
      "logps/chosen": -248.0,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6617,
      "rewards/accuracies": 0.5383333563804626,
      "rewards/chosen": 0.177978515625,
      "rewards/margins": 0.06635741889476776,
      "rewards/rejected": 0.11123047024011612,
      "step": 1305
    },
    {
      "epoch": 0.3372811534500515,
      "grad_norm": 155.0,
      "learning_rate": 3.3135942327497426e-07,
      "logits/chosen": -0.27753907442092896,
      "logits/rejected": -0.2591308653354645,
      "logps/chosen": -242.1999969482422,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6719,
      "rewards/accuracies": 0.5213311910629272,
      "rewards/chosen": 0.14033202826976776,
      "rewards/margins": 0.05971679836511612,
      "rewards/rejected": 0.0806884765625,
      "step": 1310
    },
    {
      "epoch": 0.3385684860968074,
      "grad_norm": 202.0,
      "learning_rate": 3.3071575695159625e-07,
      "logits/chosen": -0.2955078184604645,
      "logits/rejected": -0.2730468809604645,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6414,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.251953125,
      "rewards/margins": 0.12333984673023224,
      "rewards/rejected": 0.12890625,
      "step": 1315
    },
    {
      "epoch": 0.33985581874356335,
      "grad_norm": 116.5,
      "learning_rate": 3.3007209062821835e-07,
      "logits/chosen": -0.34111326932907104,
      "logits/rejected": -0.4583984315395355,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -333.6000061035156,
      "loss": 0.6805,
      "rewards/accuracies": 0.45892858505249023,
      "rewards/chosen": 0.14951172471046448,
      "rewards/margins": 0.05434570461511612,
      "rewards/rejected": 0.09528808295726776,
      "step": 1320
    },
    {
      "epoch": 0.34114315139031925,
      "grad_norm": 163.0,
      "learning_rate": 3.2942842430484033e-07,
      "logits/chosen": -0.3904785215854645,
      "logits/rejected": -0.35541993379592896,
      "logps/chosen": -304.20001220703125,
      "logps/rejected": -230.1999969482422,
      "loss": 0.6414,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": 0.279296875,
      "rewards/margins": 0.12021484225988388,
      "rewards/rejected": 0.15898437798023224,
      "step": 1325
    },
    {
      "epoch": 0.34243048403707516,
      "grad_norm": 164.0,
      "learning_rate": 3.287847579814624e-07,
      "logits/chosen": -0.09995117038488388,
      "logits/rejected": -0.43603515625,
      "logps/chosen": -284.6000061035156,
      "logps/rejected": -258.0,
      "loss": 0.6414,
      "rewards/accuracies": 0.5495238304138184,
      "rewards/chosen": 0.19765624403953552,
      "rewards/margins": 0.131591796875,
      "rewards/rejected": 0.066162109375,
      "step": 1330
    },
    {
      "epoch": 0.3437178166838311,
      "grad_norm": 138.0,
      "learning_rate": 3.2814109165808447e-07,
      "logits/chosen": -0.25947266817092896,
      "logits/rejected": -0.3285156190395355,
      "logps/chosen": -296.3999938964844,
      "logps/rejected": -288.79998779296875,
      "loss": 0.6727,
      "rewards/accuracies": 0.5400000214576721,
      "rewards/chosen": 0.16396483778953552,
      "rewards/margins": 0.06116332858800888,
      "rewards/rejected": 0.10302734375,
      "step": 1335
    },
    {
      "epoch": 0.345005149330587,
      "grad_norm": 108.0,
      "learning_rate": 3.2749742533470646e-07,
      "logits/chosen": -0.44355469942092896,
      "logits/rejected": -0.3765625059604645,
      "logps/chosen": -314.79998779296875,
      "logps/rejected": -282.79998779296875,
      "loss": 0.6453,
      "rewards/accuracies": 0.5677198171615601,
      "rewards/chosen": 0.15234375,
      "rewards/margins": 0.11484374850988388,
      "rewards/rejected": 0.03746337816119194,
      "step": 1340
    },
    {
      "epoch": 0.34629248197734297,
      "grad_norm": 95.0,
      "learning_rate": 3.268537590113285e-07,
      "logits/chosen": -0.17304687201976776,
      "logits/rejected": -0.17861327528953552,
      "logps/chosen": -244.39999389648438,
      "logps/rejected": -251.60000610351562,
      "loss": 0.6797,
      "rewards/accuracies": 0.49383115768432617,
      "rewards/chosen": 0.12929686903953552,
      "rewards/margins": 0.04700317233800888,
      "rewards/rejected": 0.08214111626148224,
      "step": 1345
    },
    {
      "epoch": 0.34757981462409887,
      "grad_norm": 117.0,
      "learning_rate": 3.262100926879506e-07,
      "logits/chosen": -0.19291992485523224,
      "logits/rejected": 0.1123046875,
      "logps/chosen": -269.70001220703125,
      "logps/rejected": -279.5,
      "loss": 0.6547,
      "rewards/accuracies": 0.49000000953674316,
      "rewards/chosen": 0.20268554985523224,
      "rewards/margins": 0.10598144680261612,
      "rewards/rejected": 0.09658203274011612,
      "step": 1350
    },
    {
      "epoch": 0.34886714727085477,
      "grad_norm": 126.5,
      "learning_rate": 3.255664263645726e-07,
      "logits/chosen": -0.3326263427734375,
      "logits/rejected": -0.3472656309604645,
      "logps/chosen": -241.0,
      "logps/rejected": -236.0,
      "loss": 0.6398,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21406249701976776,
      "rewards/margins": 0.12285156548023224,
      "rewards/rejected": 0.09099121391773224,
      "step": 1355
    },
    {
      "epoch": 0.35015447991761073,
      "grad_norm": 91.0,
      "learning_rate": 3.2492276004119463e-07,
      "logits/chosen": -0.28466796875,
      "logits/rejected": -0.3404296934604645,
      "logps/chosen": -237.60000610351562,
      "logps/rejected": -220.1999969482422,
      "loss": 0.6609,
      "rewards/accuracies": 0.47794872522354126,
      "rewards/chosen": 0.17924804985523224,
      "rewards/margins": 0.08461914211511612,
      "rewards/rejected": 0.09496460109949112,
      "step": 1360
    },
    {
      "epoch": 0.35144181256436663,
      "grad_norm": 106.0,
      "learning_rate": 3.242790937178166e-07,
      "logits/chosen": -0.4496093690395355,
      "logits/rejected": -0.38671875,
      "logps/chosen": -270.79998779296875,
      "logps/rejected": -298.3999938964844,
      "loss": 0.6734,
      "rewards/accuracies": 0.4970238208770752,
      "rewards/chosen": 0.164031982421875,
      "rewards/margins": 0.06982421875,
      "rewards/rejected": 0.09423828125,
      "step": 1365
    },
    {
      "epoch": 0.35272914521112253,
      "grad_norm": 131.0,
      "learning_rate": 3.236354273944387e-07,
      "logits/chosen": -0.35810548067092896,
      "logits/rejected": -0.35283201932907104,
      "logps/chosen": -255.0,
      "logps/rejected": -218.39999389648438,
      "loss": 0.6687,
      "rewards/accuracies": 0.48261910676956177,
      "rewards/chosen": 0.153076171875,
      "rewards/margins": 0.06501464545726776,
      "rewards/rejected": 0.08802489936351776,
      "step": 1370
    },
    {
      "epoch": 0.3540164778578785,
      "grad_norm": 202.0,
      "learning_rate": 3.2299176107106076e-07,
      "logits/chosen": -0.34034425020217896,
      "logits/rejected": -0.38398438692092896,
      "logps/chosen": -287.6000061035156,
      "logps/rejected": -332.0,
      "loss": 0.6836,
      "rewards/accuracies": 0.4651923179626465,
      "rewards/chosen": 0.11689452826976776,
      "rewards/margins": 0.03635253757238388,
      "rewards/rejected": 0.08027343451976776,
      "step": 1375
    },
    {
      "epoch": 0.3553038105046344,
      "grad_norm": 97.5,
      "learning_rate": 3.2234809474768275e-07,
      "logits/chosen": -0.3798828125,
      "logits/rejected": -0.33989256620407104,
      "logps/chosen": -320.3999938964844,
      "logps/rejected": -253.39999389648438,
      "loss": 0.6406,
      "rewards/accuracies": 0.6100000143051147,
      "rewards/chosen": 0.19833984971046448,
      "rewards/margins": 0.11789550632238388,
      "rewards/rejected": 0.08027343451976776,
      "step": 1380
    },
    {
      "epoch": 0.35659114315139034,
      "grad_norm": 98.5,
      "learning_rate": 3.2170442842430485e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.44257813692092896,
      "logps/chosen": -238.1999969482422,
      "logps/rejected": -231.8000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.5429762601852417,
      "rewards/chosen": 0.15791015326976776,
      "rewards/margins": 0.0517578125,
      "rewards/rejected": 0.10604248195886612,
      "step": 1385
    },
    {
      "epoch": 0.35787847579814624,
      "grad_norm": 176.0,
      "learning_rate": 3.2106076210092684e-07,
      "logits/chosen": -0.36699217557907104,
      "logits/rejected": -0.4261718690395355,
      "logps/chosen": -314.20001220703125,
      "logps/rejected": -283.20001220703125,
      "loss": 0.6672,
      "rewards/accuracies": 0.442976176738739,
      "rewards/chosen": 0.17578125,
      "rewards/margins": 0.07120513916015625,
      "rewards/rejected": 0.10457763820886612,
      "step": 1390
    },
    {
      "epoch": 0.35916580844490215,
      "grad_norm": 117.0,
      "learning_rate": 3.204170957775489e-07,
      "logits/chosen": -0.3670898377895355,
      "logits/rejected": -0.18095703423023224,
      "logps/chosen": -239.0,
      "logps/rejected": -223.1999969482422,
      "loss": 0.675,
      "rewards/accuracies": 0.5643590092658997,
      "rewards/chosen": 0.08120117336511612,
      "rewards/margins": 0.05629272386431694,
      "rewards/rejected": 0.02482910081744194,
      "step": 1395
    },
    {
      "epoch": 0.3604531410916581,
      "grad_norm": 90.5,
      "learning_rate": 3.19773429454171e-07,
      "logits/chosen": -0.3509765565395355,
      "logits/rejected": -0.4234375059604645,
      "logps/chosen": -296.0,
      "logps/rejected": -241.1999969482422,
      "loss": 0.6516,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.20175781846046448,
      "rewards/margins": 0.10947265475988388,
      "rewards/rejected": 0.09243164211511612,
      "step": 1400
    },
    {
      "epoch": 0.361740473738414,
      "grad_norm": 109.0,
      "learning_rate": 3.1912976313079296e-07,
      "logits/chosen": -0.3349609375,
      "logits/rejected": -0.3687500059604645,
      "logps/chosen": -316.20001220703125,
      "logps/rejected": -274.6000061035156,
      "loss": 0.6711,
      "rewards/accuracies": 0.5240476727485657,
      "rewards/chosen": 0.19550780951976776,
      "rewards/margins": 0.06968994438648224,
      "rewards/rejected": 0.12568359076976776,
      "step": 1405
    },
    {
      "epoch": 0.3630278063851699,
      "grad_norm": 117.5,
      "learning_rate": 3.18486096807415e-07,
      "logits/chosen": -0.26079100370407104,
      "logits/rejected": -0.28974610567092896,
      "logps/chosen": -296.20001220703125,
      "logps/rejected": -319.79998779296875,
      "loss": 0.6687,
      "rewards/accuracies": 0.5049999952316284,
      "rewards/chosen": 0.19624023139476776,
      "rewards/margins": 0.06992187350988388,
      "rewards/rejected": 0.12653808295726776,
      "step": 1410
    },
    {
      "epoch": 0.36431513903192586,
      "grad_norm": 101.5,
      "learning_rate": 3.178424304840371e-07,
      "logits/chosen": -0.2916015684604645,
      "logits/rejected": -0.28779298067092896,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -255.0,
      "loss": 0.6531,
      "rewards/accuracies": 0.5633333921432495,
      "rewards/chosen": 0.15009765326976776,
      "rewards/margins": 0.10080566257238388,
      "rewards/rejected": 0.049560546875,
      "step": 1415
    },
    {
      "epoch": 0.36560247167868176,
      "grad_norm": 104.0,
      "learning_rate": 3.171987641606591e-07,
      "logits/chosen": -0.13540038466453552,
      "logits/rejected": -0.2679687440395355,
      "logps/chosen": -269.6000061035156,
      "logps/rejected": -251.60000610351562,
      "loss": 0.6539,
      "rewards/accuracies": 0.5022727251052856,
      "rewards/chosen": 0.18925781548023224,
      "rewards/margins": 0.09144286811351776,
      "rewards/rejected": 0.09780273586511612,
      "step": 1420
    },
    {
      "epoch": 0.3668898043254377,
      "grad_norm": 122.0,
      "learning_rate": 3.1655509783728114e-07,
      "logits/chosen": -0.39335936307907104,
      "logits/rejected": -0.51171875,
      "logps/chosen": -318.3999938964844,
      "logps/rejected": -316.79998779296875,
      "loss": 0.6352,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.24423828721046448,
      "rewards/margins": 0.13857421278953552,
      "rewards/rejected": 0.10595703125,
      "step": 1425
    },
    {
      "epoch": 0.3681771369721936,
      "grad_norm": 140.0,
      "learning_rate": 3.159114315139032e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": -0.507031261920929,
      "logps/chosen": -279.6000061035156,
      "logps/rejected": -204.8000030517578,
      "loss": 0.6438,
      "rewards/accuracies": 0.6014286279678345,
      "rewards/chosen": 0.13837890326976776,
      "rewards/margins": 0.10507812350988388,
      "rewards/rejected": 0.03331756591796875,
      "step": 1430
    },
    {
      "epoch": 0.3694644696189495,
      "grad_norm": 122.5,
      "learning_rate": 3.152677651905252e-07,
      "logits/chosen": -0.2470703125,
      "logits/rejected": -0.33769530057907104,
      "logps/chosen": -220.8000030517578,
      "logps/rejected": -244.39999389648438,
      "loss": 0.6539,
      "rewards/accuracies": 0.5422435998916626,
      "rewards/chosen": 0.18896484375,
      "rewards/margins": 0.10527344048023224,
      "rewards/rejected": 0.08388672024011612,
      "step": 1435
    },
    {
      "epoch": 0.3707518022657055,
      "grad_norm": 178.0,
      "learning_rate": 3.1462409886714726e-07,
      "logits/chosen": -0.3421874940395355,
      "logits/rejected": -0.28339844942092896,
      "logps/chosen": -290.0,
      "logps/rejected": -246.60000610351562,
      "loss": 0.6711,
      "rewards/accuracies": 0.5269230604171753,
      "rewards/chosen": 0.11016187816858292,
      "rewards/margins": 0.05515136569738388,
      "rewards/rejected": 0.05485839769244194,
      "step": 1440
    },
    {
      "epoch": 0.3720391349124614,
      "grad_norm": 142.0,
      "learning_rate": 3.1398043254376925e-07,
      "logits/chosen": -0.26513671875,
      "logits/rejected": -0.28269654512405396,
      "logps/chosen": -280.6000061035156,
      "logps/rejected": -244.0,
      "loss": 0.6523,
      "rewards/accuracies": 0.6444047689437866,
      "rewards/chosen": 0.21171875298023224,
      "rewards/margins": 0.09560547024011612,
      "rewards/rejected": 0.11611328274011612,
      "step": 1445
    },
    {
      "epoch": 0.3733264675592173,
      "grad_norm": 167.0,
      "learning_rate": 3.1333676622039135e-07,
      "logits/chosen": -0.4039062559604645,
      "logits/rejected": -0.4742187559604645,
      "logps/chosen": -356.79998779296875,
      "logps/rejected": -289.0,
      "loss": 0.6516,
      "rewards/accuracies": 0.5449999570846558,
      "rewards/chosen": 0.14197082817554474,
      "rewards/margins": 0.09956054389476776,
      "rewards/rejected": 0.04241333156824112,
      "step": 1450
    },
    {
      "epoch": 0.37461380020597324,
      "grad_norm": 145.0,
      "learning_rate": 3.126930998970134e-07,
      "logits/chosen": -0.26875001192092896,
      "logits/rejected": -0.2847656309604645,
      "logps/chosen": -242.8000030517578,
      "logps/rejected": -224.60000610351562,
      "loss": 0.6781,
      "rewards/accuracies": 0.5344048142433167,
      "rewards/chosen": 0.20131835341453552,
      "rewards/margins": 0.05087890475988388,
      "rewards/rejected": 0.15029296278953552,
      "step": 1455
    },
    {
      "epoch": 0.37590113285272914,
      "grad_norm": 118.5,
      "learning_rate": 3.120494335736354e-07,
      "logits/chosen": -0.2676025331020355,
      "logits/rejected": -0.25124281644821167,
      "logps/chosen": -322.79998779296875,
      "logps/rejected": -300.3999938964844,
      "loss": 0.6641,
      "rewards/accuracies": 0.5995238423347473,
      "rewards/chosen": 0.16650390625,
      "rewards/margins": 0.078125,
      "rewards/rejected": 0.08852539211511612,
      "step": 1460
    },
    {
      "epoch": 0.3771884654994851,
      "grad_norm": 116.0,
      "learning_rate": 3.114057672502575e-07,
      "logits/chosen": -0.3353515565395355,
      "logits/rejected": -0.26640623807907104,
      "logps/chosen": -295.0,
      "logps/rejected": -262.6000061035156,
      "loss": 0.6492,
      "rewards/accuracies": 0.5824999809265137,
      "rewards/chosen": 0.18447265028953552,
      "rewards/margins": 0.117218017578125,
      "rewards/rejected": 0.06720886379480362,
      "step": 1465
    },
    {
      "epoch": 0.378475798146241,
      "grad_norm": 140.0,
      "learning_rate": 3.1076210092687947e-07,
      "logits/chosen": -0.4429687559604645,
      "logits/rejected": -0.42265623807907104,
      "logps/chosen": -316.0,
      "logps/rejected": -249.39999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.45673075318336487,
      "rewards/chosen": 0.14667968451976776,
      "rewards/margins": 0.09638671576976776,
      "rewards/rejected": 0.05002441257238388,
      "step": 1470
    },
    {
      "epoch": 0.3797631307929969,
      "grad_norm": 123.0,
      "learning_rate": 3.101184346035015e-07,
      "logits/chosen": -0.3358398377895355,
      "logits/rejected": -0.4095703065395355,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -299.20001220703125,
      "loss": 0.6531,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.17398682236671448,
      "rewards/margins": 0.095703125,
      "rewards/rejected": 0.07835998386144638,
      "step": 1475
    },
    {
      "epoch": 0.38105046343975285,
      "grad_norm": 256.0,
      "learning_rate": 3.094747682801236e-07,
      "logits/chosen": -0.30585938692092896,
      "logits/rejected": 0.13134765625,
      "logps/chosen": -200.3000030517578,
      "logps/rejected": -227.1999969482422,
      "loss": 0.657,
      "rewards/accuracies": 0.5071861147880554,
      "rewards/chosen": 0.15703125298023224,
      "rewards/margins": 0.0870513916015625,
      "rewards/rejected": 0.07016601413488388,
      "step": 1480
    },
    {
      "epoch": 0.38233779608650875,
      "grad_norm": 152.0,
      "learning_rate": 3.088311019567456e-07,
      "logits/chosen": -0.36054688692092896,
      "logits/rejected": -0.2787109315395355,
      "logps/chosen": -307.0,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6844,
      "rewards/accuracies": 0.5488461256027222,
      "rewards/chosen": 0.17275390028953552,
      "rewards/margins": 0.05043945461511612,
      "rewards/rejected": 0.12226562201976776,
      "step": 1485
    },
    {
      "epoch": 0.38362512873326465,
      "grad_norm": 414.0,
      "learning_rate": 3.0818743563336764e-07,
      "logits/chosen": -0.4029296934604645,
      "logits/rejected": -0.32929688692092896,
      "logps/chosen": -305.3999938964844,
      "logps/rejected": -249.1999969482422,
      "loss": 0.6508,
      "rewards/accuracies": 0.5824999809265137,
      "rewards/chosen": 0.17539063096046448,
      "rewards/margins": 0.10776367038488388,
      "rewards/rejected": 0.06748046725988388,
      "step": 1490
    },
    {
      "epoch": 0.3849124613800206,
      "grad_norm": 98.5,
      "learning_rate": 3.0754376930998973e-07,
      "logits/chosen": -0.40703123807907104,
      "logits/rejected": -0.3369140625,
      "logps/chosen": -286.20001220703125,
      "logps/rejected": -254.60000610351562,
      "loss": 0.6508,
      "rewards/accuracies": 0.5805769562721252,
      "rewards/chosen": 0.14765624701976776,
      "rewards/margins": 0.10478515923023224,
      "rewards/rejected": 0.04301757737994194,
      "step": 1495
    },
    {
      "epoch": 0.3861997940267765,
      "grad_norm": 452.0,
      "learning_rate": 3.069001029866117e-07,
      "logits/chosen": -0.3480468690395355,
      "logits/rejected": -0.31425780057907104,
      "logps/chosen": -264.3999938964844,
      "logps/rejected": -251.60000610351562,
      "loss": 0.6672,
      "rewards/accuracies": 0.48000001907348633,
      "rewards/chosen": 0.17900390923023224,
      "rewards/margins": 0.07478027045726776,
      "rewards/rejected": 0.10405273735523224,
      "step": 1500
    },
    {
      "epoch": 0.38748712667353247,
      "grad_norm": 156.0,
      "learning_rate": 3.0625643666323377e-07,
      "logits/chosen": -0.3929687440395355,
      "logits/rejected": -0.37089842557907104,
      "logps/chosen": -292.20001220703125,
      "logps/rejected": -286.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.5441666841506958,
      "rewards/chosen": 0.22832031548023224,
      "rewards/margins": 0.10517577826976776,
      "rewards/rejected": 0.123291015625,
      "step": 1505
    },
    {
      "epoch": 0.38877445932028837,
      "grad_norm": 127.5,
      "learning_rate": 3.056127703398558e-07,
      "logits/chosen": -0.31855469942092896,
      "logits/rejected": -0.358468621969223,
      "logps/chosen": -314.0,
      "logps/rejected": -257.20001220703125,
      "loss": 0.6719,
      "rewards/accuracies": 0.4732051491737366,
      "rewards/chosen": 0.11520995944738388,
      "rewards/margins": 0.04976959154009819,
      "rewards/rejected": 0.06555785983800888,
      "step": 1510
    },
    {
      "epoch": 0.39006179196704427,
      "grad_norm": 248.0,
      "learning_rate": 3.0496910401647785e-07,
      "logits/chosen": -0.3921875059604645,
      "logits/rejected": -0.30644530057907104,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -284.20001220703125,
      "loss": 0.6648,
      "rewards/accuracies": 0.45595234632492065,
      "rewards/chosen": 0.13022461533546448,
      "rewards/margins": 0.08217773586511612,
      "rewards/rejected": 0.048126220703125,
      "step": 1515
    },
    {
      "epoch": 0.3913491246138002,
      "grad_norm": 230.0,
      "learning_rate": 3.043254376930999e-07,
      "logits/chosen": -0.44999998807907104,
      "logits/rejected": -0.42451173067092896,
      "logps/chosen": -277.6000061035156,
      "logps/rejected": -281.0,
      "loss": 0.6617,
      "rewards/accuracies": 0.5978571176528931,
      "rewards/chosen": 0.21054688096046448,
      "rewards/margins": 0.08189697563648224,
      "rewards/rejected": 0.12861327826976776,
      "step": 1520
    },
    {
      "epoch": 0.3926364572605561,
      "grad_norm": 110.5,
      "learning_rate": 3.036817713697219e-07,
      "logits/chosen": -0.3705078065395355,
      "logits/rejected": -0.4175781309604645,
      "logps/chosen": -303.79998779296875,
      "logps/rejected": -265.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.17050781846046448,
      "rewards/margins": 0.1295166015625,
      "rewards/rejected": 0.04135742038488388,
      "step": 1525
    },
    {
      "epoch": 0.393923789907312,
      "grad_norm": 186.0,
      "learning_rate": 3.03038105046344e-07,
      "logits/chosen": -0.32854002714157104,
      "logits/rejected": -0.34980469942092896,
      "logps/chosen": -250.5,
      "logps/rejected": -257.6000061035156,
      "loss": 0.6539,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.19921875,
      "rewards/margins": 0.09882812201976776,
      "rewards/rejected": 0.10041503608226776,
      "step": 1530
    },
    {
      "epoch": 0.395211122554068,
      "grad_norm": 121.0,
      "learning_rate": 3.0239443872296597e-07,
      "logits/chosen": -0.2544921934604645,
      "logits/rejected": -0.3046875,
      "logps/chosen": -238.6999969482422,
      "logps/rejected": -230.8000030517578,
      "loss": 0.643,
      "rewards/accuracies": 0.6197222471237183,
      "rewards/chosen": 0.192138671875,
      "rewards/margins": 0.10544433444738388,
      "rewards/rejected": 0.08688964694738388,
      "step": 1535
    },
    {
      "epoch": 0.3964984552008239,
      "grad_norm": 124.5,
      "learning_rate": 3.01750772399588e-07,
      "logits/chosen": -0.29082030057907104,
      "logits/rejected": -0.37480467557907104,
      "logps/chosen": -291.6000061035156,
      "logps/rejected": -342.0,
      "loss": 0.6758,
      "rewards/accuracies": 0.5558333396911621,
      "rewards/chosen": 0.220458984375,
      "rewards/margins": 0.064453125,
      "rewards/rejected": 0.15634766221046448,
      "step": 1540
    },
    {
      "epoch": 0.39778578784757984,
      "grad_norm": 264.0,
      "learning_rate": 3.011071060762101e-07,
      "logits/chosen": -0.23115234076976776,
      "logits/rejected": -0.28984373807907104,
      "logps/chosen": -253.1999969482422,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6375,
      "rewards/accuracies": 0.5232142806053162,
      "rewards/chosen": 0.19399413466453552,
      "rewards/margins": 0.12734374403953552,
      "rewards/rejected": 0.06694336235523224,
      "step": 1545
    },
    {
      "epoch": 0.39907312049433574,
      "grad_norm": 107.5,
      "learning_rate": 3.004634397528321e-07,
      "logits/chosen": -0.28886717557907104,
      "logits/rejected": -0.4203124940395355,
      "logps/chosen": -285.0,
      "logps/rejected": -226.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.5326923131942749,
      "rewards/chosen": 0.13625487685203552,
      "rewards/margins": 0.10390625149011612,
      "rewards/rejected": 0.03229980543255806,
      "step": 1550
    },
    {
      "epoch": 0.40036045314109164,
      "grad_norm": 108.0,
      "learning_rate": 2.9981977342945414e-07,
      "logits/chosen": -0.4244140684604645,
      "logits/rejected": -0.4322265684604645,
      "logps/chosen": -293.3999938964844,
      "logps/rejected": -285.20001220703125,
      "loss": 0.6438,
      "rewards/accuracies": 0.6441667079925537,
      "rewards/chosen": 0.19267578423023224,
      "rewards/margins": 0.11577148735523224,
      "rewards/rejected": 0.07695312798023224,
      "step": 1555
    },
    {
      "epoch": 0.4016477857878476,
      "grad_norm": 101.0,
      "learning_rate": 2.9917610710607623e-07,
      "logits/chosen": -0.2587890625,
      "logits/rejected": -0.26484376192092896,
      "logps/chosen": -316.0,
      "logps/rejected": -271.20001220703125,
      "loss": 0.6477,
      "rewards/accuracies": 0.5566666722297668,
      "rewards/chosen": 0.2510742247104645,
      "rewards/margins": 0.12043152004480362,
      "rewards/rejected": 0.13041992485523224,
      "step": 1560
    },
    {
      "epoch": 0.4029351184346035,
      "grad_norm": 170.0,
      "learning_rate": 2.985324407826982e-07,
      "logits/chosen": -0.3863281309604645,
      "logits/rejected": -0.4007812440395355,
      "logps/chosen": -276.3999938964844,
      "logps/rejected": -262.20001220703125,
      "loss": 0.6617,
      "rewards/accuracies": 0.5244231224060059,
      "rewards/chosen": 0.15839843451976776,
      "rewards/margins": 0.083740234375,
      "rewards/rejected": 0.07463379204273224,
      "step": 1565
    },
    {
      "epoch": 0.4042224510813594,
      "grad_norm": 192.0,
      "learning_rate": 2.9788877445932027e-07,
      "logits/chosen": -0.3701171875,
      "logits/rejected": -0.32636719942092896,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6562,
      "rewards/accuracies": 0.5408333539962769,
      "rewards/chosen": 0.20371094346046448,
      "rewards/margins": 0.07912597805261612,
      "rewards/rejected": 0.12431640923023224,
      "step": 1570
    },
    {
      "epoch": 0.40550978372811536,
      "grad_norm": 110.0,
      "learning_rate": 2.972451081359423e-07,
      "logits/chosen": -0.19746093451976776,
      "logits/rejected": -0.3075195252895355,
      "logps/chosen": -245.0,
      "logps/rejected": -232.1999969482422,
      "loss": 0.6742,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2158203125,
      "rewards/margins": 0.05039062350988388,
      "rewards/rejected": 0.16533203423023224,
      "step": 1575
    },
    {
      "epoch": 0.40679711637487126,
      "grad_norm": 107.5,
      "learning_rate": 2.9660144181256435e-07,
      "logits/chosen": -0.36674803495407104,
      "logits/rejected": -0.42890626192092896,
      "logps/chosen": -267.6000061035156,
      "logps/rejected": -211.1999969482422,
      "loss": 0.6656,
      "rewards/accuracies": 0.49152931571006775,
      "rewards/chosen": 0.13916015625,
      "rewards/margins": 0.07094726711511612,
      "rewards/rejected": 0.06815490871667862,
      "step": 1580
    },
    {
      "epoch": 0.4080844490216272,
      "grad_norm": 133.0,
      "learning_rate": 2.959577754891864e-07,
      "logits/chosen": -0.3544921875,
      "logits/rejected": -0.3736328184604645,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -279.79998779296875,
      "loss": 0.6484,
      "rewards/accuracies": 0.5321428775787354,
      "rewards/chosen": 0.16259765625,
      "rewards/margins": 0.10585937649011612,
      "rewards/rejected": 0.05673827975988388,
      "step": 1585
    },
    {
      "epoch": 0.4093717816683831,
      "grad_norm": 932.0,
      "learning_rate": 2.9531410916580844e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": -0.4105468690395355,
      "logps/chosen": -264.3999938964844,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6398,
      "rewards/accuracies": 0.7228571772575378,
      "rewards/chosen": 0.18369141221046448,
      "rewards/margins": 0.12387695163488388,
      "rewards/rejected": 0.06002197414636612,
      "step": 1590
    },
    {
      "epoch": 0.410659114315139,
      "grad_norm": 176.0,
      "learning_rate": 2.946704428424305e-07,
      "logits/chosen": -0.4058593809604645,
      "logits/rejected": -0.33281248807907104,
      "logps/chosen": -302.79998779296875,
      "logps/rejected": -271.6000061035156,
      "loss": 0.6562,
      "rewards/accuracies": 0.5741666555404663,
      "rewards/chosen": 0.17680664360523224,
      "rewards/margins": 0.10500488430261612,
      "rewards/rejected": 0.071563720703125,
      "step": 1595
    },
    {
      "epoch": 0.411946446961895,
      "grad_norm": 177.0,
      "learning_rate": 2.940267765190525e-07,
      "logits/chosen": -0.3017578125,
      "logits/rejected": -0.2886718809604645,
      "logps/chosen": -305.0,
      "logps/rejected": -248.60000610351562,
      "loss": 0.6617,
      "rewards/accuracies": 0.4763888716697693,
      "rewards/chosen": 0.1026611328125,
      "rewards/margins": 0.07270507514476776,
      "rewards/rejected": 0.02990417554974556,
      "step": 1600
    },
    {
      "epoch": 0.4132337796086509,
      "grad_norm": 121.0,
      "learning_rate": 2.933831101956745e-07,
      "logits/chosen": -0.3873046934604645,
      "logits/rejected": -0.3714843690395355,
      "logps/chosen": -294.79998779296875,
      "logps/rejected": -238.8000030517578,
      "loss": 0.6641,
      "rewards/accuracies": 0.4743589758872986,
      "rewards/chosen": 0.14101561903953552,
      "rewards/margins": 0.0701904296875,
      "rewards/rejected": 0.07082519680261612,
      "step": 1605
    },
    {
      "epoch": 0.4145211122554068,
      "grad_norm": 131.0,
      "learning_rate": 2.927394438722966e-07,
      "logits/chosen": -0.41523438692092896,
      "logits/rejected": -0.33613282442092896,
      "logps/chosen": -236.39999389648438,
      "logps/rejected": -221.0,
      "loss": 0.6367,
      "rewards/accuracies": 0.5698052048683167,
      "rewards/chosen": 0.22592774033546448,
      "rewards/margins": 0.15703125298023224,
      "rewards/rejected": 0.06914062798023224,
      "step": 1610
    },
    {
      "epoch": 0.41580844490216273,
      "grad_norm": 148.0,
      "learning_rate": 2.920957775489186e-07,
      "logits/chosen": -0.2768257260322571,
      "logits/rejected": -0.34003907442092896,
      "logps/chosen": -292.0,
      "logps/rejected": -278.79998779296875,
      "loss": 0.6648,
      "rewards/accuracies": 0.5535256266593933,
      "rewards/chosen": 0.19902344048023224,
      "rewards/margins": 0.07993163913488388,
      "rewards/rejected": 0.11904297024011612,
      "step": 1615
    },
    {
      "epoch": 0.41709577754891863,
      "grad_norm": 185.0,
      "learning_rate": 2.9145211122554064e-07,
      "logits/chosen": -0.46015626192092896,
      "logits/rejected": -0.39433592557907104,
      "logps/chosen": -331.3999938964844,
      "logps/rejected": -294.20001220703125,
      "loss": 0.6641,
      "rewards/accuracies": 0.5433333516120911,
      "rewards/chosen": 0.13798828423023224,
      "rewards/margins": 0.07363281399011612,
      "rewards/rejected": 0.06406249850988388,
      "step": 1620
    },
    {
      "epoch": 0.41838311019567453,
      "grad_norm": 140.0,
      "learning_rate": 2.9080844490216274e-07,
      "logits/chosen": -0.3705078065395355,
      "logits/rejected": -0.2931884825229645,
      "logps/chosen": -251.1999969482422,
      "logps/rejected": -226.0,
      "loss": 0.6586,
      "rewards/accuracies": 0.5169230699539185,
      "rewards/chosen": 0.15371093153953552,
      "rewards/margins": 0.08803711086511612,
      "rewards/rejected": 0.06591796875,
      "step": 1625
    },
    {
      "epoch": 0.4196704428424305,
      "grad_norm": 104.5,
      "learning_rate": 2.9016477857878473e-07,
      "logits/chosen": -0.44062501192092896,
      "logits/rejected": -0.44648438692092896,
      "logps/chosen": -298.3999938964844,
      "logps/rejected": -246.39999389648438,
      "loss": 0.6469,
      "rewards/accuracies": 0.5940934419631958,
      "rewards/chosen": 0.2021484375,
      "rewards/margins": 0.11259765923023224,
      "rewards/rejected": 0.0897216796875,
      "step": 1630
    },
    {
      "epoch": 0.4209577754891864,
      "grad_norm": 145.0,
      "learning_rate": 2.8952111225540677e-07,
      "logits/chosen": -0.24609375,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -281.0,
      "logps/rejected": -254.14999389648438,
      "loss": 0.6531,
      "rewards/accuracies": 0.5617856979370117,
      "rewards/chosen": 0.16269531846046448,
      "rewards/margins": 0.09514160454273224,
      "rewards/rejected": 0.06740722805261612,
      "step": 1635
    },
    {
      "epoch": 0.42224510813594235,
      "grad_norm": 190.0,
      "learning_rate": 2.8887744593202886e-07,
      "logits/chosen": -0.4046874940395355,
      "logits/rejected": -0.283447265625,
      "logps/chosen": -308.0,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6687,
      "rewards/accuracies": 0.4791666865348816,
      "rewards/chosen": 0.24228516221046448,
      "rewards/margins": 0.072998046875,
      "rewards/rejected": 0.1689453125,
      "step": 1640
    },
    {
      "epoch": 0.42353244078269825,
      "grad_norm": 182.0,
      "learning_rate": 2.8823377960865085e-07,
      "logits/chosen": -0.4027343690395355,
      "logits/rejected": -0.4351562559604645,
      "logps/chosen": -288.20001220703125,
      "logps/rejected": -282.6000061035156,
      "loss": 0.6578,
      "rewards/accuracies": 0.5441666841506958,
      "rewards/chosen": 0.18710938096046448,
      "rewards/margins": 0.09380493313074112,
      "rewards/rejected": 0.09348144382238388,
      "step": 1645
    },
    {
      "epoch": 0.42481977342945415,
      "grad_norm": 134.0,
      "learning_rate": 2.875901132852729e-07,
      "logits/chosen": -0.42500001192092896,
      "logits/rejected": -0.376953125,
      "logps/chosen": -237.1999969482422,
      "logps/rejected": -236.0,
      "loss": 0.6289,
      "rewards/accuracies": 0.6316667199134827,
      "rewards/chosen": 0.17519530653953552,
      "rewards/margins": 0.14345702528953552,
      "rewards/rejected": 0.03142089769244194,
      "step": 1650
    },
    {
      "epoch": 0.4261071060762101,
      "grad_norm": 102.0,
      "learning_rate": 2.8694644696189494e-07,
      "logits/chosen": -0.3539062440395355,
      "logits/rejected": -0.3232421875,
      "logps/chosen": -246.8000030517578,
      "logps/rejected": -239.39999389648438,
      "loss": 0.6633,
      "rewards/accuracies": 0.4753788113594055,
      "rewards/chosen": 0.16704101860523224,
      "rewards/margins": 0.07192382961511612,
      "rewards/rejected": 0.09482421725988388,
      "step": 1655
    },
    {
      "epoch": 0.427394438722966,
      "grad_norm": 114.0,
      "learning_rate": 2.86302780638517e-07,
      "logits/chosen": -0.3316406309604645,
      "logits/rejected": -0.3482421934604645,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -306.3999938964844,
      "loss": 0.675,
      "rewards/accuracies": 0.54666668176651,
      "rewards/chosen": 0.2724609375,
      "rewards/margins": 0.06416015326976776,
      "rewards/rejected": 0.20839843153953552,
      "step": 1660
    },
    {
      "epoch": 0.4286817713697219,
      "grad_norm": 272.0,
      "learning_rate": 2.85659114315139e-07,
      "logits/chosen": -0.4175781309604645,
      "logits/rejected": -0.5347656011581421,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -348.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.5592857599258423,
      "rewards/chosen": 0.19423827528953552,
      "rewards/margins": 0.08144531399011612,
      "rewards/rejected": 0.11263952404260635,
      "step": 1665
    },
    {
      "epoch": 0.42996910401647787,
      "grad_norm": 290.0,
      "learning_rate": 2.8501544799176107e-07,
      "logits/chosen": -0.4072265625,
      "logits/rejected": -0.26567381620407104,
      "logps/chosen": -326.79998779296875,
      "logps/rejected": -290.79998779296875,
      "loss": 0.6617,
      "rewards/accuracies": 0.4732142984867096,
      "rewards/chosen": 0.11909179389476776,
      "rewards/margins": 0.07384033501148224,
      "rewards/rejected": 0.04541015625,
      "step": 1670
    },
    {
      "epoch": 0.43125643666323377,
      "grad_norm": 141.0,
      "learning_rate": 2.843717816683831e-07,
      "logits/chosen": -0.3687500059604645,
      "logits/rejected": -0.4476562440395355,
      "logps/chosen": -246.8000030517578,
      "logps/rejected": -202.1999969482422,
      "loss": 0.668,
      "rewards/accuracies": 0.5049999952316284,
      "rewards/chosen": 0.16640624403953552,
      "rewards/margins": 0.07391357421875,
      "rewards/rejected": 0.09250488132238388,
      "step": 1675
    },
    {
      "epoch": 0.4325437693099897,
      "grad_norm": 612.0,
      "learning_rate": 2.837281153450051e-07,
      "logits/chosen": -0.4800781309604645,
      "logits/rejected": -0.598437488079071,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -198.3000030517578,
      "loss": 0.6672,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.22397461533546448,
      "rewards/margins": 0.07147216796875,
      "rewards/rejected": 0.1527099609375,
      "step": 1680
    },
    {
      "epoch": 0.4338311019567456,
      "grad_norm": 123.5,
      "learning_rate": 2.830844490216272e-07,
      "logits/chosen": -0.4457031190395355,
      "logits/rejected": -0.501953125,
      "logps/chosen": -323.6000061035156,
      "logps/rejected": -275.79998779296875,
      "loss": 0.6711,
      "rewards/accuracies": 0.4880952835083008,
      "rewards/chosen": 0.16132812201976776,
      "rewards/margins": 0.05958252027630806,
      "rewards/rejected": 0.10185547173023224,
      "step": 1685
    },
    {
      "epoch": 0.4351184346035015,
      "grad_norm": 153.0,
      "learning_rate": 2.8244078269824924e-07,
      "logits/chosen": -0.37773436307907104,
      "logits/rejected": -0.33867186307907104,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -264.3999938964844,
      "loss": 0.6359,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": 0.2572265565395355,
      "rewards/margins": 0.14130859076976776,
      "rewards/rejected": 0.11612548679113388,
      "step": 1690
    },
    {
      "epoch": 0.4364057672502575,
      "grad_norm": 122.0,
      "learning_rate": 2.8179711637487123e-07,
      "logits/chosen": -0.2554687559604645,
      "logits/rejected": -0.3095703125,
      "logps/chosen": -269.79998779296875,
      "logps/rejected": -253.39999389648438,
      "loss": 0.6375,
      "rewards/accuracies": 0.5608333349227905,
      "rewards/chosen": 0.20009765028953552,
      "rewards/margins": 0.13425293564796448,
      "rewards/rejected": 0.06601562350988388,
      "step": 1695
    },
    {
      "epoch": 0.4376930998970134,
      "grad_norm": 235.0,
      "learning_rate": 2.8115345005149327e-07,
      "logits/chosen": -0.24335937201976776,
      "logits/rejected": -0.08320312201976776,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -257.70001220703125,
      "loss": 0.6508,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.21083983778953552,
      "rewards/margins": 0.10366211086511612,
      "rewards/rejected": 0.10673828423023224,
      "step": 1700
    },
    {
      "epoch": 0.4389804325437693,
      "grad_norm": 304.0,
      "learning_rate": 2.8050978372811537e-07,
      "logits/chosen": -0.26787108182907104,
      "logits/rejected": -0.2511230409145355,
      "logps/chosen": -291.79998779296875,
      "logps/rejected": -310.6000061035156,
      "loss": 0.6383,
      "rewards/accuracies": 0.6567857265472412,
      "rewards/chosen": 0.20468750596046448,
      "rewards/margins": 0.13886718451976776,
      "rewards/rejected": 0.06569824367761612,
      "step": 1705
    },
    {
      "epoch": 0.44026776519052524,
      "grad_norm": 155.0,
      "learning_rate": 2.7986611740473736e-07,
      "logits/chosen": -0.21826171875,
      "logits/rejected": -0.2916015684604645,
      "logps/chosen": -298.79998779296875,
      "logps/rejected": -290.3999938964844,
      "loss": 0.6383,
      "rewards/accuracies": 0.5423809289932251,
      "rewards/chosen": 0.2523437440395355,
      "rewards/margins": 0.14215087890625,
      "rewards/rejected": 0.11015625298023224,
      "step": 1710
    },
    {
      "epoch": 0.44155509783728114,
      "grad_norm": 446.0,
      "learning_rate": 2.792224510813594e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": 0.23945312201976776,
      "logps/chosen": -220.10000610351562,
      "logps/rejected": -241.3000030517578,
      "loss": 0.6711,
      "rewards/accuracies": 0.40761905908584595,
      "rewards/chosen": 0.19431152939796448,
      "rewards/margins": 0.07258300483226776,
      "rewards/rejected": 0.12191162258386612,
      "step": 1715
    },
    {
      "epoch": 0.4428424304840371,
      "grad_norm": 100.0,
      "learning_rate": 2.7857878475798144e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.3589843809604645,
      "logps/chosen": -318.79998779296875,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6727,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": 0.23320312798023224,
      "rewards/margins": 0.06900329887866974,
      "rewards/rejected": 0.16435547173023224,
      "step": 1720
    },
    {
      "epoch": 0.444129763130793,
      "grad_norm": 105.5,
      "learning_rate": 2.779351184346035e-07,
      "logits/chosen": -0.271728515625,
      "logits/rejected": -0.38896483182907104,
      "logps/chosen": -273.0,
      "logps/rejected": -245.8000030517578,
      "loss": 0.6703,
      "rewards/accuracies": 0.4699999690055847,
      "rewards/chosen": 0.11757812649011612,
      "rewards/margins": 0.05598144605755806,
      "rewards/rejected": 0.0616455078125,
      "step": 1725
    },
    {
      "epoch": 0.4454170957775489,
      "grad_norm": 109.0,
      "learning_rate": 2.7729145211122553e-07,
      "logits/chosen": -0.3960937559604645,
      "logits/rejected": -0.427734375,
      "logps/chosen": -315.20001220703125,
      "logps/rejected": -294.3999938964844,
      "loss": 0.6578,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.15498046576976776,
      "rewards/margins": 0.08749999850988388,
      "rewards/rejected": 0.0673675537109375,
      "step": 1730
    },
    {
      "epoch": 0.44670442842430486,
      "grad_norm": 139.0,
      "learning_rate": 2.7664778578784757e-07,
      "logits/chosen": -0.2744140625,
      "logits/rejected": -0.26123046875,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -276.79998779296875,
      "loss": 0.65,
      "rewards/accuracies": 0.521593451499939,
      "rewards/chosen": 0.15439453721046448,
      "rewards/margins": 0.109619140625,
      "rewards/rejected": 0.04499511793255806,
      "step": 1735
    },
    {
      "epoch": 0.44799176107106076,
      "grad_norm": 157.0,
      "learning_rate": 2.760041194644696e-07,
      "logits/chosen": -0.3929687440395355,
      "logits/rejected": -0.230010986328125,
      "logps/chosen": -253.0,
      "logps/rejected": -201.0,
      "loss": 0.6484,
      "rewards/accuracies": 0.5433333516120911,
      "rewards/chosen": 0.23056641221046448,
      "rewards/margins": 0.10966797173023224,
      "rewards/rejected": 0.121429443359375,
      "step": 1740
    },
    {
      "epoch": 0.44927909371781666,
      "grad_norm": 155.0,
      "learning_rate": 2.7536045314109166e-07,
      "logits/chosen": -0.3998046815395355,
      "logits/rejected": -0.4287109375,
      "logps/chosen": -275.70001220703125,
      "logps/rejected": -237.0,
      "loss": 0.6516,
      "rewards/accuracies": 0.5483333468437195,
      "rewards/chosen": 0.166748046875,
      "rewards/margins": 0.10966797173023224,
      "rewards/rejected": 0.05732421949505806,
      "step": 1745
    },
    {
      "epoch": 0.4505664263645726,
      "grad_norm": 136.0,
      "learning_rate": 2.747167868177137e-07,
      "logits/chosen": -0.37578123807907104,
      "logits/rejected": -0.3871093690395355,
      "logps/chosen": -296.79998779296875,
      "logps/rejected": -286.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.45500001311302185,
      "rewards/chosen": 0.15229491889476776,
      "rewards/margins": 0.02622070349752903,
      "rewards/rejected": 0.12612304091453552,
      "step": 1750
    },
    {
      "epoch": 0.4518537590113285,
      "grad_norm": 101.5,
      "learning_rate": 2.7407312049433574e-07,
      "logits/chosen": -0.2999511659145355,
      "logits/rejected": -0.3002685606479645,
      "logps/chosen": -329.20001220703125,
      "logps/rejected": -238.0,
      "loss": 0.6531,
      "rewards/accuracies": 0.5451923608779907,
      "rewards/chosen": 0.19853515923023224,
      "rewards/margins": 0.09707031399011612,
      "rewards/rejected": 0.10161133110523224,
      "step": 1755
    },
    {
      "epoch": 0.45314109165808447,
      "grad_norm": 142.0,
      "learning_rate": 2.7342945417095773e-07,
      "logits/chosen": -0.32353514432907104,
      "logits/rejected": -0.3119140565395355,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -239.39999389648438,
      "loss": 0.6773,
      "rewards/accuracies": 0.5116666555404663,
      "rewards/chosen": 0.16999511420726776,
      "rewards/margins": 0.04973907396197319,
      "rewards/rejected": 0.1202392578125,
      "step": 1760
    },
    {
      "epoch": 0.4544284243048404,
      "grad_norm": 152.0,
      "learning_rate": 2.727857878475798e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": -0.43525391817092896,
      "logps/chosen": -283.3999938964844,
      "logps/rejected": -327.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.4466666579246521,
      "rewards/chosen": 0.16884765028953552,
      "rewards/margins": 0.03461914137005806,
      "rewards/rejected": 0.13449707627296448,
      "step": 1765
    },
    {
      "epoch": 0.4557157569515963,
      "grad_norm": 115.5,
      "learning_rate": 2.7214212152420187e-07,
      "logits/chosen": -0.23598632216453552,
      "logits/rejected": -0.20126953721046448,
      "logps/chosen": -318.3999938964844,
      "logps/rejected": -302.0,
      "loss": 0.6594,
      "rewards/accuracies": 0.5858333706855774,
      "rewards/chosen": 0.17158202826976776,
      "rewards/margins": 0.09306640923023224,
      "rewards/rejected": 0.07807616889476776,
      "step": 1770
    },
    {
      "epoch": 0.45700308959835223,
      "grad_norm": 97.0,
      "learning_rate": 2.7149845520082386e-07,
      "logits/chosen": -0.283203125,
      "logits/rejected": -0.4408203065395355,
      "logps/chosen": -323.3999938964844,
      "logps/rejected": -249.60000610351562,
      "loss": 0.6641,
      "rewards/accuracies": 0.569007933139801,
      "rewards/chosen": 0.18300780653953552,
      "rewards/margins": 0.07797851413488388,
      "rewards/rejected": 0.10498046875,
      "step": 1775
    },
    {
      "epoch": 0.45829042224510813,
      "grad_norm": 178.0,
      "learning_rate": 2.708547888774459e-07,
      "logits/chosen": -0.31621092557907104,
      "logits/rejected": -0.4351562559604645,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -270.20001220703125,
      "loss": 0.668,
      "rewards/accuracies": 0.49142855405807495,
      "rewards/chosen": 0.18935546278953552,
      "rewards/margins": 0.06889648735523224,
      "rewards/rejected": 0.12050781399011612,
      "step": 1780
    },
    {
      "epoch": 0.45957775489186403,
      "grad_norm": 100.0,
      "learning_rate": 2.70211122554068e-07,
      "logits/chosen": -0.4546875059604645,
      "logits/rejected": -0.510937511920929,
      "logps/chosen": -258.6000061035156,
      "logps/rejected": -214.8000030517578,
      "loss": 0.6438,
      "rewards/accuracies": 0.5559524297714233,
      "rewards/chosen": 0.18574218451976776,
      "rewards/margins": 0.11293945461511612,
      "rewards/rejected": 0.07285156100988388,
      "step": 1785
    },
    {
      "epoch": 0.46086508753862,
      "grad_norm": 105.0,
      "learning_rate": 2.6956745623069e-07,
      "logits/chosen": -0.25048828125,
      "logits/rejected": -0.3775390684604645,
      "logps/chosen": -265.0,
      "logps/rejected": -251.39999389648438,
      "loss": 0.6781,
      "rewards/accuracies": 0.4979487359523773,
      "rewards/chosen": 0.10126952826976776,
      "rewards/margins": 0.0665283203125,
      "rewards/rejected": 0.03483886644244194,
      "step": 1790
    },
    {
      "epoch": 0.4621524201853759,
      "grad_norm": 133.0,
      "learning_rate": 2.6892378990731203e-07,
      "logits/chosen": -0.3326171934604645,
      "logits/rejected": -0.3226562440395355,
      "logps/chosen": -275.6000061035156,
      "logps/rejected": -234.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.6116666793823242,
      "rewards/chosen": 0.21855469048023224,
      "rewards/margins": 0.10725097358226776,
      "rewards/rejected": 0.11115722358226776,
      "step": 1795
    },
    {
      "epoch": 0.46343975283213185,
      "grad_norm": 344.0,
      "learning_rate": 2.6828012358393407e-07,
      "logits/chosen": -0.3941406309604645,
      "logits/rejected": -0.3086914122104645,
      "logps/chosen": -257.20001220703125,
      "logps/rejected": -283.20001220703125,
      "loss": 0.6375,
      "rewards/accuracies": 0.5575000047683716,
      "rewards/chosen": 0.24150390923023224,
      "rewards/margins": 0.13603515923023224,
      "rewards/rejected": 0.10551758110523224,
      "step": 1800
    },
    {
      "epoch": 0.46472708547888775,
      "grad_norm": 113.0,
      "learning_rate": 2.676364572605561e-07,
      "logits/chosen": -0.22998046875,
      "logits/rejected": -0.2661376893520355,
      "logps/chosen": -216.3000030517578,
      "logps/rejected": -203.39999389648438,
      "loss": 0.657,
      "rewards/accuracies": 0.4816666543483734,
      "rewards/chosen": 0.12294922024011612,
      "rewards/margins": 0.08193359524011612,
      "rewards/rejected": 0.04112548753619194,
      "step": 1805
    },
    {
      "epoch": 0.46601441812564365,
      "grad_norm": 129.0,
      "learning_rate": 2.6699279093717816e-07,
      "logits/chosen": 0.0423736572265625,
      "logits/rejected": -0.06040038913488388,
      "logps/chosen": -213.0,
      "logps/rejected": -224.89999389648438,
      "loss": 0.7063,
      "rewards/accuracies": 0.37261906266212463,
      "rewards/chosen": 0.08193359524011612,
      "rewards/margins": 0.0021820068359375,
      "rewards/rejected": 0.079833984375,
      "step": 1810
    },
    {
      "epoch": 0.4673017507723996,
      "grad_norm": 152.0,
      "learning_rate": 2.663491246138002e-07,
      "logits/chosen": -0.17091064155101776,
      "logits/rejected": -0.25517576932907104,
      "logps/chosen": -225.60000610351562,
      "logps/rejected": -232.39999389648438,
      "loss": 0.6445,
      "rewards/accuracies": 0.621666669845581,
      "rewards/chosen": 0.26054686307907104,
      "rewards/margins": 0.12333984673023224,
      "rewards/rejected": 0.13740234076976776,
      "step": 1815
    },
    {
      "epoch": 0.4685890834191555,
      "grad_norm": 114.5,
      "learning_rate": 2.6570545829042224e-07,
      "logits/chosen": -0.15058593451976776,
      "logits/rejected": -0.2555786073207855,
      "logps/chosen": -266.0,
      "logps/rejected": -285.0,
      "loss": 0.6383,
      "rewards/accuracies": 0.5927563905715942,
      "rewards/chosen": 0.19565430283546448,
      "rewards/margins": 0.11699219048023224,
      "rewards/rejected": 0.07861938327550888,
      "step": 1820
    },
    {
      "epoch": 0.4698764160659114,
      "grad_norm": 100.0,
      "learning_rate": 2.6506179196704423e-07,
      "logits/chosen": -0.37177735567092896,
      "logits/rejected": -0.3824218809604645,
      "logps/chosen": -316.3999938964844,
      "logps/rejected": -279.6000061035156,
      "loss": 0.6453,
      "rewards/accuracies": 0.536474347114563,
      "rewards/chosen": 0.21162109076976776,
      "rewards/margins": 0.1241455078125,
      "rewards/rejected": 0.08763428032398224,
      "step": 1825
    },
    {
      "epoch": 0.47116374871266736,
      "grad_norm": 130.0,
      "learning_rate": 2.6441812564366633e-07,
      "logits/chosen": -0.39472657442092896,
      "logits/rejected": -0.4390625059604645,
      "logps/chosen": -204.39999389648438,
      "logps/rejected": -205.8000030517578,
      "loss": 0.6359,
      "rewards/accuracies": 0.6176190972328186,
      "rewards/chosen": 0.22773437201976776,
      "rewards/margins": 0.13339844346046448,
      "rewards/rejected": 0.09375,
      "step": 1830
    },
    {
      "epoch": 0.47245108135942326,
      "grad_norm": 87.0,
      "learning_rate": 2.6377445932028837e-07,
      "logits/chosen": -0.23872070014476776,
      "logits/rejected": -0.3031249940395355,
      "logps/chosen": -275.6000061035156,
      "logps/rejected": -269.0,
      "loss": 0.6766,
      "rewards/accuracies": 0.46416670083999634,
      "rewards/chosen": 0.15439453721046448,
      "rewards/margins": 0.04631347581744194,
      "rewards/rejected": 0.10806884616613388,
      "step": 1835
    },
    {
      "epoch": 0.4737384140061792,
      "grad_norm": 207.0,
      "learning_rate": 2.6313079299691036e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.25078123807907104,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -221.8000030517578,
      "loss": 0.625,
      "rewards/accuracies": 0.5958333611488342,
      "rewards/chosen": 0.24580077826976776,
      "rewards/margins": 0.16616210341453552,
      "rewards/rejected": 0.07956542819738388,
      "step": 1840
    },
    {
      "epoch": 0.4750257466529351,
      "grad_norm": 181.0,
      "learning_rate": 2.6248712667353246e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": -0.4984374940395355,
      "logps/chosen": -308.3999938964844,
      "logps/rejected": -254.60000610351562,
      "loss": 0.6148,
      "rewards/accuracies": 0.7016667127609253,
      "rewards/chosen": 0.2607421875,
      "rewards/margins": 0.19033202528953552,
      "rewards/rejected": 0.0703125,
      "step": 1845
    },
    {
      "epoch": 0.476313079299691,
      "grad_norm": 134.0,
      "learning_rate": 2.618434603501545e-07,
      "logits/chosen": -0.32929688692092896,
      "logits/rejected": -0.29511719942092896,
      "logps/chosen": -340.79998779296875,
      "logps/rejected": -309.0,
      "loss": 0.6578,
      "rewards/accuracies": 0.59333336353302,
      "rewards/chosen": 0.16249999403953552,
      "rewards/margins": 0.08830566704273224,
      "rewards/rejected": 0.07432861626148224,
      "step": 1850
    },
    {
      "epoch": 0.477600411946447,
      "grad_norm": 158.0,
      "learning_rate": 2.611997940267765e-07,
      "logits/chosen": -0.2874999940395355,
      "logits/rejected": -0.3277343809604645,
      "logps/chosen": -264.3999938964844,
      "logps/rejected": -232.39999389648438,
      "loss": 0.6703,
      "rewards/accuracies": 0.5160606503486633,
      "rewards/chosen": 0.16875000298023224,
      "rewards/margins": 0.08338622748851776,
      "rewards/rejected": 0.08549804985523224,
      "step": 1855
    },
    {
      "epoch": 0.4788877445932029,
      "grad_norm": 132.0,
      "learning_rate": 2.6055612770339853e-07,
      "logits/chosen": -0.19599609076976776,
      "logits/rejected": -0.21835938096046448,
      "logps/chosen": -195.60000610351562,
      "logps/rejected": -225.60000610351562,
      "loss": 0.668,
      "rewards/accuracies": 0.5385714769363403,
      "rewards/chosen": 0.173095703125,
      "rewards/margins": 0.08303222805261612,
      "rewards/rejected": 0.08999023586511612,
      "step": 1860
    },
    {
      "epoch": 0.4801750772399588,
      "grad_norm": 144.0,
      "learning_rate": 2.599124613800206e-07,
      "logits/chosen": -0.38251954317092896,
      "logits/rejected": -0.42851561307907104,
      "logps/chosen": -259.79998779296875,
      "logps/rejected": -242.8000030517578,
      "loss": 0.6875,
      "rewards/accuracies": 0.43166667222976685,
      "rewards/chosen": 0.1334228515625,
      "rewards/margins": 0.03719482570886612,
      "rewards/rejected": 0.09619140625,
      "step": 1865
    },
    {
      "epoch": 0.48146240988671474,
      "grad_norm": 386.0,
      "learning_rate": 2.592687950566426e-07,
      "logits/chosen": -0.3092285096645355,
      "logits/rejected": -0.3931640684604645,
      "logps/chosen": -285.79998779296875,
      "logps/rejected": -254.1999969482422,
      "loss": 0.6094,
      "rewards/accuracies": 0.7716667056083679,
      "rewards/chosen": 0.32441407442092896,
      "rewards/margins": 0.19736328721046448,
      "rewards/rejected": 0.12763671576976776,
      "step": 1870
    },
    {
      "epoch": 0.48274974253347064,
      "grad_norm": 151.0,
      "learning_rate": 2.5862512873326466e-07,
      "logits/chosen": -0.509765625,
      "logits/rejected": -0.4605468809604645,
      "logps/chosen": -373.3999938964844,
      "logps/rejected": -289.0,
      "loss": 0.6422,
      "rewards/accuracies": 0.5447221994400024,
      "rewards/chosen": 0.2523437440395355,
      "rewards/margins": 0.12539061903953552,
      "rewards/rejected": 0.12685546278953552,
      "step": 1875
    },
    {
      "epoch": 0.4840370751802266,
      "grad_norm": 140.0,
      "learning_rate": 2.579814624098867e-07,
      "logits/chosen": -0.2689453065395355,
      "logits/rejected": -0.15937499701976776,
      "logps/chosen": -234.1999969482422,
      "logps/rejected": -210.6999969482422,
      "loss": 0.6687,
      "rewards/accuracies": 0.5372619032859802,
      "rewards/chosen": 0.13193359971046448,
      "rewards/margins": 0.06235351413488388,
      "rewards/rejected": 0.06911621242761612,
      "step": 1880
    },
    {
      "epoch": 0.4853244078269825,
      "grad_norm": 138.0,
      "learning_rate": 2.5733779608650874e-07,
      "logits/chosen": -0.3341308534145355,
      "logits/rejected": -0.33710938692092896,
      "logps/chosen": -365.3999938964844,
      "logps/rejected": -297.6000061035156,
      "loss": 0.6398,
      "rewards/accuracies": 0.5114285945892334,
      "rewards/chosen": 0.1922607421875,
      "rewards/margins": 0.1439208984375,
      "rewards/rejected": 0.04863281175494194,
      "step": 1885
    },
    {
      "epoch": 0.4866117404737384,
      "grad_norm": 118.5,
      "learning_rate": 2.566941297631308e-07,
      "logits/chosen": -0.322265625,
      "logits/rejected": -0.3628906309604645,
      "logps/chosen": -245.0,
      "logps/rejected": -234.1999969482422,
      "loss": 0.6438,
      "rewards/accuracies": 0.6560714840888977,
      "rewards/chosen": 0.28828126192092896,
      "rewards/margins": 0.12509766221046448,
      "rewards/rejected": 0.16357421875,
      "step": 1890
    },
    {
      "epoch": 0.48789907312049435,
      "grad_norm": 117.0,
      "learning_rate": 2.5605046343975283e-07,
      "logits/chosen": -0.13046875596046448,
      "logits/rejected": -0.41777342557907104,
      "logps/chosen": -177.10000610351562,
      "logps/rejected": -170.5,
      "loss": 0.668,
      "rewards/accuracies": 0.508461594581604,
      "rewards/chosen": 0.1175537109375,
      "rewards/margins": 0.06672362983226776,
      "rewards/rejected": 0.05089111253619194,
      "step": 1895
    },
    {
      "epoch": 0.48918640576725025,
      "grad_norm": 139.0,
      "learning_rate": 2.5540679711637487e-07,
      "logits/chosen": -0.4613281190395355,
      "logits/rejected": -0.44804686307907104,
      "logps/chosen": -279.20001220703125,
      "logps/rejected": -233.39999389648438,
      "loss": 0.6633,
      "rewards/accuracies": 0.4856410622596741,
      "rewards/chosen": 0.20607909560203552,
      "rewards/margins": 0.07763671875,
      "rewards/rejected": 0.12836913764476776,
      "step": 1900
    },
    {
      "epoch": 0.49047373841400616,
      "grad_norm": 131.0,
      "learning_rate": 2.5476313079299686e-07,
      "logits/chosen": -0.188720703125,
      "logits/rejected": -0.22031250596046448,
      "logps/chosen": -281.79998779296875,
      "logps/rejected": -267.79998779296875,
      "loss": 0.6734,
      "rewards/accuracies": 0.5383332967758179,
      "rewards/chosen": 0.19453124701976776,
      "rewards/margins": 0.06608887016773224,
      "rewards/rejected": 0.12838134169578552,
      "step": 1905
    },
    {
      "epoch": 0.4917610710607621,
      "grad_norm": 272.0,
      "learning_rate": 2.5411946446961896e-07,
      "logits/chosen": -0.4166015684604645,
      "logits/rejected": -0.35126954317092896,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6242,
      "rewards/accuracies": 0.60833340883255,
      "rewards/chosen": 0.24990233778953552,
      "rewards/margins": 0.16191406548023224,
      "rewards/rejected": 0.088134765625,
      "step": 1910
    },
    {
      "epoch": 0.493048403707518,
      "grad_norm": 111.5,
      "learning_rate": 2.53475798146241e-07,
      "logits/chosen": -0.29082030057907104,
      "logits/rejected": -0.3388671875,
      "logps/chosen": -246.39999389648438,
      "logps/rejected": -261.3999938964844,
      "loss": 0.65,
      "rewards/accuracies": 0.6253571510314941,
      "rewards/chosen": 0.23906250298023224,
      "rewards/margins": 0.1131591796875,
      "rewards/rejected": 0.12559814751148224,
      "step": 1915
    },
    {
      "epoch": 0.49433573635427397,
      "grad_norm": 113.0,
      "learning_rate": 2.52832131822863e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.43408203125,
      "logps/chosen": -314.0,
      "logps/rejected": -272.6000061035156,
      "loss": 0.6531,
      "rewards/accuracies": 0.4833333492279053,
      "rewards/chosen": 0.19384765625,
      "rewards/margins": 0.09565429389476776,
      "rewards/rejected": 0.09833984076976776,
      "step": 1920
    },
    {
      "epoch": 0.49562306900102987,
      "grad_norm": 108.0,
      "learning_rate": 2.521884654994851e-07,
      "logits/chosen": -0.3083740174770355,
      "logits/rejected": -0.41093748807907104,
      "logps/chosen": -299.3999938964844,
      "logps/rejected": -257.6000061035156,
      "loss": 0.6633,
      "rewards/accuracies": 0.4991758465766907,
      "rewards/chosen": 0.16269531846046448,
      "rewards/margins": 0.07880859076976776,
      "rewards/rejected": 0.08406829833984375,
      "step": 1925
    },
    {
      "epoch": 0.49691040164778577,
      "grad_norm": 94.0,
      "learning_rate": 2.5154479917610713e-07,
      "logits/chosen": -0.30146485567092896,
      "logits/rejected": -0.41688233613967896,
      "logps/chosen": -268.0,
      "logps/rejected": -246.0,
      "loss": 0.6766,
      "rewards/accuracies": 0.4484523832798004,
      "rewards/chosen": 0.14243164658546448,
      "rewards/margins": 0.04033203050494194,
      "rewards/rejected": 0.10164184868335724,
      "step": 1930
    },
    {
      "epoch": 0.4981977342945417,
      "grad_norm": 115.5,
      "learning_rate": 2.509011328527291e-07,
      "logits/chosen": -0.37519532442092896,
      "logits/rejected": -0.2995361387729645,
      "logps/chosen": -270.0,
      "logps/rejected": -197.0,
      "loss": 0.6453,
      "rewards/accuracies": 0.5858333706855774,
      "rewards/chosen": 0.16015625,
      "rewards/margins": 0.10932616889476776,
      "rewards/rejected": 0.05095519870519638,
      "step": 1935
    },
    {
      "epoch": 0.49948506694129763,
      "grad_norm": 99.5,
      "learning_rate": 2.5025746652935116e-07,
      "logits/chosen": -0.28242188692092896,
      "logits/rejected": -0.264404296875,
      "logps/chosen": -276.3999938964844,
      "logps/rejected": -260.3999938964844,
      "loss": 0.6484,
      "rewards/accuracies": 0.5585256814956665,
      "rewards/chosen": 0.18300780653953552,
      "rewards/margins": 0.11542968451976776,
      "rewards/rejected": 0.06756591796875,
      "step": 1940
    },
    {
      "epoch": 0.5007723995880535,
      "grad_norm": 140.0,
      "learning_rate": 2.496138002059732e-07,
      "logits/chosen": -0.3978515565395355,
      "logits/rejected": -0.2403564453125,
      "logps/chosen": -251.39999389648438,
      "logps/rejected": -206.39999389648438,
      "loss": 0.6625,
      "rewards/accuracies": 0.49741506576538086,
      "rewards/chosen": 0.14785155653953552,
      "rewards/margins": 0.06694336235523224,
      "rewards/rejected": 0.08092041313648224,
      "step": 1945
    },
    {
      "epoch": 0.5020597322348095,
      "grad_norm": 134.0,
      "learning_rate": 2.4897013388259525e-07,
      "logits/chosen": -0.24843749403953552,
      "logits/rejected": -0.2699218690395355,
      "logps/chosen": -314.0,
      "logps/rejected": -276.6000061035156,
      "loss": 0.6602,
      "rewards/accuracies": 0.5541666746139526,
      "rewards/chosen": 0.18386229872703552,
      "rewards/margins": 0.08610840141773224,
      "rewards/rejected": 0.09785155951976776,
      "step": 1950
    },
    {
      "epoch": 0.5033470648815654,
      "grad_norm": 130.0,
      "learning_rate": 2.483264675592173e-07,
      "logits/chosen": -0.27421873807907104,
      "logits/rejected": -0.24335937201976776,
      "logps/chosen": -249.0,
      "logps/rejected": -230.39999389648438,
      "loss": 0.6516,
      "rewards/accuracies": 0.5450000166893005,
      "rewards/chosen": 0.1566162109375,
      "rewards/margins": 0.0947265625,
      "rewards/rejected": 0.06160888820886612,
      "step": 1955
    },
    {
      "epoch": 0.5046343975283213,
      "grad_norm": 100.5,
      "learning_rate": 2.4768280123583933e-07,
      "logits/chosen": -0.3726562559604645,
      "logits/rejected": -0.39140623807907104,
      "logps/chosen": -287.0,
      "logps/rejected": -293.3999938964844,
      "loss": 0.6641,
      "rewards/accuracies": 0.548214316368103,
      "rewards/chosen": 0.22812500596046448,
      "rewards/margins": 0.09470214694738388,
      "rewards/rejected": 0.13359375298023224,
      "step": 1960
    },
    {
      "epoch": 0.5059217301750772,
      "grad_norm": 148.0,
      "learning_rate": 2.470391349124614e-07,
      "logits/chosen": -0.3443359434604645,
      "logits/rejected": -0.29072266817092896,
      "logps/chosen": -278.6000061035156,
      "logps/rejected": -246.60000610351562,
      "loss": 0.6914,
      "rewards/accuracies": 0.3161905109882355,
      "rewards/chosen": 0.094482421875,
      "rewards/margins": 0.01987304724752903,
      "rewards/rejected": 0.07441405951976776,
      "step": 1965
    },
    {
      "epoch": 0.5072090628218332,
      "grad_norm": 124.5,
      "learning_rate": 2.463954685890834e-07,
      "logits/chosen": -0.33088380098342896,
      "logits/rejected": -0.3642578125,
      "logps/chosen": -406.3999938964844,
      "logps/rejected": -346.6000061035156,
      "loss": 0.65,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.17207030951976776,
      "rewards/margins": 0.10224609076976776,
      "rewards/rejected": 0.06972656399011612,
      "step": 1970
    },
    {
      "epoch": 0.508496395468589,
      "grad_norm": 112.5,
      "learning_rate": 2.4575180226570546e-07,
      "logits/chosen": -0.30498045682907104,
      "logits/rejected": -0.290771484375,
      "logps/chosen": -282.79998779296875,
      "logps/rejected": -280.0,
      "loss": 0.6664,
      "rewards/accuracies": 0.5300000309944153,
      "rewards/chosen": 0.16953125596046448,
      "rewards/margins": 0.083984375,
      "rewards/rejected": 0.08576659858226776,
      "step": 1975
    },
    {
      "epoch": 0.509783728115345,
      "grad_norm": 177.0,
      "learning_rate": 2.451081359423275e-07,
      "logits/chosen": -0.3236328065395355,
      "logits/rejected": -0.32646483182907104,
      "logps/chosen": -246.60000610351562,
      "logps/rejected": -285.79998779296875,
      "loss": 0.6484,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.24160155653953552,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": 0.12841796875,
      "step": 1980
    },
    {
      "epoch": 0.511071060762101,
      "grad_norm": 134.0,
      "learning_rate": 2.4446446961894955e-07,
      "logits/chosen": -0.3726562559604645,
      "logits/rejected": -0.361328125,
      "logps/chosen": -256.20001220703125,
      "logps/rejected": -267.6000061035156,
      "loss": 0.668,
      "rewards/accuracies": 0.5280952453613281,
      "rewards/chosen": 0.17587891221046448,
      "rewards/margins": 0.06298828125,
      "rewards/rejected": 0.11285400390625,
      "step": 1985
    },
    {
      "epoch": 0.5123583934088568,
      "grad_norm": 336.0,
      "learning_rate": 2.4382080329557153e-07,
      "logits/chosen": -0.2933593690395355,
      "logits/rejected": -0.123046875,
      "logps/chosen": -282.6000061035156,
      "logps/rejected": -231.39999389648438,
      "loss": 0.6242,
      "rewards/accuracies": 0.6329761743545532,
      "rewards/chosen": 0.25153809785842896,
      "rewards/margins": 0.165802001953125,
      "rewards/rejected": 0.08530273288488388,
      "step": 1990
    },
    {
      "epoch": 0.5136457260556128,
      "grad_norm": 282.0,
      "learning_rate": 2.4317713697219363e-07,
      "logits/chosen": -0.36933594942092896,
      "logits/rejected": -0.39570313692092896,
      "logps/chosen": -238.5,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6867,
      "rewards/accuracies": 0.4633333683013916,
      "rewards/chosen": 0.19863280653953552,
      "rewards/margins": 0.02159423753619194,
      "rewards/rejected": 0.17724609375,
      "step": 1995
    },
    {
      "epoch": 0.5149330587023687,
      "grad_norm": 156.0,
      "learning_rate": 2.425334706488156e-07,
      "logits/chosen": -0.5054687261581421,
      "logits/rejected": -0.4007812440395355,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -215.39999389648438,
      "loss": 0.6492,
      "rewards/accuracies": 0.5738734006881714,
      "rewards/chosen": 0.17919921875,
      "rewards/margins": 0.11479492485523224,
      "rewards/rejected": 0.064483642578125,
      "step": 2000
    },
    {
      "epoch": 0.5162203913491246,
      "grad_norm": 82.5,
      "learning_rate": 2.4188980432543766e-07,
      "logits/chosen": -0.37519532442092896,
      "logits/rejected": -0.2724609375,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -239.0,
      "loss": 0.6484,
      "rewards/accuracies": 0.5483333468437195,
      "rewards/chosen": 0.22138671576976776,
      "rewards/margins": 0.10869140923023224,
      "rewards/rejected": 0.11279296875,
      "step": 2005
    },
    {
      "epoch": 0.5175077239958805,
      "grad_norm": 108.5,
      "learning_rate": 2.412461380020597e-07,
      "logits/chosen": -0.38526612520217896,
      "logits/rejected": -0.3421386778354645,
      "logps/chosen": -315.6000061035156,
      "logps/rejected": -262.6000061035156,
      "loss": 0.6641,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.20014648139476776,
      "rewards/margins": 0.08784179389476776,
      "rewards/rejected": 0.11220703274011612,
      "step": 2010
    },
    {
      "epoch": 0.5187950566426365,
      "grad_norm": 132.0,
      "learning_rate": 2.4060247167868175e-07,
      "logits/chosen": -0.2392578125,
      "logits/rejected": -0.22756652534008026,
      "logps/chosen": -247.1999969482422,
      "logps/rejected": -241.89999389648438,
      "loss": 0.6891,
      "rewards/accuracies": 0.432023823261261,
      "rewards/chosen": 0.0806884765625,
      "rewards/margins": 0.01349487341940403,
      "rewards/rejected": 0.06734619289636612,
      "step": 2015
    },
    {
      "epoch": 0.5200823892893924,
      "grad_norm": 260.0,
      "learning_rate": 2.399588053553038e-07,
      "logits/chosen": -0.16396483778953552,
      "logits/rejected": -0.16367188096046448,
      "logps/chosen": -249.0,
      "logps/rejected": -215.60000610351562,
      "loss": 0.6547,
      "rewards/accuracies": 0.5797008872032166,
      "rewards/chosen": 0.16997070610523224,
      "rewards/margins": 0.09947510063648224,
      "rewards/rejected": 0.07070312649011612,
      "step": 2020
    },
    {
      "epoch": 0.5213697219361483,
      "grad_norm": 193.0,
      "learning_rate": 2.3931513903192583e-07,
      "logits/chosen": -0.43476563692092896,
      "logits/rejected": -0.3169921934604645,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -230.39999389648438,
      "loss": 0.6516,
      "rewards/accuracies": 0.5964285731315613,
      "rewards/chosen": 0.2552734315395355,
      "rewards/margins": 0.11126098781824112,
      "rewards/rejected": 0.14404296875,
      "step": 2025
    },
    {
      "epoch": 0.5226570545829042,
      "grad_norm": 390.0,
      "learning_rate": 2.386714727085479e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.35039061307907104,
      "logps/chosen": -319.0,
      "logps/rejected": -279.3999938964844,
      "loss": 0.6484,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.22158202528953552,
      "rewards/margins": 0.11274413764476776,
      "rewards/rejected": 0.10868529975414276,
      "step": 2030
    },
    {
      "epoch": 0.5239443872296602,
      "grad_norm": 102.5,
      "learning_rate": 2.3802780638516992e-07,
      "logits/chosen": -0.2554687559604645,
      "logits/rejected": -0.2669921815395355,
      "logps/chosen": -214.60000610351562,
      "logps/rejected": -183.3000030517578,
      "loss": 0.6742,
      "rewards/accuracies": 0.5200000405311584,
      "rewards/chosen": 0.12215576320886612,
      "rewards/margins": 0.06264648586511612,
      "rewards/rejected": 0.0595703125,
      "step": 2035
    },
    {
      "epoch": 0.525231719876416,
      "grad_norm": 126.0,
      "learning_rate": 2.3738414006179194e-07,
      "logits/chosen": -0.26053160429000854,
      "logits/rejected": -0.257080078125,
      "logps/chosen": -255.8000030517578,
      "logps/rejected": -260.20001220703125,
      "loss": 0.6711,
      "rewards/accuracies": 0.5375458002090454,
      "rewards/chosen": 0.25361329317092896,
      "rewards/margins": 0.07978515326976776,
      "rewards/rejected": 0.17412109673023224,
      "step": 2040
    },
    {
      "epoch": 0.526519052523172,
      "grad_norm": 130.0,
      "learning_rate": 2.36740473738414e-07,
      "logits/chosen": -0.4124999940395355,
      "logits/rejected": -0.4027343690395355,
      "logps/chosen": -313.3999938964844,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6273,
      "rewards/accuracies": 0.6266666650772095,
      "rewards/chosen": 0.2818359434604645,
      "rewards/margins": 0.16464844346046448,
      "rewards/rejected": 0.11733398586511612,
      "step": 2045
    },
    {
      "epoch": 0.527806385169928,
      "grad_norm": 266.0,
      "learning_rate": 2.3609680741503605e-07,
      "logits/chosen": -0.3407226502895355,
      "logits/rejected": -0.34003907442092896,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -257.0,
      "loss": 0.6562,
      "rewards/accuracies": 0.5714177489280701,
      "rewards/chosen": 0.16201171278953552,
      "rewards/margins": 0.09733887016773224,
      "rewards/rejected": 0.06474609673023224,
      "step": 2050
    },
    {
      "epoch": 0.5290937178166838,
      "grad_norm": 101.0,
      "learning_rate": 2.3545314109165806e-07,
      "logits/chosen": -0.35371094942092896,
      "logits/rejected": -0.29838865995407104,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -230.8000030517578,
      "loss": 0.668,
      "rewards/accuracies": 0.5433333516120911,
      "rewards/chosen": 0.143890380859375,
      "rewards/margins": 0.07082519680261612,
      "rewards/rejected": 0.07307128608226776,
      "step": 2055
    },
    {
      "epoch": 0.5303810504634398,
      "grad_norm": 129.0,
      "learning_rate": 2.348094747682801e-07,
      "logits/chosen": -0.23261718451976776,
      "logits/rejected": -0.40117186307907104,
      "logps/chosen": -256.6000061035156,
      "logps/rejected": -236.8000030517578,
      "loss": 0.6297,
      "rewards/accuracies": 0.6315476298332214,
      "rewards/chosen": 0.21220703423023224,
      "rewards/margins": 0.14887695014476776,
      "rewards/rejected": 0.06303711235523224,
      "step": 2060
    },
    {
      "epoch": 0.5316683831101957,
      "grad_norm": 262.0,
      "learning_rate": 2.3416580844490218e-07,
      "logits/chosen": -0.29169923067092896,
      "logits/rejected": -0.34882813692092896,
      "logps/chosen": -279.3999938964844,
      "logps/rejected": -292.20001220703125,
      "loss": 0.6555,
      "rewards/accuracies": 0.5508333444595337,
      "rewards/chosen": 0.22275391221046448,
      "rewards/margins": 0.08763428032398224,
      "rewards/rejected": 0.13547363877296448,
      "step": 2065
    },
    {
      "epoch": 0.5329557157569516,
      "grad_norm": 476.0,
      "learning_rate": 2.335221421215242e-07,
      "logits/chosen": -0.3271484375,
      "logits/rejected": -0.3492187559604645,
      "logps/chosen": -228.60000610351562,
      "logps/rejected": -230.1999969482422,
      "loss": 0.6734,
      "rewards/accuracies": 0.48719701170921326,
      "rewards/chosen": 0.21198424696922302,
      "rewards/margins": 0.06914062798023224,
      "rewards/rejected": 0.14301757514476776,
      "step": 2070
    },
    {
      "epoch": 0.5342430484037075,
      "grad_norm": 392.0,
      "learning_rate": 2.3287847579814623e-07,
      "logits/chosen": -0.44921875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -251.8000030517578,
      "logps/rejected": -226.10000610351562,
      "loss": 0.6719,
      "rewards/accuracies": 0.5883333683013916,
      "rewards/chosen": 0.15327148139476776,
      "rewards/margins": 0.06586913764476776,
      "rewards/rejected": 0.08729247748851776,
      "step": 2075
    },
    {
      "epoch": 0.5355303810504635,
      "grad_norm": 92.5,
      "learning_rate": 2.3223480947476825e-07,
      "logits/chosen": -0.3805175721645355,
      "logits/rejected": -0.27910155057907104,
      "logps/chosen": -327.20001220703125,
      "logps/rejected": -265.0,
      "loss": 0.6422,
      "rewards/accuracies": 0.5543955564498901,
      "rewards/chosen": 0.2529296875,
      "rewards/margins": 0.113037109375,
      "rewards/rejected": 0.13996581733226776,
      "step": 2080
    },
    {
      "epoch": 0.5368177136972193,
      "grad_norm": 151.0,
      "learning_rate": 2.315911431513903e-07,
      "logits/chosen": -0.32465821504592896,
      "logits/rejected": -0.21845702826976776,
      "logps/chosen": -286.79998779296875,
      "logps/rejected": -255.8000030517578,
      "loss": 0.6508,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": 0.22890624403953552,
      "rewards/margins": 0.09967041015625,
      "rewards/rejected": 0.12934570014476776,
      "step": 2085
    },
    {
      "epoch": 0.5381050463439753,
      "grad_norm": 113.5,
      "learning_rate": 2.3094747682801236e-07,
      "logits/chosen": -0.3394531309604645,
      "logits/rejected": -0.4253906309604645,
      "logps/chosen": -258.0,
      "logps/rejected": -236.8000030517578,
      "loss": 0.6594,
      "rewards/accuracies": 0.5279329419136047,
      "rewards/chosen": 0.18276366591453552,
      "rewards/margins": 0.07656250149011612,
      "rewards/rejected": 0.10629882663488388,
      "step": 2090
    },
    {
      "epoch": 0.5393923789907312,
      "grad_norm": 159.0,
      "learning_rate": 2.3030381050463438e-07,
      "logits/chosen": -0.32734376192092896,
      "logits/rejected": -0.3091796934604645,
      "logps/chosen": -250.8000030517578,
      "logps/rejected": -260.20001220703125,
      "loss": 0.6492,
      "rewards/accuracies": 0.5425000190734863,
      "rewards/chosen": 0.23427733778953552,
      "rewards/margins": 0.112060546875,
      "rewards/rejected": 0.12177734076976776,
      "step": 2095
    },
    {
      "epoch": 0.5406797116374872,
      "grad_norm": 126.0,
      "learning_rate": 2.2966014418125642e-07,
      "logits/chosen": -0.35761719942092896,
      "logits/rejected": -0.35820311307907104,
      "logps/chosen": -281.3999938964844,
      "logps/rejected": -261.6000061035156,
      "loss": 0.675,
      "rewards/accuracies": 0.41805195808410645,
      "rewards/chosen": 0.10483398288488388,
      "rewards/margins": 0.05225830152630806,
      "rewards/rejected": 0.052490234375,
      "step": 2100
    },
    {
      "epoch": 0.541967044284243,
      "grad_norm": 128.0,
      "learning_rate": 2.2901647785787846e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.46484375,
      "logps/chosen": -309.6000061035156,
      "logps/rejected": -301.20001220703125,
      "loss": 0.6617,
      "rewards/accuracies": 0.5649999380111694,
      "rewards/chosen": 0.2783203125,
      "rewards/margins": 0.08417968451976776,
      "rewards/rejected": 0.19423827528953552,
      "step": 2105
    },
    {
      "epoch": 0.543254376930999,
      "grad_norm": 113.5,
      "learning_rate": 2.283728115345005e-07,
      "logits/chosen": -0.22956542670726776,
      "logits/rejected": -0.31787109375,
      "logps/chosen": -299.0,
      "logps/rejected": -236.39999389648438,
      "loss": 0.6656,
      "rewards/accuracies": 0.47698718309402466,
      "rewards/chosen": 0.19145508110523224,
      "rewards/margins": 0.07319335639476776,
      "rewards/rejected": 0.118408203125,
      "step": 2110
    },
    {
      "epoch": 0.5445417095777549,
      "grad_norm": 130.0,
      "learning_rate": 2.2772914521112255e-07,
      "logits/chosen": -0.29222410917282104,
      "logits/rejected": -0.23403319716453552,
      "logps/chosen": -302.6000061035156,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6672,
      "rewards/accuracies": 0.4888095259666443,
      "rewards/chosen": 0.19077149033546448,
      "rewards/margins": 0.07509765774011612,
      "rewards/rejected": 0.11572265625,
      "step": 2115
    },
    {
      "epoch": 0.5458290422245108,
      "grad_norm": 141.0,
      "learning_rate": 2.2708547888774457e-07,
      "logits/chosen": -0.44609373807907104,
      "logits/rejected": -0.4105468690395355,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -235.60000610351562,
      "loss": 0.6531,
      "rewards/accuracies": 0.5441666841506958,
      "rewards/chosen": 0.17592772841453552,
      "rewards/margins": 0.10205078125,
      "rewards/rejected": 0.07392577826976776,
      "step": 2120
    },
    {
      "epoch": 0.5471163748712667,
      "grad_norm": 420.0,
      "learning_rate": 2.264418125643666e-07,
      "logits/chosen": -0.37739259004592896,
      "logits/rejected": -0.3650146424770355,
      "logps/chosen": -312.0,
      "logps/rejected": -301.20001220703125,
      "loss": 0.6195,
      "rewards/accuracies": 0.6371861696243286,
      "rewards/chosen": 0.25605469942092896,
      "rewards/margins": 0.18427734076976776,
      "rewards/rejected": 0.07178954780101776,
      "step": 2125
    },
    {
      "epoch": 0.5484037075180227,
      "grad_norm": 154.0,
      "learning_rate": 2.2579814624098868e-07,
      "logits/chosen": -0.3308349549770355,
      "logits/rejected": -0.3148437440395355,
      "logps/chosen": -330.79998779296875,
      "logps/rejected": -261.79998779296875,
      "loss": 0.6305,
      "rewards/accuracies": 0.6208333373069763,
      "rewards/chosen": 0.17636719346046448,
      "rewards/margins": 0.14716796576976776,
      "rewards/rejected": 0.02913818322122097,
      "step": 2130
    },
    {
      "epoch": 0.5496910401647785,
      "grad_norm": 124.5,
      "learning_rate": 2.251544799176107e-07,
      "logits/chosen": -0.33247071504592896,
      "logits/rejected": -0.25004881620407104,
      "logps/chosen": -282.20001220703125,
      "logps/rejected": -263.6000061035156,
      "loss": 0.6789,
      "rewards/accuracies": 0.5808333158493042,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 0.06516113132238388,
      "rewards/rejected": 0.18193359673023224,
      "step": 2135
    },
    {
      "epoch": 0.5509783728115345,
      "grad_norm": 130.0,
      "learning_rate": 2.2451081359423274e-07,
      "logits/chosen": -0.26264649629592896,
      "logits/rejected": -0.30632323026657104,
      "logps/chosen": -252.39999389648438,
      "logps/rejected": -290.0,
      "loss": 0.6289,
      "rewards/accuracies": 0.6266667246818542,
      "rewards/chosen": 0.212890625,
      "rewards/margins": 0.14716796576976776,
      "rewards/rejected": 0.06572265923023224,
      "step": 2140
    },
    {
      "epoch": 0.5522657054582905,
      "grad_norm": 134.0,
      "learning_rate": 2.2386714727085478e-07,
      "logits/chosen": -0.3389648497104645,
      "logits/rejected": -0.322265625,
      "logps/chosen": -320.0,
      "logps/rejected": -242.60000610351562,
      "loss": 0.6562,
      "rewards/accuracies": 0.5464285612106323,
      "rewards/chosen": 0.20224609971046448,
      "rewards/margins": 0.08818359673023224,
      "rewards/rejected": 0.11435546725988388,
      "step": 2145
    },
    {
      "epoch": 0.5535530381050463,
      "grad_norm": 171.0,
      "learning_rate": 2.2322348094747682e-07,
      "logits/chosen": -0.2759765684604645,
      "logits/rejected": -0.2632812559604645,
      "logps/chosen": -271.20001220703125,
      "logps/rejected": -283.79998779296875,
      "loss": 0.6352,
      "rewards/accuracies": 0.6091667413711548,
      "rewards/chosen": 0.24062499403953552,
      "rewards/margins": 0.13363036513328552,
      "rewards/rejected": 0.10708007961511612,
      "step": 2150
    },
    {
      "epoch": 0.5548403707518023,
      "grad_norm": 126.0,
      "learning_rate": 2.2257981462409886e-07,
      "logits/chosen": -0.32294923067092896,
      "logits/rejected": -0.3203125,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -254.39999389648438,
      "loss": 0.6297,
      "rewards/accuracies": 0.6818939447402954,
      "rewards/chosen": 0.2606445252895355,
      "rewards/margins": 0.1436767578125,
      "rewards/rejected": 0.11733398586511612,
      "step": 2155
    },
    {
      "epoch": 0.5561277033985582,
      "grad_norm": 151.0,
      "learning_rate": 2.219361483007209e-07,
      "logits/chosen": -0.25927734375,
      "logits/rejected": -0.23779296875,
      "logps/chosen": -283.20001220703125,
      "logps/rejected": -277.6000061035156,
      "loss": 0.6531,
      "rewards/accuracies": 0.5266667008399963,
      "rewards/chosen": 0.2800537049770355,
      "rewards/margins": 0.10877685248851776,
      "rewards/rejected": 0.17158202826976776,
      "step": 2160
    },
    {
      "epoch": 0.5574150360453141,
      "grad_norm": 152.0,
      "learning_rate": 2.2129248197734292e-07,
      "logits/chosen": -0.39921873807907104,
      "logits/rejected": -0.47187501192092896,
      "logps/chosen": -348.3999938964844,
      "logps/rejected": -266.3999938964844,
      "loss": 0.6578,
      "rewards/accuracies": 0.5066666603088379,
      "rewards/chosen": 0.21230468153953552,
      "rewards/margins": 0.10141601413488388,
      "rewards/rejected": 0.11086425930261612,
      "step": 2165
    },
    {
      "epoch": 0.55870236869207,
      "grad_norm": 127.0,
      "learning_rate": 2.20648815653965e-07,
      "logits/chosen": -0.3265624940395355,
      "logits/rejected": -0.2901855409145355,
      "logps/chosen": -264.0,
      "logps/rejected": -228.60000610351562,
      "loss": 0.6469,
      "rewards/accuracies": 0.5839285850524902,
      "rewards/chosen": 0.214111328125,
      "rewards/margins": 0.10341797024011612,
      "rewards/rejected": 0.11081542819738388,
      "step": 2170
    },
    {
      "epoch": 0.559989701338826,
      "grad_norm": 446.0,
      "learning_rate": 2.20005149330587e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.4349609315395355,
      "logps/chosen": -247.60000610351562,
      "logps/rejected": -211.1999969482422,
      "loss": 0.6516,
      "rewards/accuracies": 0.6025000214576721,
      "rewards/chosen": 0.14882507920265198,
      "rewards/margins": 0.09748534858226776,
      "rewards/rejected": 0.0516357421875,
      "step": 2175
    },
    {
      "epoch": 0.5612770339855818,
      "grad_norm": 112.5,
      "learning_rate": 2.1936148300720905e-07,
      "logits/chosen": -0.4632812440395355,
      "logits/rejected": -0.4457031190395355,
      "logps/chosen": -266.6000061035156,
      "logps/rejected": -256.0,
      "loss": 0.6344,
      "rewards/accuracies": 0.6876190900802612,
      "rewards/chosen": 0.24755859375,
      "rewards/margins": 0.14462891221046448,
      "rewards/rejected": 0.10277099907398224,
      "step": 2180
    },
    {
      "epoch": 0.5625643666323378,
      "grad_norm": 111.0,
      "learning_rate": 2.187178166838311e-07,
      "logits/chosen": -0.35566407442092896,
      "logits/rejected": -0.48906248807907104,
      "logps/chosen": -251.60000610351562,
      "logps/rejected": -286.20001220703125,
      "loss": 0.6336,
      "rewards/accuracies": 0.6476281881332397,
      "rewards/chosen": 0.21826171875,
      "rewards/margins": 0.14111328125,
      "rewards/rejected": 0.07729492336511612,
      "step": 2185
    },
    {
      "epoch": 0.5638516992790937,
      "grad_norm": 189.0,
      "learning_rate": 2.1807415036045314e-07,
      "logits/chosen": -0.31858521699905396,
      "logits/rejected": -0.2621093690395355,
      "logps/chosen": -283.6000061035156,
      "logps/rejected": -263.20001220703125,
      "loss": 0.6539,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.24941405653953552,
      "rewards/margins": 0.10574951022863388,
      "rewards/rejected": 0.14399413764476776,
      "step": 2190
    },
    {
      "epoch": 0.5651390319258497,
      "grad_norm": 102.5,
      "learning_rate": 2.1743048403707518e-07,
      "logits/chosen": -0.22007140517234802,
      "logits/rejected": -0.38749998807907104,
      "logps/chosen": -261.6000061035156,
      "logps/rejected": -234.60000610351562,
      "loss": 0.65,
      "rewards/accuracies": 0.5569444894790649,
      "rewards/chosen": 0.17050781846046448,
      "rewards/margins": 0.09951172024011612,
      "rewards/rejected": 0.07106933742761612,
      "step": 2195
    },
    {
      "epoch": 0.5664263645726055,
      "grad_norm": 768.0,
      "learning_rate": 2.1678681771369722e-07,
      "logits/chosen": -0.4173828065395355,
      "logits/rejected": -0.306640625,
      "logps/chosen": -272.79998779296875,
      "logps/rejected": -266.0,
      "loss": 0.6453,
      "rewards/accuracies": 0.5850000381469727,
      "rewards/chosen": 0.23222656548023224,
      "rewards/margins": 0.11967773735523224,
      "rewards/rejected": 0.11259765923023224,
      "step": 2200
    },
    {
      "epoch": 0.5677136972193615,
      "grad_norm": 107.5,
      "learning_rate": 2.1614315139031924e-07,
      "logits/chosen": -0.4527343809604645,
      "logits/rejected": -0.4154296815395355,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -307.3999938964844,
      "loss": 0.6797,
      "rewards/accuracies": 0.5745238065719604,
      "rewards/chosen": 0.086669921875,
      "rewards/margins": 0.04450683668255806,
      "rewards/rejected": 0.04184570163488388,
      "step": 2205
    },
    {
      "epoch": 0.5690010298661174,
      "grad_norm": 122.0,
      "learning_rate": 2.154994850669413e-07,
      "logits/chosen": -0.30732423067092896,
      "logits/rejected": -0.29133301973342896,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -228.8000030517578,
      "loss": 0.6508,
      "rewards/accuracies": 0.5410256385803223,
      "rewards/chosen": 0.19951172173023224,
      "rewards/margins": 0.1123046875,
      "rewards/rejected": 0.08769531548023224,
      "step": 2210
    },
    {
      "epoch": 0.5702883625128733,
      "grad_norm": 135.0,
      "learning_rate": 2.1485581874356332e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.35546875,
      "logps/chosen": -258.20001220703125,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6477,
      "rewards/accuracies": 0.5992857217788696,
      "rewards/chosen": 0.24272461235523224,
      "rewards/margins": 0.11103515326976776,
      "rewards/rejected": 0.13148193061351776,
      "step": 2215
    },
    {
      "epoch": 0.5715756951596292,
      "grad_norm": 103.5,
      "learning_rate": 2.1421215242018537e-07,
      "logits/chosen": -0.25703126192092896,
      "logits/rejected": -0.3667968809604645,
      "logps/chosen": -231.1999969482422,
      "logps/rejected": -217.8000030517578,
      "loss": 0.6312,
      "rewards/accuracies": 0.7360714673995972,
      "rewards/chosen": 0.24824218451976776,
      "rewards/margins": 0.14814452826976776,
      "rewards/rejected": 0.10061035305261612,
      "step": 2220
    },
    {
      "epoch": 0.5728630278063852,
      "grad_norm": 102.0,
      "learning_rate": 2.135684860968074e-07,
      "logits/chosen": -0.524218738079071,
      "logits/rejected": -0.42509764432907104,
      "logps/chosen": -262.3999938964844,
      "logps/rejected": -317.79998779296875,
      "loss": 0.6523,
      "rewards/accuracies": 0.6232784390449524,
      "rewards/chosen": 0.1416015625,
      "rewards/margins": 0.10184326022863388,
      "rewards/rejected": 0.03980712965130806,
      "step": 2225
    },
    {
      "epoch": 0.574150360453141,
      "grad_norm": 181.0,
      "learning_rate": 2.1292481977342942e-07,
      "logits/chosen": -0.31855469942092896,
      "logits/rejected": -0.26875001192092896,
      "logps/chosen": -275.6000061035156,
      "logps/rejected": -256.20001220703125,
      "loss": 0.6492,
      "rewards/accuracies": 0.5699175596237183,
      "rewards/chosen": 0.205810546875,
      "rewards/margins": 0.11095275729894638,
      "rewards/rejected": 0.09479980170726776,
      "step": 2230
    },
    {
      "epoch": 0.575437693099897,
      "grad_norm": 127.5,
      "learning_rate": 2.122811534500515e-07,
      "logits/chosen": -0.33867186307907104,
      "logits/rejected": -0.3431640565395355,
      "logps/chosen": -284.20001220703125,
      "logps/rejected": -234.60000610351562,
      "loss": 0.643,
      "rewards/accuracies": 0.6452381014823914,
      "rewards/chosen": 0.2505859434604645,
      "rewards/margins": 0.13212890923023224,
      "rewards/rejected": 0.11872558295726776,
      "step": 2235
    },
    {
      "epoch": 0.576725025746653,
      "grad_norm": 99.5,
      "learning_rate": 2.1163748712667354e-07,
      "logits/chosen": -0.42851561307907104,
      "logits/rejected": -0.30205076932907104,
      "logps/chosen": -288.79998779296875,
      "logps/rejected": -250.1999969482422,
      "loss": 0.6617,
      "rewards/accuracies": 0.49140113592147827,
      "rewards/chosen": 0.12712402641773224,
      "rewards/margins": 0.0832061767578125,
      "rewards/rejected": 0.04423828050494194,
      "step": 2240
    },
    {
      "epoch": 0.5780123583934088,
      "grad_norm": 107.0,
      "learning_rate": 2.1099382080329555e-07,
      "logits/chosen": -0.40434569120407104,
      "logits/rejected": -0.37324219942092896,
      "logps/chosen": -303.3999938964844,
      "logps/rejected": -251.8000030517578,
      "loss": 0.6586,
      "rewards/accuracies": 0.5684523582458496,
      "rewards/chosen": 0.19418945908546448,
      "rewards/margins": 0.09196777641773224,
      "rewards/rejected": 0.10203857719898224,
      "step": 2245
    },
    {
      "epoch": 0.5792996910401648,
      "grad_norm": 104.5,
      "learning_rate": 2.103501544799176e-07,
      "logits/chosen": -0.13750000298023224,
      "logits/rejected": -0.3189453184604645,
      "logps/chosen": -244.39999389648438,
      "logps/rejected": -292.3999938964844,
      "loss": 0.6711,
      "rewards/accuracies": 0.526978075504303,
      "rewards/chosen": 0.21547850966453552,
      "rewards/margins": 0.06674804538488388,
      "rewards/rejected": 0.14882811903953552,
      "step": 2250
    },
    {
      "epoch": 0.5805870236869207,
      "grad_norm": 100.5,
      "learning_rate": 2.0970648815653964e-07,
      "logits/chosen": -0.3988281190395355,
      "logits/rejected": -0.2681640684604645,
      "logps/chosen": -332.20001220703125,
      "logps/rejected": -254.60000610351562,
      "loss": 0.65,
      "rewards/accuracies": 0.49345239996910095,
      "rewards/chosen": 0.23520508408546448,
      "rewards/margins": 0.10268554836511612,
      "rewards/rejected": 0.1322021484375,
      "step": 2255
    },
    {
      "epoch": 0.5818743563336766,
      "grad_norm": 97.5,
      "learning_rate": 2.0906282183316168e-07,
      "logits/chosen": -0.28437501192092896,
      "logits/rejected": -0.28547364473342896,
      "logps/chosen": -276.20001220703125,
      "logps/rejected": -238.3000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.5224999785423279,
      "rewards/chosen": 0.21542969346046448,
      "rewards/margins": 0.05588836595416069,
      "rewards/rejected": 0.15957030653953552,
      "step": 2260
    },
    {
      "epoch": 0.5831616889804325,
      "grad_norm": 198.0,
      "learning_rate": 2.0841915550978372e-07,
      "logits/chosen": -0.25537109375,
      "logits/rejected": -0.36918944120407104,
      "logps/chosen": -279.3999938964844,
      "logps/rejected": -248.1999969482422,
      "loss": 0.6445,
      "rewards/accuracies": 0.5297619104385376,
      "rewards/chosen": 0.17573241889476776,
      "rewards/margins": 0.12316284328699112,
      "rewards/rejected": 0.052398681640625,
      "step": 2265
    },
    {
      "epoch": 0.5844490216271885,
      "grad_norm": 113.5,
      "learning_rate": 2.0777548918640574e-07,
      "logits/chosen": -0.26249998807907104,
      "logits/rejected": -0.2626953125,
      "logps/chosen": -250.0,
      "logps/rejected": -229.0,
      "loss": 0.6547,
      "rewards/accuracies": 0.5791666507720947,
      "rewards/chosen": 0.17988280951976776,
      "rewards/margins": 0.09969482570886612,
      "rewards/rejected": 0.0797119140625,
      "step": 2270
    },
    {
      "epoch": 0.5857363542739444,
      "grad_norm": 106.5,
      "learning_rate": 2.071318228630278e-07,
      "logits/chosen": -0.35429686307907104,
      "logits/rejected": -0.33378905057907104,
      "logps/chosen": -308.3999938964844,
      "logps/rejected": -301.20001220703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.18896484375,
      "rewards/margins": 0.07229004055261612,
      "rewards/rejected": 0.1168212890625,
      "step": 2275
    },
    {
      "epoch": 0.5870236869207003,
      "grad_norm": 99.0,
      "learning_rate": 2.0648815653964985e-07,
      "logits/chosen": -0.22148437798023224,
      "logits/rejected": -0.3060546815395355,
      "logps/chosen": -232.39999389648438,
      "logps/rejected": -214.39999389648438,
      "loss": 0.6656,
      "rewards/accuracies": 0.5753571391105652,
      "rewards/chosen": 0.16591796278953552,
      "rewards/margins": 0.0716552734375,
      "rewards/rejected": 0.09433593600988388,
      "step": 2280
    },
    {
      "epoch": 0.5883110195674562,
      "grad_norm": 243.0,
      "learning_rate": 2.0584449021627187e-07,
      "logits/chosen": -0.3460937440395355,
      "logits/rejected": -0.2568359375,
      "logps/chosen": -318.6000061035156,
      "logps/rejected": -273.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.442976176738739,
      "rewards/chosen": 0.12514647841453552,
      "rewards/margins": 0.067278191447258,
      "rewards/rejected": 0.0579833984375,
      "step": 2285
    },
    {
      "epoch": 0.5895983522142122,
      "grad_norm": 87.0,
      "learning_rate": 2.052008238928939e-07,
      "logits/chosen": -0.4964843690395355,
      "logits/rejected": -0.4605468809604645,
      "logps/chosen": -303.0,
      "logps/rejected": -297.6000061035156,
      "loss": 0.6695,
      "rewards/accuracies": 0.503333330154419,
      "rewards/chosen": 0.2138671875,
      "rewards/margins": 0.06174316257238388,
      "rewards/rejected": 0.15224608778953552,
      "step": 2290
    },
    {
      "epoch": 0.590885684860968,
      "grad_norm": 122.0,
      "learning_rate": 2.0455715756951595e-07,
      "logits/chosen": -0.2621093690395355,
      "logits/rejected": -0.298583984375,
      "logps/chosen": -261.3999938964844,
      "logps/rejected": -269.20001220703125,
      "loss": 0.6547,
      "rewards/accuracies": 0.5925000309944153,
      "rewards/chosen": 0.19277343153953552,
      "rewards/margins": 0.09316406399011612,
      "rewards/rejected": 0.09995117038488388,
      "step": 2295
    },
    {
      "epoch": 0.592173017507724,
      "grad_norm": 224.0,
      "learning_rate": 2.03913491246138e-07,
      "logits/chosen": -0.23613281548023224,
      "logits/rejected": -0.34765625,
      "logps/chosen": -302.0,
      "logps/rejected": -290.79998779296875,
      "loss": 0.6578,
      "rewards/accuracies": 0.5799816846847534,
      "rewards/chosen": 0.19246216118335724,
      "rewards/margins": 0.09294433891773224,
      "rewards/rejected": 0.09963379055261612,
      "step": 2300
    },
    {
      "epoch": 0.59346035015448,
      "grad_norm": 125.5,
      "learning_rate": 2.0326982492276004e-07,
      "logits/chosen": -0.30937498807907104,
      "logits/rejected": -0.3394531309604645,
      "logps/chosen": -331.6000061035156,
      "logps/rejected": -323.20001220703125,
      "loss": 0.6508,
      "rewards/accuracies": 0.5825000405311584,
      "rewards/chosen": 0.23361817002296448,
      "rewards/margins": 0.103759765625,
      "rewards/rejected": 0.12961426377296448,
      "step": 2305
    },
    {
      "epoch": 0.5947476828012358,
      "grad_norm": 86.5,
      "learning_rate": 2.0262615859938205e-07,
      "logits/chosen": -0.3597656190395355,
      "logits/rejected": -0.3583984375,
      "logps/chosen": -336.0,
      "logps/rejected": -307.79998779296875,
      "loss": 0.6281,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.3125,
      "rewards/margins": 0.16025391221046448,
      "rewards/rejected": 0.15253905951976776,
      "step": 2310
    },
    {
      "epoch": 0.5960350154479918,
      "grad_norm": 144.0,
      "learning_rate": 2.0198249227600412e-07,
      "logits/chosen": -0.28974610567092896,
      "logits/rejected": -0.2845703065395355,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -241.0,
      "loss": 0.6586,
      "rewards/accuracies": 0.4838828146457672,
      "rewards/chosen": 0.15927734971046448,
      "rewards/margins": 0.09174804389476776,
      "rewards/rejected": 0.06728515774011612,
      "step": 2315
    },
    {
      "epoch": 0.5973223480947477,
      "grad_norm": 90.0,
      "learning_rate": 2.0133882595262617e-07,
      "logits/chosen": -0.3763671815395355,
      "logits/rejected": -0.3941406309604645,
      "logps/chosen": -288.20001220703125,
      "logps/rejected": -259.6000061035156,
      "loss": 0.6609,
      "rewards/accuracies": 0.46916666626930237,
      "rewards/chosen": 0.18671874701976776,
      "rewards/margins": 0.07756347954273224,
      "rewards/rejected": 0.10935058444738388,
      "step": 2320
    },
    {
      "epoch": 0.5986096807415036,
      "grad_norm": 119.0,
      "learning_rate": 2.0069515962924818e-07,
      "logits/chosen": -0.3375000059604645,
      "logits/rejected": -0.3456054627895355,
      "logps/chosen": -241.1999969482422,
      "logps/rejected": -273.79998779296875,
      "loss": 0.6484,
      "rewards/accuracies": 0.5709524154663086,
      "rewards/chosen": 0.2041015625,
      "rewards/margins": 0.10295410454273224,
      "rewards/rejected": 0.10063476860523224,
      "step": 2325
    },
    {
      "epoch": 0.5998970133882595,
      "grad_norm": 101.0,
      "learning_rate": 2.0005149330587023e-07,
      "logits/chosen": -0.39179688692092896,
      "logits/rejected": -0.34208983182907104,
      "logps/chosen": -248.5,
      "logps/rejected": -226.8000030517578,
      "loss": 0.6687,
      "rewards/accuracies": 0.444444477558136,
      "rewards/chosen": 0.07145996391773224,
      "rewards/margins": 0.05913085862994194,
      "rewards/rejected": 0.01236572302877903,
      "step": 2330
    },
    {
      "epoch": 0.6011843460350155,
      "grad_norm": 113.5,
      "learning_rate": 1.9940782698249227e-07,
      "logits/chosen": -0.4320312440395355,
      "logits/rejected": -0.29316407442092896,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -283.6000061035156,
      "loss": 0.6578,
      "rewards/accuracies": 0.5209091305732727,
      "rewards/chosen": 0.24912109971046448,
      "rewards/margins": 0.08446045219898224,
      "rewards/rejected": 0.16472168266773224,
      "step": 2335
    },
    {
      "epoch": 0.6024716786817713,
      "grad_norm": 148.0,
      "learning_rate": 1.987641606591143e-07,
      "logits/chosen": -0.20253905653953552,
      "logits/rejected": -0.10808372497558594,
      "logps/chosen": -263.3999938964844,
      "logps/rejected": -262.3999938964844,
      "loss": 0.6664,
      "rewards/accuracies": 0.5091666579246521,
      "rewards/chosen": 0.21220703423023224,
      "rewards/margins": 0.07001952826976776,
      "rewards/rejected": 0.14238281548023224,
      "step": 2340
    },
    {
      "epoch": 0.6037590113285273,
      "grad_norm": 115.5,
      "learning_rate": 1.9812049433573635e-07,
      "logits/chosen": -0.3902343809604645,
      "logits/rejected": -0.4068359434604645,
      "logps/chosen": -289.6000061035156,
      "logps/rejected": -275.79998779296875,
      "loss": 0.6609,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.20000000298023224,
      "rewards/margins": 0.08369140326976776,
      "rewards/rejected": 0.11614990234375,
      "step": 2345
    },
    {
      "epoch": 0.6050463439752832,
      "grad_norm": 107.0,
      "learning_rate": 1.9747682801235837e-07,
      "logits/chosen": -0.3580078184604645,
      "logits/rejected": -0.39472657442092896,
      "logps/chosen": -287.79998779296875,
      "logps/rejected": -285.0,
      "loss": 0.6648,
      "rewards/accuracies": 0.4829029440879822,
      "rewards/chosen": 0.151123046875,
      "rewards/margins": 0.06293945014476776,
      "rewards/rejected": 0.08791504055261612,
      "step": 2350
    },
    {
      "epoch": 0.6063336766220392,
      "grad_norm": 93.5,
      "learning_rate": 1.9683316168898044e-07,
      "logits/chosen": -0.289703369140625,
      "logits/rejected": -0.3065429627895355,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -258.79998779296875,
      "loss": 0.6336,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.3285156190395355,
      "rewards/margins": 0.14560547471046448,
      "rewards/rejected": 0.18300780653953552,
      "step": 2355
    },
    {
      "epoch": 0.607621009268795,
      "grad_norm": 123.5,
      "learning_rate": 1.9618949536560248e-07,
      "logits/chosen": -0.33320313692092896,
      "logits/rejected": -0.37968748807907104,
      "logps/chosen": -260.0,
      "logps/rejected": -268.6000061035156,
      "loss": 0.6469,
      "rewards/accuracies": 0.602857232093811,
      "rewards/chosen": 0.26953125,
      "rewards/margins": 0.115234375,
      "rewards/rejected": 0.15410156548023224,
      "step": 2360
    },
    {
      "epoch": 0.608908341915551,
      "grad_norm": 127.0,
      "learning_rate": 1.955458290422245e-07,
      "logits/chosen": -0.4527343809604645,
      "logits/rejected": -0.3896484375,
      "logps/chosen": -248.39999389648438,
      "logps/rejected": -227.39999389648438,
      "loss": 0.6531,
      "rewards/accuracies": 0.5257875323295593,
      "rewards/chosen": 0.13461914658546448,
      "rewards/margins": 0.09495849907398224,
      "rewards/rejected": 0.03969726711511612,
      "step": 2365
    },
    {
      "epoch": 0.6101956745623069,
      "grad_norm": 107.0,
      "learning_rate": 1.9490216271884654e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": -0.14326171576976776,
      "logps/chosen": -265.79998779296875,
      "logps/rejected": -252.39999389648438,
      "loss": 0.6531,
      "rewards/accuracies": 0.553946852684021,
      "rewards/chosen": 0.16201171278953552,
      "rewards/margins": 0.09755859524011612,
      "rewards/rejected": 0.06430663913488388,
      "step": 2370
    },
    {
      "epoch": 0.6114830072090628,
      "grad_norm": 125.0,
      "learning_rate": 1.9425849639546856e-07,
      "logits/chosen": -0.24199219048023224,
      "logits/rejected": -0.25971680879592896,
      "logps/chosen": -249.39999389648438,
      "logps/rejected": -257.20001220703125,
      "loss": 0.6617,
      "rewards/accuracies": 0.5489102602005005,
      "rewards/chosen": 0.1826171875,
      "rewards/margins": 0.08102111518383026,
      "rewards/rejected": 0.10141601413488388,
      "step": 2375
    },
    {
      "epoch": 0.6127703398558187,
      "grad_norm": 97.5,
      "learning_rate": 1.9361483007209063e-07,
      "logits/chosen": -0.45820313692092896,
      "logits/rejected": -0.4429687559604645,
      "logps/chosen": -293.6000061035156,
      "logps/rejected": -269.79998779296875,
      "loss": 0.6461,
      "rewards/accuracies": 0.6501190662384033,
      "rewards/chosen": 0.21162109076976776,
      "rewards/margins": 0.1083984375,
      "rewards/rejected": 0.103271484375,
      "step": 2380
    },
    {
      "epoch": 0.6140576725025747,
      "grad_norm": 110.0,
      "learning_rate": 1.9297116374871267e-07,
      "logits/chosen": -0.3658203184604645,
      "logits/rejected": -0.3207031190395355,
      "logps/chosen": -257.0,
      "logps/rejected": -222.0,
      "loss": 0.6617,
      "rewards/accuracies": 0.45571431517601013,
      "rewards/chosen": 0.12873534858226776,
      "rewards/margins": 0.076171875,
      "rewards/rejected": 0.05267333984375,
      "step": 2385
    },
    {
      "epoch": 0.6153450051493305,
      "grad_norm": 129.0,
      "learning_rate": 1.9232749742533468e-07,
      "logits/chosen": -0.24687500298023224,
      "logits/rejected": -0.423828125,
      "logps/chosen": -235.39999389648438,
      "logps/rejected": -237.0,
      "loss": 0.6641,
      "rewards/accuracies": 0.5317857265472412,
      "rewards/chosen": 0.16484375298023224,
      "rewards/margins": 0.07424316555261612,
      "rewards/rejected": 0.09062500298023224,
      "step": 2390
    },
    {
      "epoch": 0.6166323377960865,
      "grad_norm": 696.0,
      "learning_rate": 1.9168383110195673e-07,
      "logits/chosen": -0.287109375,
      "logits/rejected": -0.30644530057907104,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -283.79998779296875,
      "loss": 0.6547,
      "rewards/accuracies": 0.6100000143051147,
      "rewards/chosen": 0.24570313096046448,
      "rewards/margins": 0.10708007961511612,
      "rewards/rejected": 0.13907471299171448,
      "step": 2395
    },
    {
      "epoch": 0.6179196704428425,
      "grad_norm": 112.0,
      "learning_rate": 1.910401647785788e-07,
      "logits/chosen": -0.3477539122104645,
      "logits/rejected": -0.21816405653953552,
      "logps/chosen": -242.39999389648438,
      "logps/rejected": -253.1999969482422,
      "loss": 0.6609,
      "rewards/accuracies": 0.526025652885437,
      "rewards/chosen": 0.23964843153953552,
      "rewards/margins": 0.09111328423023224,
      "rewards/rejected": 0.14833983778953552,
      "step": 2400
    },
    {
      "epoch": 0.6192070030895983,
      "grad_norm": 272.0,
      "learning_rate": 1.903964984552008e-07,
      "logits/chosen": -0.35029298067092896,
      "logits/rejected": -0.3031005859375,
      "logps/chosen": -278.20001220703125,
      "logps/rejected": -227.1999969482422,
      "loss": 0.6531,
      "rewards/accuracies": 0.5267857313156128,
      "rewards/chosen": 0.20517578721046448,
      "rewards/margins": 0.10906982421875,
      "rewards/rejected": 0.09577636420726776,
      "step": 2405
    },
    {
      "epoch": 0.6204943357363543,
      "grad_norm": 102.0,
      "learning_rate": 1.8975283213182286e-07,
      "logits/chosen": -0.30238038301467896,
      "logits/rejected": -0.3023437559604645,
      "logps/chosen": -317.6000061035156,
      "logps/rejected": -253.8000030517578,
      "loss": 0.6188,
      "rewards/accuracies": 0.7225000262260437,
      "rewards/chosen": 0.2666992247104645,
      "rewards/margins": 0.17939452826976776,
      "rewards/rejected": 0.08793945610523224,
      "step": 2410
    },
    {
      "epoch": 0.6217816683831102,
      "grad_norm": 93.0,
      "learning_rate": 1.8910916580844487e-07,
      "logits/chosen": -0.4068359434604645,
      "logits/rejected": -0.3759765625,
      "logps/chosen": -303.20001220703125,
      "logps/rejected": -255.60000610351562,
      "loss": 0.6672,
      "rewards/accuracies": 0.47678571939468384,
      "rewards/chosen": 0.20039062201976776,
      "rewards/margins": 0.07421875,
      "rewards/rejected": 0.12593993544578552,
      "step": 2415
    },
    {
      "epoch": 0.6230690010298661,
      "grad_norm": 148.0,
      "learning_rate": 1.8846549948506694e-07,
      "logits/chosen": -0.43632811307907104,
      "logits/rejected": -0.34550780057907104,
      "logps/chosen": -268.20001220703125,
      "logps/rejected": -248.8000030517578,
      "loss": 0.6547,
      "rewards/accuracies": 0.5866667032241821,
      "rewards/chosen": 0.20322266221046448,
      "rewards/margins": 0.09379883110523224,
      "rewards/rejected": 0.10988769680261612,
      "step": 2420
    },
    {
      "epoch": 0.624356333676622,
      "grad_norm": 112.5,
      "learning_rate": 1.8782183316168898e-07,
      "logits/chosen": -0.33085936307907104,
      "logits/rejected": -0.2650390565395355,
      "logps/chosen": -288.20001220703125,
      "logps/rejected": -267.20001220703125,
      "loss": 0.6406,
      "rewards/accuracies": 0.5741666555404663,
      "rewards/chosen": 0.22119140625,
      "rewards/margins": 0.12797851860523224,
      "rewards/rejected": 0.09311523288488388,
      "step": 2425
    },
    {
      "epoch": 0.625643666323378,
      "grad_norm": 107.0,
      "learning_rate": 1.87178166838311e-07,
      "logits/chosen": -0.33037108182907104,
      "logits/rejected": -0.39082032442092896,
      "logps/chosen": -294.0,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6758,
      "rewards/accuracies": 0.5269047617912292,
      "rewards/chosen": 0.22109374403953552,
      "rewards/margins": 0.05648193508386612,
      "rewards/rejected": 0.16425780951976776,
      "step": 2430
    },
    {
      "epoch": 0.6269309989701339,
      "grad_norm": 108.0,
      "learning_rate": 1.8653450051493304e-07,
      "logits/chosen": -0.4267578125,
      "logits/rejected": -0.4765625,
      "logps/chosen": -294.79998779296875,
      "logps/rejected": -268.6000061035156,
      "loss": 0.6609,
      "rewards/accuracies": 0.5826923251152039,
      "rewards/chosen": 0.20390625298023224,
      "rewards/margins": 0.07270507514476776,
      "rewards/rejected": 0.13081054389476776,
      "step": 2435
    },
    {
      "epoch": 0.6282183316168898,
      "grad_norm": 173.0,
      "learning_rate": 1.858908341915551e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.3824218809604645,
      "logps/chosen": -282.3999938964844,
      "logps/rejected": -295.20001220703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.205322265625,
      "rewards/margins": 0.06217346340417862,
      "rewards/rejected": 0.14307251572608948,
      "step": 2440
    },
    {
      "epoch": 0.6295056642636457,
      "grad_norm": 134.0,
      "learning_rate": 1.8524716786817713e-07,
      "logits/chosen": -0.17919921875,
      "logits/rejected": -0.15712890028953552,
      "logps/chosen": -249.8000030517578,
      "logps/rejected": -259.3999938964844,
      "loss": 0.6484,
      "rewards/accuracies": 0.5948809385299683,
      "rewards/chosen": 0.20585937798023224,
      "rewards/margins": 0.10800781100988388,
      "rewards/rejected": 0.09760741889476776,
      "step": 2445
    },
    {
      "epoch": 0.6307929969104017,
      "grad_norm": 125.5,
      "learning_rate": 1.8460350154479917e-07,
      "logits/chosen": -0.3394531309604645,
      "logits/rejected": -0.4019531309604645,
      "logps/chosen": -278.79998779296875,
      "logps/rejected": -241.60000610351562,
      "loss": 0.6547,
      "rewards/accuracies": 0.5533333420753479,
      "rewards/chosen": 0.20217284560203552,
      "rewards/margins": 0.09189452975988388,
      "rewards/rejected": 0.11025390774011612,
      "step": 2450
    },
    {
      "epoch": 0.6320803295571575,
      "grad_norm": 121.0,
      "learning_rate": 1.8395983522142119e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": -0.35078126192092896,
      "logps/chosen": -268.79998779296875,
      "logps/rejected": -275.0,
      "loss": 0.6523,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.23203125596046448,
      "rewards/margins": 0.11406250298023224,
      "rewards/rejected": 0.11806640774011612,
      "step": 2455
    },
    {
      "epoch": 0.6333676622039135,
      "grad_norm": 125.5,
      "learning_rate": 1.8331616889804326e-07,
      "logits/chosen": -0.365234375,
      "logits/rejected": -0.32499998807907104,
      "logps/chosen": -291.0,
      "logps/rejected": -279.0,
      "loss": 0.6344,
      "rewards/accuracies": 0.5382051467895508,
      "rewards/chosen": 0.28173828125,
      "rewards/margins": 0.14545898139476776,
      "rewards/rejected": 0.13613280653953552,
      "step": 2460
    },
    {
      "epoch": 0.6346549948506695,
      "grad_norm": 120.5,
      "learning_rate": 1.826725025746653e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": -0.4496093690395355,
      "logps/chosen": -325.0,
      "logps/rejected": -323.20001220703125,
      "loss": 0.6461,
      "rewards/accuracies": 0.607619047164917,
      "rewards/chosen": 0.21884766221046448,
      "rewards/margins": 0.11015625298023224,
      "rewards/rejected": 0.108612060546875,
      "step": 2465
    },
    {
      "epoch": 0.6359423274974253,
      "grad_norm": 87.5,
      "learning_rate": 1.8202883625128731e-07,
      "logits/chosen": -0.35429686307907104,
      "logits/rejected": -0.29472655057907104,
      "logps/chosen": -274.20001220703125,
      "logps/rejected": -264.20001220703125,
      "loss": 0.6625,
      "rewards/accuracies": 0.5498718023300171,
      "rewards/chosen": 0.166259765625,
      "rewards/margins": 0.09056396782398224,
      "rewards/rejected": 0.07568359375,
      "step": 2470
    },
    {
      "epoch": 0.6372296601441813,
      "grad_norm": 200.0,
      "learning_rate": 1.8138516992790936e-07,
      "logits/chosen": -0.28203123807907104,
      "logits/rejected": -0.3480468690395355,
      "logps/chosen": -292.79998779296875,
      "logps/rejected": -269.0,
      "loss": 0.65,
      "rewards/accuracies": 0.5677381157875061,
      "rewards/chosen": 0.24189452826976776,
      "rewards/margins": 0.112060546875,
      "rewards/rejected": 0.13007812201976776,
      "step": 2475
    },
    {
      "epoch": 0.6385169927909372,
      "grad_norm": 115.0,
      "learning_rate": 1.8074150360453143e-07,
      "logits/chosen": -0.2759765684604645,
      "logits/rejected": -0.2728515565395355,
      "logps/chosen": -264.0,
      "logps/rejected": -280.3999938964844,
      "loss": 0.6539,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.24423828721046448,
      "rewards/margins": 0.107177734375,
      "rewards/rejected": 0.13710936903953552,
      "step": 2480
    },
    {
      "epoch": 0.6398043254376931,
      "grad_norm": 115.5,
      "learning_rate": 1.8009783728115344e-07,
      "logits/chosen": -0.38007813692092896,
      "logits/rejected": -0.41289061307907104,
      "logps/chosen": -284.79998779296875,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6508,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.23642578721046448,
      "rewards/margins": 0.11301269382238388,
      "rewards/rejected": 0.12360839545726776,
      "step": 2485
    },
    {
      "epoch": 0.641091658084449,
      "grad_norm": 203.0,
      "learning_rate": 1.7945417095777549e-07,
      "logits/chosen": -0.38908690214157104,
      "logits/rejected": -0.35502928495407104,
      "logps/chosen": -213.8000030517578,
      "logps/rejected": -232.8000030517578,
      "loss": 0.6516,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": 0.22998046875,
      "rewards/margins": 0.09882812201976776,
      "rewards/rejected": 0.13151855766773224,
      "step": 2490
    },
    {
      "epoch": 0.642378990731205,
      "grad_norm": 171.0,
      "learning_rate": 1.788105046343975e-07,
      "logits/chosen": -0.3119140565395355,
      "logits/rejected": -0.3550781309604645,
      "logps/chosen": -250.1999969482422,
      "logps/rejected": -229.1999969482422,
      "loss": 0.6695,
      "rewards/accuracies": 0.5168614983558655,
      "rewards/chosen": 0.13334961235523224,
      "rewards/margins": 0.06949462741613388,
      "rewards/rejected": 0.06401367485523224,
      "step": 2495
    },
    {
      "epoch": 0.6436663233779608,
      "grad_norm": 105.0,
      "learning_rate": 1.7816683831101954e-07,
      "logits/chosen": -0.36503905057907104,
      "logits/rejected": -0.38945311307907104,
      "logps/chosen": -318.3999938964844,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6648,
      "rewards/accuracies": 0.49416670203208923,
      "rewards/chosen": 0.20449218153953552,
      "rewards/margins": 0.07508544623851776,
      "rewards/rejected": 0.1298828125,
      "step": 2500
    },
    {
      "epoch": 0.6449536560247168,
      "grad_norm": 172.0,
      "learning_rate": 1.7752317198764161e-07,
      "logits/chosen": -0.21923828125,
      "logits/rejected": -0.33378905057907104,
      "logps/chosen": -284.20001220703125,
      "logps/rejected": -231.8000030517578,
      "loss": 0.6484,
      "rewards/accuracies": 0.5689743757247925,
      "rewards/chosen": 0.16943359375,
      "rewards/margins": 0.10085143893957138,
      "rewards/rejected": 0.06889648735523224,
      "step": 2505
    },
    {
      "epoch": 0.6462409886714727,
      "grad_norm": 145.0,
      "learning_rate": 1.7687950566426363e-07,
      "logits/chosen": -0.3255859315395355,
      "logits/rejected": -0.3597656190395355,
      "logps/chosen": -287.0,
      "logps/rejected": -241.1999969482422,
      "loss": 0.6625,
      "rewards/accuracies": 0.4926648736000061,
      "rewards/chosen": 0.11082763969898224,
      "rewards/margins": 0.07229004055261612,
      "rewards/rejected": 0.03874511644244194,
      "step": 2510
    },
    {
      "epoch": 0.6475283213182287,
      "grad_norm": 122.0,
      "learning_rate": 1.7623583934088567e-07,
      "logits/chosen": -0.357421875,
      "logits/rejected": -0.4146484434604645,
      "logps/chosen": -354.3999938964844,
      "logps/rejected": -268.3999938964844,
      "loss": 0.6891,
      "rewards/accuracies": 0.4067857265472412,
      "rewards/chosen": 0.15712890028953552,
      "rewards/margins": 0.03066406212747097,
      "rewards/rejected": 0.12656250596046448,
      "step": 2515
    },
    {
      "epoch": 0.6488156539649845,
      "grad_norm": 176.0,
      "learning_rate": 1.7559217301750771e-07,
      "logits/chosen": -0.40546876192092896,
      "logits/rejected": -0.4136718809604645,
      "logps/chosen": -291.0,
      "logps/rejected": -266.3999938964844,
      "loss": 0.6594,
      "rewards/accuracies": 0.5360714197158813,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 0.10804443061351776,
      "rewards/rejected": 0.11347655951976776,
      "step": 2520
    },
    {
      "epoch": 0.6501029866117405,
      "grad_norm": 112.0,
      "learning_rate": 1.7494850669412976e-07,
      "logits/chosen": -0.3158203065395355,
      "logits/rejected": -0.3541015684604645,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -261.20001220703125,
      "loss": 0.6406,
      "rewards/accuracies": 0.6294047832489014,
      "rewards/chosen": 0.23652343451976776,
      "rewards/margins": 0.12563475966453552,
      "rewards/rejected": 0.11110839992761612,
      "step": 2525
    },
    {
      "epoch": 0.6513903192584964,
      "grad_norm": 101.0,
      "learning_rate": 1.743048403707518e-07,
      "logits/chosen": -0.30904388427734375,
      "logits/rejected": -0.3011474609375,
      "logps/chosen": -304.0,
      "logps/rejected": -283.6000061035156,
      "loss": 0.625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.29179686307907104,
      "rewards/margins": 0.1611328125,
      "rewards/rejected": 0.13095703721046448,
      "step": 2530
    },
    {
      "epoch": 0.6526776519052523,
      "grad_norm": 112.0,
      "learning_rate": 1.7366117404737382e-07,
      "logits/chosen": -0.35761719942092896,
      "logits/rejected": -0.38496094942092896,
      "logps/chosen": -329.20001220703125,
      "logps/rejected": -223.0,
      "loss": 0.6406,
      "rewards/accuracies": 0.6150000095367432,
      "rewards/chosen": 0.2578125,
      "rewards/margins": 0.13735350966453552,
      "rewards/rejected": 0.12062988430261612,
      "step": 2535
    },
    {
      "epoch": 0.6539649845520082,
      "grad_norm": 88.0,
      "learning_rate": 1.7301750772399586e-07,
      "logits/chosen": -0.44999998807907104,
      "logits/rejected": -0.4144531190395355,
      "logps/chosen": -296.79998779296875,
      "logps/rejected": -251.1999969482422,
      "loss": 0.6609,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.2744140625,
      "rewards/margins": 0.09549560397863388,
      "rewards/rejected": 0.17890624701976776,
      "step": 2540
    },
    {
      "epoch": 0.6552523171987642,
      "grad_norm": 143.0,
      "learning_rate": 1.7237384140061793e-07,
      "logits/chosen": -0.23808594048023224,
      "logits/rejected": -0.291015625,
      "logps/chosen": -204.60000610351562,
      "logps/rejected": -200.3000030517578,
      "loss": 0.6672,
      "rewards/accuracies": 0.4816666543483734,
      "rewards/chosen": 0.14433594048023224,
      "rewards/margins": 0.0693359375,
      "rewards/rejected": 0.07504882663488388,
      "step": 2545
    },
    {
      "epoch": 0.65653964984552,
      "grad_norm": 141.0,
      "learning_rate": 1.7173017507723994e-07,
      "logits/chosen": -0.2964111268520355,
      "logits/rejected": -0.35429686307907104,
      "logps/chosen": -269.20001220703125,
      "logps/rejected": -233.8000030517578,
      "loss": 0.6406,
      "rewards/accuracies": 0.65666663646698,
      "rewards/chosen": 0.2800048887729645,
      "rewards/margins": 0.13671875,
      "rewards/rejected": 0.14276123046875,
      "step": 2550
    },
    {
      "epoch": 0.657826982492276,
      "grad_norm": 134.0,
      "learning_rate": 1.71086508753862e-07,
      "logits/chosen": -0.4566406309604645,
      "logits/rejected": -0.3133789002895355,
      "logps/chosen": -285.0,
      "logps/rejected": -241.0,
      "loss": 0.632,
      "rewards/accuracies": 0.5824999809265137,
      "rewards/chosen": 0.19423827528953552,
      "rewards/margins": 0.14511719346046448,
      "rewards/rejected": 0.04902343824505806,
      "step": 2555
    },
    {
      "epoch": 0.659114315139032,
      "grad_norm": 144.0,
      "learning_rate": 1.7044284243048403e-07,
      "logits/chosen": -0.33305662870407104,
      "logits/rejected": -0.3880859315395355,
      "logps/chosen": -262.3999938964844,
      "logps/rejected": -268.6000061035156,
      "loss": 0.6695,
      "rewards/accuracies": 0.5371794700622559,
      "rewards/chosen": 0.112060546875,
      "rewards/margins": 0.071533203125,
      "rewards/rejected": 0.04021301120519638,
      "step": 2560
    },
    {
      "epoch": 0.6604016477857878,
      "grad_norm": 161.0,
      "learning_rate": 1.6979917610710607e-07,
      "logits/chosen": -0.3375000059604645,
      "logits/rejected": -0.4443359375,
      "logps/chosen": -248.60000610351562,
      "logps/rejected": -270.3999938964844,
      "loss": 0.6602,
      "rewards/accuracies": 0.6232143044471741,
      "rewards/chosen": 0.19121094048023224,
      "rewards/margins": 0.08442077785730362,
      "rewards/rejected": 0.106719970703125,
      "step": 2565
    },
    {
      "epoch": 0.6616889804325438,
      "grad_norm": 115.0,
      "learning_rate": 1.6915550978372812e-07,
      "logits/chosen": -0.31982421875,
      "logits/rejected": -0.3248046934604645,
      "logps/chosen": -349.20001220703125,
      "logps/rejected": -266.79998779296875,
      "loss": 0.6383,
      "rewards/accuracies": 0.5505952835083008,
      "rewards/chosen": 0.24433593451976776,
      "rewards/margins": 0.14423827826976776,
      "rewards/rejected": 0.10017089545726776,
      "step": 2570
    },
    {
      "epoch": 0.6629763130792997,
      "grad_norm": 100.0,
      "learning_rate": 1.6851184346035013e-07,
      "logits/chosen": -0.4603515565395355,
      "logits/rejected": -0.4085449278354645,
      "logps/chosen": -310.3999938964844,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6617,
      "rewards/accuracies": 0.5763095617294312,
      "rewards/chosen": 0.20351561903953552,
      "rewards/margins": 0.09003905951976776,
      "rewards/rejected": 0.11333008110523224,
      "step": 2575
    },
    {
      "epoch": 0.6642636457260556,
      "grad_norm": 111.0,
      "learning_rate": 1.6786817713697217e-07,
      "logits/chosen": -0.3902343809604645,
      "logits/rejected": -0.423828125,
      "logps/chosen": -313.3999938964844,
      "logps/rejected": -330.0,
      "loss": 0.6687,
      "rewards/accuracies": 0.5830952525138855,
      "rewards/chosen": 0.2689453065395355,
      "rewards/margins": 0.07707519829273224,
      "rewards/rejected": 0.19189453125,
      "step": 2580
    },
    {
      "epoch": 0.6655509783728115,
      "grad_norm": 127.5,
      "learning_rate": 1.6722451081359424e-07,
      "logits/chosen": -0.32499998807907104,
      "logits/rejected": -0.3701171875,
      "logps/chosen": -286.20001220703125,
      "logps/rejected": -291.6000061035156,
      "loss": 0.6461,
      "rewards/accuracies": 0.4872511029243469,
      "rewards/chosen": 0.14594726264476776,
      "rewards/margins": 0.11098632961511612,
      "rewards/rejected": 0.03500976413488388,
      "step": 2585
    },
    {
      "epoch": 0.6668383110195675,
      "grad_norm": 110.5,
      "learning_rate": 1.6658084449021626e-07,
      "logits/chosen": -0.2548828125,
      "logits/rejected": -0.2669921815395355,
      "logps/chosen": -242.0,
      "logps/rejected": -210.39999389648438,
      "loss": 0.6398,
      "rewards/accuracies": 0.6648351550102234,
      "rewards/chosen": 0.234375,
      "rewards/margins": 0.12358398735523224,
      "rewards/rejected": 0.11062011867761612,
      "step": 2590
    },
    {
      "epoch": 0.6681256436663234,
      "grad_norm": 171.0,
      "learning_rate": 1.659371781668383e-07,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.322509765625,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -238.39999389648438,
      "loss": 0.6555,
      "rewards/accuracies": 0.5644047260284424,
      "rewards/chosen": 0.2640624940395355,
      "rewards/margins": 0.11264648288488388,
      "rewards/rejected": 0.15139159560203552,
      "step": 2595
    },
    {
      "epoch": 0.6694129763130793,
      "grad_norm": 99.5,
      "learning_rate": 1.6529351184346034e-07,
      "logits/chosen": -0.5166015625,
      "logits/rejected": -0.43769532442092896,
      "logps/chosen": -323.79998779296875,
      "logps/rejected": -262.20001220703125,
      "loss": 0.6508,
      "rewards/accuracies": 0.5258333086967468,
      "rewards/chosen": 0.22050781548023224,
      "rewards/margins": 0.103271484375,
      "rewards/rejected": 0.1168212890625,
      "step": 2600
    },
    {
      "epoch": 0.6707003089598352,
      "grad_norm": 106.5,
      "learning_rate": 1.646498455200824e-07,
      "logits/chosen": -0.32890623807907104,
      "logits/rejected": -0.43867188692092896,
      "logps/chosen": -223.60000610351562,
      "logps/rejected": -235.1999969482422,
      "loss": 0.6422,
      "rewards/accuracies": 0.6089102625846863,
      "rewards/chosen": 0.23945312201976776,
      "rewards/margins": 0.12080077826976776,
      "rewards/rejected": 0.11904297024011612,
      "step": 2605
    },
    {
      "epoch": 0.6719876416065912,
      "grad_norm": 123.0,
      "learning_rate": 1.6400617919670443e-07,
      "logits/chosen": -0.39482420682907104,
      "logits/rejected": -0.46367186307907104,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -229.1999969482422,
      "loss": 0.6391,
      "rewards/accuracies": 0.5491880178451538,
      "rewards/chosen": 0.22958984971046448,
      "rewards/margins": 0.12270507961511612,
      "rewards/rejected": 0.10697021335363388,
      "step": 2610
    },
    {
      "epoch": 0.673274974253347,
      "grad_norm": 115.0,
      "learning_rate": 1.6336251287332645e-07,
      "logits/chosen": -0.3529296815395355,
      "logits/rejected": -0.4330078065395355,
      "logps/chosen": -294.6000061035156,
      "logps/rejected": -264.0,
      "loss": 0.6578,
      "rewards/accuracies": 0.4816666543483734,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 0.10385742038488388,
      "rewards/rejected": 0.16777344048023224,
      "step": 2615
    },
    {
      "epoch": 0.674562306900103,
      "grad_norm": 97.5,
      "learning_rate": 1.627188465499485e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.32988280057907104,
      "logps/chosen": -264.6000061035156,
      "logps/rejected": -299.79998779296875,
      "loss": 0.6711,
      "rewards/accuracies": 0.4818589687347412,
      "rewards/chosen": 0.14853516221046448,
      "rewards/margins": 0.06865234673023224,
      "rewards/rejected": 0.07978515326976776,
      "step": 2620
    },
    {
      "epoch": 0.675849639546859,
      "grad_norm": 101.0,
      "learning_rate": 1.6207518022657056e-07,
      "logits/chosen": -0.3267578184604645,
      "logits/rejected": -0.107421875,
      "logps/chosen": -253.75,
      "logps/rejected": -197.5,
      "loss": 0.6703,
      "rewards/accuracies": 0.4098193645477295,
      "rewards/chosen": 0.11344604194164276,
      "rewards/margins": 0.05928955227136612,
      "rewards/rejected": 0.05446166917681694,
      "step": 2625
    },
    {
      "epoch": 0.6771369721936148,
      "grad_norm": 117.5,
      "learning_rate": 1.6143151390319257e-07,
      "logits/chosen": -0.3597656190395355,
      "logits/rejected": -0.37890625,
      "logps/chosen": -252.1999969482422,
      "logps/rejected": -229.60000610351562,
      "loss": 0.6477,
      "rewards/accuracies": 0.5794047117233276,
      "rewards/chosen": 0.2984375059604645,
      "rewards/margins": 0.11103515326976776,
      "rewards/rejected": 0.18730469048023224,
      "step": 2630
    },
    {
      "epoch": 0.6784243048403708,
      "grad_norm": 114.5,
      "learning_rate": 1.6078784757981462e-07,
      "logits/chosen": -0.2879882752895355,
      "logits/rejected": -0.2660156190395355,
      "logps/chosen": -244.60000610351562,
      "logps/rejected": -208.0,
      "loss": 0.6609,
      "rewards/accuracies": 0.6183333396911621,
      "rewards/chosen": 0.17031249403953552,
      "rewards/margins": 0.08011474460363388,
      "rewards/rejected": 0.09050903469324112,
      "step": 2635
    },
    {
      "epoch": 0.6797116374871267,
      "grad_norm": 123.5,
      "learning_rate": 1.6014418125643666e-07,
      "logits/chosen": -0.35820311307907104,
      "logits/rejected": -0.3587890565395355,
      "logps/chosen": -303.20001220703125,
      "logps/rejected": -291.79998779296875,
      "loss": 0.6297,
      "rewards/accuracies": 0.6292948722839355,
      "rewards/chosen": 0.20263671875,
      "rewards/margins": 0.162109375,
      "rewards/rejected": 0.04036865383386612,
      "step": 2640
    },
    {
      "epoch": 0.6809989701338826,
      "grad_norm": 206.0,
      "learning_rate": 1.5950051493305868e-07,
      "logits/chosen": -0.33613282442092896,
      "logits/rejected": -0.322265625,
      "logps/chosen": -272.0,
      "logps/rejected": -233.10000610351562,
      "loss": 0.6555,
      "rewards/accuracies": 0.5396367907524109,
      "rewards/chosen": 0.16640624403953552,
      "rewards/margins": 0.09023437649011612,
      "rewards/rejected": 0.07605590671300888,
      "step": 2645
    },
    {
      "epoch": 0.6822863027806385,
      "grad_norm": 104.5,
      "learning_rate": 1.5885684860968075e-07,
      "logits/chosen": -0.3990234434604645,
      "logits/rejected": -0.2782226502895355,
      "logps/chosen": -261.6000061035156,
      "logps/rejected": -234.39999389648438,
      "loss": 0.6516,
      "rewards/accuracies": 0.5141667127609253,
      "rewards/chosen": 0.21469727158546448,
      "rewards/margins": 0.10417480766773224,
      "rewards/rejected": 0.11081848293542862,
      "step": 2650
    },
    {
      "epoch": 0.6835736354273945,
      "grad_norm": 103.5,
      "learning_rate": 1.5821318228630276e-07,
      "logits/chosen": -0.3798828125,
      "logits/rejected": -0.3265624940395355,
      "logps/chosen": -280.79998779296875,
      "logps/rejected": -277.0,
      "loss": 0.6469,
      "rewards/accuracies": 0.6050000190734863,
      "rewards/chosen": 0.27460938692092896,
      "rewards/margins": 0.10689697414636612,
      "rewards/rejected": 0.16757813096046448,
      "step": 2655
    },
    {
      "epoch": 0.6848609680741503,
      "grad_norm": 92.0,
      "learning_rate": 1.575695159629248e-07,
      "logits/chosen": -0.4251953065395355,
      "logits/rejected": -0.4390625059604645,
      "logps/chosen": -309.3999938964844,
      "logps/rejected": -226.8000030517578,
      "loss": 0.6156,
      "rewards/accuracies": 0.6573809385299683,
      "rewards/chosen": 0.3216796815395355,
      "rewards/margins": 0.19755859673023224,
      "rewards/rejected": 0.12441406399011612,
      "step": 2660
    },
    {
      "epoch": 0.6861483007209063,
      "grad_norm": 100.0,
      "learning_rate": 1.5692584963954685e-07,
      "logits/chosen": -0.3394531309604645,
      "logits/rejected": -0.287109375,
      "logps/chosen": -245.39999389648438,
      "logps/rejected": -230.1999969482422,
      "loss": 0.6391,
      "rewards/accuracies": 0.5410606265068054,
      "rewards/chosen": 0.1875,
      "rewards/margins": 0.12109375,
      "rewards/rejected": 0.06641845405101776,
      "step": 2665
    },
    {
      "epoch": 0.6874356333676622,
      "grad_norm": 127.0,
      "learning_rate": 1.562821833161689e-07,
      "logits/chosen": -0.27910155057907104,
      "logits/rejected": -0.3773437440395355,
      "logps/chosen": -287.20001220703125,
      "logps/rejected": -282.6000061035156,
      "loss": 0.6539,
      "rewards/accuracies": 0.5476190447807312,
      "rewards/chosen": 0.166748046875,
      "rewards/margins": 0.11312256008386612,
      "rewards/rejected": 0.053466796875,
      "step": 2670
    },
    {
      "epoch": 0.6887229660144182,
      "grad_norm": 133.0,
      "learning_rate": 1.5563851699279093e-07,
      "logits/chosen": -0.5406249761581421,
      "logits/rejected": -0.44843751192092896,
      "logps/chosen": -265.79998779296875,
      "logps/rejected": -253.39999389648438,
      "loss": 0.6469,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.22585448622703552,
      "rewards/margins": 0.11998901516199112,
      "rewards/rejected": 0.10566405951976776,
      "step": 2675
    },
    {
      "epoch": 0.690010298661174,
      "grad_norm": 564.0,
      "learning_rate": 1.5499485066941297e-07,
      "logits/chosen": -0.3031249940395355,
      "logits/rejected": -0.3375000059604645,
      "logps/chosen": -306.3999938964844,
      "logps/rejected": -297.0,
      "loss": 0.6602,
      "rewards/accuracies": 0.4766666293144226,
      "rewards/chosen": 0.23076172173023224,
      "rewards/margins": 0.08818359673023224,
      "rewards/rejected": 0.14275512099266052,
      "step": 2680
    },
    {
      "epoch": 0.69129763130793,
      "grad_norm": 104.0,
      "learning_rate": 1.54351184346035e-07,
      "logits/chosen": -0.2904296815395355,
      "logits/rejected": -0.24375000596046448,
      "logps/chosen": -284.0,
      "logps/rejected": -277.0,
      "loss": 0.6438,
      "rewards/accuracies": 0.6058333516120911,
      "rewards/chosen": 0.17246094346046448,
      "rewards/margins": 0.09707031399011612,
      "rewards/rejected": 0.0751953125,
      "step": 2685
    },
    {
      "epoch": 0.6925849639546859,
      "grad_norm": 86.0,
      "learning_rate": 1.5370751802265706e-07,
      "logits/chosen": -0.3248046934604645,
      "logits/rejected": -0.24160155653953552,
      "logps/chosen": -292.6000061035156,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6367,
      "rewards/accuracies": 0.5958333611488342,
      "rewards/chosen": 0.35859376192092896,
      "rewards/margins": 0.12807616591453552,
      "rewards/rejected": 0.23066405951976776,
      "step": 2690
    },
    {
      "epoch": 0.6938722966014418,
      "grad_norm": 142.0,
      "learning_rate": 1.5306385169927908e-07,
      "logits/chosen": -0.2266494780778885,
      "logits/rejected": -0.16435547173023224,
      "logps/chosen": -280.20001220703125,
      "logps/rejected": -212.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.5088095664978027,
      "rewards/chosen": 0.18671874701976776,
      "rewards/margins": 0.10224609076976776,
      "rewards/rejected": 0.08452530205249786,
      "step": 2695
    },
    {
      "epoch": 0.6951596292481977,
      "grad_norm": 125.0,
      "learning_rate": 1.5242018537590112e-07,
      "logits/chosen": -0.38671875,
      "logits/rejected": 0.015551757998764515,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -264.79998779296875,
      "loss": 0.6648,
      "rewards/accuracies": 0.5238462090492249,
      "rewards/chosen": 0.11220703274011612,
      "rewards/margins": 0.07094726711511612,
      "rewards/rejected": 0.04129943996667862,
      "step": 2700
    },
    {
      "epoch": 0.6964469618949537,
      "grad_norm": 130.0,
      "learning_rate": 1.5177651905252316e-07,
      "logits/chosen": -0.3368164002895355,
      "logits/rejected": -0.43828123807907104,
      "logps/chosen": -322.79998779296875,
      "logps/rejected": -263.79998779296875,
      "loss": 0.6383,
      "rewards/accuracies": 0.46358975768089294,
      "rewards/chosen": 0.20419922471046448,
      "rewards/margins": 0.12187500298023224,
      "rewards/rejected": 0.08247070014476776,
      "step": 2705
    },
    {
      "epoch": 0.6977342945417095,
      "grad_norm": 102.5,
      "learning_rate": 1.511328527291452e-07,
      "logits/chosen": -0.4066406190395355,
      "logits/rejected": -0.36835938692092896,
      "logps/chosen": -277.79998779296875,
      "logps/rejected": -254.0,
      "loss": 0.6719,
      "rewards/accuracies": 0.5233333706855774,
      "rewards/chosen": 0.24506835639476776,
      "rewards/margins": 0.07386474311351776,
      "rewards/rejected": 0.17128905653953552,
      "step": 2710
    },
    {
      "epoch": 0.6990216271884655,
      "grad_norm": 107.5,
      "learning_rate": 1.5048918640576725e-07,
      "logits/chosen": -0.3821777403354645,
      "logits/rejected": -0.2923828065395355,
      "logps/chosen": -266.6000061035156,
      "logps/rejected": -267.20001220703125,
      "loss": 0.6523,
      "rewards/accuracies": 0.5600000619888306,
      "rewards/chosen": 0.24179688096046448,
      "rewards/margins": 0.10551758110523224,
      "rewards/rejected": 0.13625487685203552,
      "step": 2715
    },
    {
      "epoch": 0.7003089598352215,
      "grad_norm": 109.0,
      "learning_rate": 1.498455200823893e-07,
      "logits/chosen": -0.3597656190395355,
      "logits/rejected": -0.31464844942092896,
      "logps/chosen": -321.6000061035156,
      "logps/rejected": -269.79998779296875,
      "loss": 0.632,
      "rewards/accuracies": 0.5598992705345154,
      "rewards/chosen": 0.330078125,
      "rewards/margins": 0.16376952826976776,
      "rewards/rejected": 0.16621093451976776,
      "step": 2720
    },
    {
      "epoch": 0.7015962924819773,
      "grad_norm": 164.0,
      "learning_rate": 1.492018537590113e-07,
      "logits/chosen": -0.3746093809604645,
      "logits/rejected": -0.392578125,
      "logps/chosen": -295.0,
      "logps/rejected": -236.60000610351562,
      "loss": 0.6586,
      "rewards/accuracies": 0.6241666674613953,
      "rewards/chosen": 0.21499022841453552,
      "rewards/margins": 0.09772948920726776,
      "rewards/rejected": 0.11713866889476776,
      "step": 2725
    },
    {
      "epoch": 0.7028836251287333,
      "grad_norm": 137.0,
      "learning_rate": 1.4855818743563338e-07,
      "logits/chosen": -0.35664063692092896,
      "logits/rejected": -0.3783203065395355,
      "logps/chosen": -296.6000061035156,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6648,
      "rewards/accuracies": 0.5383332967758179,
      "rewards/chosen": 0.24208983778953552,
      "rewards/margins": 0.07656250149011612,
      "rewards/rejected": 0.16547851264476776,
      "step": 2730
    },
    {
      "epoch": 0.7041709577754892,
      "grad_norm": 190.0,
      "learning_rate": 1.4791452111225542e-07,
      "logits/chosen": -0.3677734434604645,
      "logits/rejected": -0.3460937440395355,
      "logps/chosen": -272.3999938964844,
      "logps/rejected": -262.20001220703125,
      "loss": 0.6781,
      "rewards/accuracies": 0.48869046568870544,
      "rewards/chosen": 0.17836913466453552,
      "rewards/margins": 0.06928710639476776,
      "rewards/rejected": 0.10927734524011612,
      "step": 2735
    },
    {
      "epoch": 0.7054582904222451,
      "grad_norm": 108.0,
      "learning_rate": 1.4727085478887743e-07,
      "logits/chosen": -0.3575195372104645,
      "logits/rejected": -0.29218751192092896,
      "logps/chosen": -288.79998779296875,
      "logps/rejected": -248.0,
      "loss": 0.6531,
      "rewards/accuracies": 0.5908333659172058,
      "rewards/chosen": 0.15415039658546448,
      "rewards/margins": 0.09775390475988388,
      "rewards/rejected": 0.05667724460363388,
      "step": 2740
    },
    {
      "epoch": 0.706745623069001,
      "grad_norm": 972.0,
      "learning_rate": 1.4662718846549948e-07,
      "logits/chosen": -0.4300781190395355,
      "logits/rejected": -0.42500001192092896,
      "logps/chosen": -264.0,
      "logps/rejected": -249.39999389648438,
      "loss": 0.6711,
      "rewards/accuracies": 0.5285897850990295,
      "rewards/chosen": 0.2560058534145355,
      "rewards/margins": 0.0567626953125,
      "rewards/rejected": 0.19882813096046448,
      "step": 2745
    },
    {
      "epoch": 0.708032955715757,
      "grad_norm": 110.0,
      "learning_rate": 1.4598352214212152e-07,
      "logits/chosen": -0.4107421934604645,
      "logits/rejected": -0.29472655057907104,
      "logps/chosen": -302.79998779296875,
      "logps/rejected": -283.79998779296875,
      "loss": 0.6695,
      "rewards/accuracies": 0.5781410932540894,
      "rewards/chosen": 0.11777343600988388,
      "rewards/margins": 0.071533203125,
      "rewards/rejected": 0.04631347581744194,
      "step": 2750
    },
    {
      "epoch": 0.7093202883625128,
      "grad_norm": 86.0,
      "learning_rate": 1.4533985581874356e-07,
      "logits/chosen": -0.33818358182907104,
      "logits/rejected": -0.3423828184604645,
      "logps/chosen": -250.0,
      "logps/rejected": -229.0,
      "loss": 0.6406,
      "rewards/accuracies": 0.5940934419631958,
      "rewards/chosen": 0.28515625,
      "rewards/margins": 0.12149658054113388,
      "rewards/rejected": 0.16337890923023224,
      "step": 2755
    },
    {
      "epoch": 0.7106076210092688,
      "grad_norm": 109.0,
      "learning_rate": 1.446961894953656e-07,
      "logits/chosen": -0.24541015923023224,
      "logits/rejected": -0.3134765625,
      "logps/chosen": -254.8000030517578,
      "logps/rejected": -267.20001220703125,
      "loss": 0.6664,
      "rewards/accuracies": 0.4729670584201813,
      "rewards/chosen": 0.1865234375,
      "rewards/margins": 0.06484375149011612,
      "rewards/rejected": 0.12143554538488388,
      "step": 2760
    },
    {
      "epoch": 0.7118949536560247,
      "grad_norm": 167.0,
      "learning_rate": 1.4405252317198762e-07,
      "logits/chosen": -0.2755981385707855,
      "logits/rejected": -0.16835936903953552,
      "logps/chosen": -213.5,
      "logps/rejected": -228.8000030517578,
      "loss": 0.6625,
      "rewards/accuracies": 0.5246428847312927,
      "rewards/chosen": 0.18092040717601776,
      "rewards/margins": 0.09404487907886505,
      "rewards/rejected": 0.08708496391773224,
      "step": 2765
    },
    {
      "epoch": 0.7131822863027807,
      "grad_norm": 114.0,
      "learning_rate": 1.434088568486097e-07,
      "logits/chosen": -0.24208983778953552,
      "logits/rejected": -0.34765625,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -265.3999938964844,
      "loss": 0.6672,
      "rewards/accuracies": 0.5425000190734863,
      "rewards/chosen": 0.22255858778953552,
      "rewards/margins": 0.07829590141773224,
      "rewards/rejected": 0.14448241889476776,
      "step": 2770
    },
    {
      "epoch": 0.7144696189495365,
      "grad_norm": 104.5,
      "learning_rate": 1.4276519052523173e-07,
      "logits/chosen": -0.29248046875,
      "logits/rejected": -0.40166014432907104,
      "logps/chosen": -243.0,
      "logps/rejected": -223.1999969482422,
      "loss": 0.6734,
      "rewards/accuracies": 0.5589286088943481,
      "rewards/chosen": 0.16513672471046448,
      "rewards/margins": 0.06728515774011612,
      "rewards/rejected": 0.09812011569738388,
      "step": 2775
    },
    {
      "epoch": 0.7157569515962925,
      "grad_norm": 127.5,
      "learning_rate": 1.4212152420185375e-07,
      "logits/chosen": -0.3597656190395355,
      "logits/rejected": -0.29570311307907104,
      "logps/chosen": -277.20001220703125,
      "logps/rejected": -229.60000610351562,
      "loss": 0.6359,
      "rewards/accuracies": 0.6525000333786011,
      "rewards/chosen": 0.3228515684604645,
      "rewards/margins": 0.14438477158546448,
      "rewards/rejected": 0.17861327528953552,
      "step": 2780
    },
    {
      "epoch": 0.7170442842430484,
      "grad_norm": 143.0,
      "learning_rate": 1.414778578784758e-07,
      "logits/chosen": -0.31399840116500854,
      "logits/rejected": -0.21406249701976776,
      "logps/chosen": -250.1999969482422,
      "logps/rejected": -271.79998779296875,
      "loss": 0.6773,
      "rewards/accuracies": 0.5006410479545593,
      "rewards/chosen": 0.12595215439796448,
      "rewards/margins": 0.03546752780675888,
      "rewards/rejected": 0.09066162258386612,
      "step": 2785
    },
    {
      "epoch": 0.7183316168898043,
      "grad_norm": 124.5,
      "learning_rate": 1.408341915550978e-07,
      "logits/chosen": -0.3587890565395355,
      "logits/rejected": -0.327606201171875,
      "logps/chosen": -280.3999938964844,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6195,
      "rewards/accuracies": 0.6264743804931641,
      "rewards/chosen": 0.3199218809604645,
      "rewards/margins": 0.17675475776195526,
      "rewards/rejected": 0.14360351860523224,
      "step": 2790
    },
    {
      "epoch": 0.7196189495365602,
      "grad_norm": 142.0,
      "learning_rate": 1.4019052523171988e-07,
      "logits/chosen": -0.40546876192092896,
      "logits/rejected": -0.37421876192092896,
      "logps/chosen": -263.20001220703125,
      "logps/rejected": -267.6000061035156,
      "loss": 0.6648,
      "rewards/accuracies": 0.6008332967758179,
      "rewards/chosen": 0.296875,
      "rewards/margins": 0.09184570610523224,
      "rewards/rejected": 0.20527343451976776,
      "step": 2795
    },
    {
      "epoch": 0.7209062821833162,
      "grad_norm": 125.5,
      "learning_rate": 1.3954685890834192e-07,
      "logits/chosen": -0.36894530057907104,
      "logits/rejected": -0.3104492127895355,
      "logps/chosen": -245.1999969482422,
      "logps/rejected": -273.79998779296875,
      "loss": 0.6477,
      "rewards/accuracies": 0.570402979850769,
      "rewards/chosen": 0.166259765625,
      "rewards/margins": 0.10848388820886612,
      "rewards/rejected": 0.05795898288488388,
      "step": 2800
    },
    {
      "epoch": 0.722193614830072,
      "grad_norm": 97.0,
      "learning_rate": 1.3890319258496394e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.35468751192092896,
      "logps/chosen": -302.79998779296875,
      "logps/rejected": -304.79998779296875,
      "loss": 0.6664,
      "rewards/accuracies": 0.5067857503890991,
      "rewards/chosen": 0.11865234375,
      "rewards/margins": 0.05439453199505806,
      "rewards/rejected": 0.064208984375,
      "step": 2805
    },
    {
      "epoch": 0.723480947476828,
      "grad_norm": 140.0,
      "learning_rate": 1.3825952626158598e-07,
      "logits/chosen": -0.4378906190395355,
      "logits/rejected": -0.46992188692092896,
      "logps/chosen": -261.20001220703125,
      "logps/rejected": -255.0,
      "loss": 0.6664,
      "rewards/accuracies": 0.5175000429153442,
      "rewards/chosen": 0.20166015625,
      "rewards/margins": 0.07181701809167862,
      "rewards/rejected": 0.13015136122703552,
      "step": 2810
    },
    {
      "epoch": 0.724768280123584,
      "grad_norm": 142.0,
      "learning_rate": 1.3761585993820805e-07,
      "logits/chosen": -0.43281251192092896,
      "logits/rejected": -0.4417968690395355,
      "logps/chosen": -312.20001220703125,
      "logps/rejected": -292.0,
      "loss": 0.6484,
      "rewards/accuracies": 0.5774999856948853,
      "rewards/chosen": 0.20878906548023224,
      "rewards/margins": 0.10103912651538849,
      "rewards/rejected": 0.10776367038488388,
      "step": 2815
    },
    {
      "epoch": 0.7260556127703398,
      "grad_norm": 105.5,
      "learning_rate": 1.3697219361483006e-07,
      "logits/chosen": -0.3272460997104645,
      "logits/rejected": -0.3892578184604645,
      "logps/chosen": -245.1999969482422,
      "logps/rejected": -243.8000030517578,
      "loss": 0.6453,
      "rewards/accuracies": 0.6017857193946838,
      "rewards/chosen": 0.27421873807907104,
      "rewards/margins": 0.11738280951976776,
      "rewards/rejected": 0.15683594346046448,
      "step": 2820
    },
    {
      "epoch": 0.7273429454170958,
      "grad_norm": 86.5,
      "learning_rate": 1.363285272914521e-07,
      "logits/chosen": -0.23095703125,
      "logits/rejected": -0.21469727158546448,
      "logps/chosen": -269.0,
      "logps/rejected": -230.8000030517578,
      "loss": 0.6164,
      "rewards/accuracies": 0.6675000190734863,
      "rewards/chosen": 0.34492188692092896,
      "rewards/margins": 0.18691405653953552,
      "rewards/rejected": 0.15878906846046448,
      "step": 2825
    },
    {
      "epoch": 0.7286302780638517,
      "grad_norm": 119.5,
      "learning_rate": 1.3568486096807412e-07,
      "logits/chosen": -0.36796873807907104,
      "logits/rejected": -0.27727049589157104,
      "logps/chosen": -275.79998779296875,
      "logps/rejected": -222.10000610351562,
      "loss": 0.6328,
      "rewards/accuracies": 0.6186563372612,
      "rewards/chosen": 0.3125,
      "rewards/margins": 0.160888671875,
      "rewards/rejected": 0.15131835639476776,
      "step": 2830
    },
    {
      "epoch": 0.7299176107106076,
      "grad_norm": 108.5,
      "learning_rate": 1.350411946446962e-07,
      "logits/chosen": -0.33906251192092896,
      "logits/rejected": -0.4000000059604645,
      "logps/chosen": -264.0,
      "logps/rejected": -261.6000061035156,
      "loss": 0.6477,
      "rewards/accuracies": 0.5725000500679016,
      "rewards/chosen": 0.26123046875,
      "rewards/margins": 0.11003418266773224,
      "rewards/rejected": 0.15107421576976776,
      "step": 2835
    },
    {
      "epoch": 0.7312049433573635,
      "grad_norm": 94.5,
      "learning_rate": 1.3439752832131823e-07,
      "logits/chosen": -0.24980469048023224,
      "logits/rejected": -0.20458984375,
      "logps/chosen": -339.20001220703125,
      "logps/rejected": -278.3999938964844,
      "loss": 0.6305,
      "rewards/accuracies": 0.59416663646698,
      "rewards/chosen": 0.2699218690395355,
      "rewards/margins": 0.15253905951976776,
      "rewards/rejected": 0.11708984524011612,
      "step": 2840
    },
    {
      "epoch": 0.7324922760041195,
      "grad_norm": 148.0,
      "learning_rate": 1.3375386199794025e-07,
      "logits/chosen": -0.2669921815395355,
      "logits/rejected": -0.3949218690395355,
      "logps/chosen": -234.0,
      "logps/rejected": -203.1999969482422,
      "loss": 0.6422,
      "rewards/accuracies": 0.6240367889404297,
      "rewards/chosen": 0.21274414658546448,
      "rewards/margins": 0.11662597954273224,
      "rewards/rejected": 0.09655151516199112,
      "step": 2845
    },
    {
      "epoch": 0.7337796086508754,
      "grad_norm": 95.5,
      "learning_rate": 1.331101956745623e-07,
      "logits/chosen": -0.2552246153354645,
      "logits/rejected": -0.348947137594223,
      "logps/chosen": -261.3999938964844,
      "logps/rejected": -253.39999389648438,
      "loss": 0.6773,
      "rewards/accuracies": 0.43666666746139526,
      "rewards/chosen": 0.14023438096046448,
      "rewards/margins": 0.04013671725988388,
      "rewards/rejected": 0.10000000149011612,
      "step": 2850
    },
    {
      "epoch": 0.7350669412976313,
      "grad_norm": 97.5,
      "learning_rate": 1.3246652935118436e-07,
      "logits/chosen": -0.4134765565395355,
      "logits/rejected": -0.474609375,
      "logps/chosen": -281.20001220703125,
      "logps/rejected": -259.6000061035156,
      "loss": 0.6383,
      "rewards/accuracies": 0.6035256385803223,
      "rewards/chosen": 0.24687500298023224,
      "rewards/margins": 0.13164062798023224,
      "rewards/rejected": 0.11495361477136612,
      "step": 2855
    },
    {
      "epoch": 0.7363542739443872,
      "grad_norm": 133.0,
      "learning_rate": 1.3182286302780638e-07,
      "logits/chosen": -0.2918945252895355,
      "logits/rejected": -0.37128907442092896,
      "logps/chosen": -250.39999389648438,
      "logps/rejected": -218.8000030517578,
      "loss": 0.668,
      "rewards/accuracies": 0.48366522789001465,
      "rewards/chosen": 0.20498046278953552,
      "rewards/margins": 0.06706543266773224,
      "rewards/rejected": 0.13798828423023224,
      "step": 2860
    },
    {
      "epoch": 0.7376416065911432,
      "grad_norm": 114.0,
      "learning_rate": 1.3117919670442842e-07,
      "logits/chosen": -0.3450683653354645,
      "logits/rejected": -0.3075195252895355,
      "logps/chosen": -218.60000610351562,
      "logps/rejected": -214.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.6283333897590637,
      "rewards/chosen": 0.24101562798023224,
      "rewards/margins": 0.09371338039636612,
      "rewards/rejected": 0.14754638075828552,
      "step": 2865
    },
    {
      "epoch": 0.738928939237899,
      "grad_norm": 112.0,
      "learning_rate": 1.3053553038105044e-07,
      "logits/chosen": -0.43828123807907104,
      "logits/rejected": -0.41874998807907104,
      "logps/chosen": -303.79998779296875,
      "logps/rejected": -285.0,
      "loss": 0.6633,
      "rewards/accuracies": 0.559404730796814,
      "rewards/chosen": 0.16542968153953552,
      "rewards/margins": 0.083251953125,
      "rewards/rejected": 0.08188476413488388,
      "step": 2870
    },
    {
      "epoch": 0.740216271884655,
      "grad_norm": 124.5,
      "learning_rate": 1.298918640576725e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.43242186307907104,
      "logps/chosen": -332.20001220703125,
      "logps/rejected": -283.79998779296875,
      "loss": 0.6398,
      "rewards/accuracies": 0.5701190233230591,
      "rewards/chosen": 0.2525390684604645,
      "rewards/margins": 0.12968750298023224,
      "rewards/rejected": 0.12252197414636612,
      "step": 2875
    },
    {
      "epoch": 0.741503604531411,
      "grad_norm": 91.5,
      "learning_rate": 1.2924819773429455e-07,
      "logits/chosen": -0.31230467557907104,
      "logits/rejected": -0.36445313692092896,
      "logps/chosen": -247.1999969482422,
      "logps/rejected": -217.39999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.5431410670280457,
      "rewards/chosen": 0.16635742783546448,
      "rewards/margins": 0.10329589992761612,
      "rewards/rejected": 0.06297607719898224,
      "step": 2880
    },
    {
      "epoch": 0.7427909371781668,
      "grad_norm": 113.5,
      "learning_rate": 1.2860453141091657e-07,
      "logits/chosen": -0.22807617485523224,
      "logits/rejected": -0.17812499403953552,
      "logps/chosen": -216.1999969482422,
      "logps/rejected": -228.1999969482422,
      "loss": 0.6703,
      "rewards/accuracies": 0.6026190519332886,
      "rewards/chosen": 0.19091796875,
      "rewards/margins": 0.05856933444738388,
      "rewards/rejected": 0.13211670517921448,
      "step": 2885
    },
    {
      "epoch": 0.7440782698249228,
      "grad_norm": 91.5,
      "learning_rate": 1.279608650875386e-07,
      "logits/chosen": -0.3968749940395355,
      "logits/rejected": -0.35966795682907104,
      "logps/chosen": -285.20001220703125,
      "logps/rejected": -293.3999938964844,
      "loss": 0.6523,
      "rewards/accuracies": 0.4769230782985687,
      "rewards/chosen": 0.19077149033546448,
      "rewards/margins": 0.10496826469898224,
      "rewards/rejected": 0.08583984524011612,
      "step": 2890
    },
    {
      "epoch": 0.7453656024716787,
      "grad_norm": 126.5,
      "learning_rate": 1.2731719876416068e-07,
      "logits/chosen": -0.21845702826976776,
      "logits/rejected": -0.2884521484375,
      "logps/chosen": -354.0,
      "logps/rejected": -277.0,
      "loss": 0.6523,
      "rewards/accuracies": 0.5035713911056519,
      "rewards/chosen": 0.20703125,
      "rewards/margins": 0.11176757514476776,
      "rewards/rejected": 0.09519042819738388,
      "step": 2895
    },
    {
      "epoch": 0.7466529351184346,
      "grad_norm": 121.5,
      "learning_rate": 1.266735324407827e-07,
      "logits/chosen": -0.4095703065395355,
      "logits/rejected": -0.4097656309604645,
      "logps/chosen": -274.0,
      "logps/rejected": -335.20001220703125,
      "loss": 0.6547,
      "rewards/accuracies": 0.4925000071525574,
      "rewards/chosen": 0.21894530951976776,
      "rewards/margins": 0.09208984673023224,
      "rewards/rejected": 0.12656250596046448,
      "step": 2900
    },
    {
      "epoch": 0.7479402677651905,
      "grad_norm": 96.0,
      "learning_rate": 1.2602986611740474e-07,
      "logits/chosen": -0.31621092557907104,
      "logits/rejected": -0.37421876192092896,
      "logps/chosen": -299.3999938964844,
      "logps/rejected": -241.60000610351562,
      "loss": 0.6453,
      "rewards/accuracies": 0.5975000262260437,
      "rewards/chosen": 0.18925781548023224,
      "rewards/margins": 0.118896484375,
      "rewards/rejected": 0.07050780951976776,
      "step": 2905
    },
    {
      "epoch": 0.7492276004119465,
      "grad_norm": 121.5,
      "learning_rate": 1.2538619979402675e-07,
      "logits/chosen": -0.3037109375,
      "logits/rejected": -0.06994018703699112,
      "logps/chosen": -256.3999938964844,
      "logps/rejected": -218.0,
      "loss": 0.6297,
      "rewards/accuracies": 0.5465368628501892,
      "rewards/chosen": 0.17304687201976776,
      "rewards/margins": 0.1513671875,
      "rewards/rejected": 0.02158203162252903,
      "step": 2910
    },
    {
      "epoch": 0.7505149330587023,
      "grad_norm": 86.5,
      "learning_rate": 1.247425334706488e-07,
      "logits/chosen": -0.4625000059604645,
      "logits/rejected": -0.3526367247104645,
      "logps/chosen": -232.60000610351562,
      "logps/rejected": -233.8000030517578,
      "loss": 0.6609,
      "rewards/accuracies": 0.5573077201843262,
      "rewards/chosen": 0.20624999701976776,
      "rewards/margins": 0.08945312350988388,
      "rewards/rejected": 0.11679687350988388,
      "step": 2915
    },
    {
      "epoch": 0.7518022657054583,
      "grad_norm": 117.0,
      "learning_rate": 1.2409886714727084e-07,
      "logits/chosen": -0.41191405057907104,
      "logits/rejected": -0.3681640625,
      "logps/chosen": -312.0,
      "logps/rejected": -278.6000061035156,
      "loss": 0.6391,
      "rewards/accuracies": 0.6025000214576721,
      "rewards/chosen": 0.2733398377895355,
      "rewards/margins": 0.1417236328125,
      "rewards/rejected": 0.13137206435203552,
      "step": 2920
    },
    {
      "epoch": 0.7530895983522142,
      "grad_norm": 112.5,
      "learning_rate": 1.2345520082389288e-07,
      "logits/chosen": -0.2640624940395355,
      "logits/rejected": -0.29472655057907104,
      "logps/chosen": -312.20001220703125,
      "logps/rejected": -276.3999938964844,
      "loss": 0.6539,
      "rewards/accuracies": 0.514285683631897,
      "rewards/chosen": 0.3026367127895355,
      "rewards/margins": 0.11372070014476776,
      "rewards/rejected": 0.18902587890625,
      "step": 2925
    },
    {
      "epoch": 0.7543769309989702,
      "grad_norm": 121.0,
      "learning_rate": 1.2281153450051492e-07,
      "logits/chosen": -0.31904298067092896,
      "logits/rejected": -0.22685547173023224,
      "logps/chosen": -292.3999938964844,
      "logps/rejected": -269.6000061035156,
      "loss": 0.6562,
      "rewards/accuracies": 0.47261905670166016,
      "rewards/chosen": 0.1094970703125,
      "rewards/margins": 0.08124999701976776,
      "rewards/rejected": 0.02840576134622097,
      "step": 2930
    },
    {
      "epoch": 0.755664263645726,
      "grad_norm": 93.5,
      "learning_rate": 1.2216786817713697e-07,
      "logits/chosen": -0.47539061307907104,
      "logits/rejected": -0.41484373807907104,
      "logps/chosen": -290.6000061035156,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6336,
      "rewards/accuracies": 0.6171093583106995,
      "rewards/chosen": 0.19428710639476776,
      "rewards/margins": 0.134521484375,
      "rewards/rejected": 0.05986328050494194,
      "step": 2935
    },
    {
      "epoch": 0.756951596292482,
      "grad_norm": 140.0,
      "learning_rate": 1.21524201853759e-07,
      "logits/chosen": -0.36445313692092896,
      "logits/rejected": -0.2919921875,
      "logps/chosen": -320.79998779296875,
      "logps/rejected": -281.79998779296875,
      "loss": 0.65,
      "rewards/accuracies": 0.5436905026435852,
      "rewards/chosen": 0.19638672471046448,
      "rewards/margins": 0.11396484076976776,
      "rewards/rejected": 0.08244629204273224,
      "step": 2940
    },
    {
      "epoch": 0.7582389289392379,
      "grad_norm": 161.0,
      "learning_rate": 1.2088053553038105e-07,
      "logits/chosen": -0.3306640684604645,
      "logits/rejected": -0.39433592557907104,
      "logps/chosen": -280.20001220703125,
      "logps/rejected": -254.8000030517578,
      "loss": 0.6773,
      "rewards/accuracies": 0.546107292175293,
      "rewards/chosen": 0.20839843153953552,
      "rewards/margins": 0.05217285081744194,
      "rewards/rejected": 0.15571288764476776,
      "step": 2945
    },
    {
      "epoch": 0.7595262615859938,
      "grad_norm": 83.5,
      "learning_rate": 1.202368692070031e-07,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.23129883408546448,
      "logps/chosen": -245.1999969482422,
      "logps/rejected": -224.8000030517578,
      "loss": 0.6273,
      "rewards/accuracies": 0.6901190876960754,
      "rewards/chosen": 0.22607421875,
      "rewards/margins": 0.16523437201976776,
      "rewards/rejected": 0.06102294847369194,
      "step": 2950
    },
    {
      "epoch": 0.7608135942327497,
      "grad_norm": 127.5,
      "learning_rate": 1.195932028836251e-07,
      "logits/chosen": -0.3082031309604645,
      "logits/rejected": -0.2659668028354645,
      "logps/chosen": -284.20001220703125,
      "logps/rejected": -257.0,
      "loss": 0.6539,
      "rewards/accuracies": 0.5040476322174072,
      "rewards/chosen": 0.21542969346046448,
      "rewards/margins": 0.10346679389476776,
      "rewards/rejected": 0.11223144829273224,
      "step": 2955
    },
    {
      "epoch": 0.7621009268795057,
      "grad_norm": 127.0,
      "learning_rate": 1.1894953656024715e-07,
      "logits/chosen": -0.35273438692092896,
      "logits/rejected": -0.2928222715854645,
      "logps/chosen": -234.0,
      "logps/rejected": -249.60000610351562,
      "loss": 0.657,
      "rewards/accuracies": 0.5471428632736206,
      "rewards/chosen": 0.24609375,
      "rewards/margins": 0.09055481106042862,
      "rewards/rejected": 0.15585938096046448,
      "step": 2960
    },
    {
      "epoch": 0.7633882595262615,
      "grad_norm": 117.5,
      "learning_rate": 1.1830587023686921e-07,
      "logits/chosen": -0.47773438692092896,
      "logits/rejected": -0.4623046815395355,
      "logps/chosen": -286.20001220703125,
      "logps/rejected": -247.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.47654762864112854,
      "rewards/chosen": 0.10356445610523224,
      "rewards/margins": 0.034912109375,
      "rewards/rejected": 0.06862793117761612,
      "step": 2965
    },
    {
      "epoch": 0.7646755921730175,
      "grad_norm": 87.5,
      "learning_rate": 1.1766220391349124e-07,
      "logits/chosen": -0.14077147841453552,
      "logits/rejected": -0.15478515625,
      "logps/chosen": -310.79998779296875,
      "logps/rejected": -218.60000610351562,
      "loss": 0.6594,
      "rewards/accuracies": 0.5240384340286255,
      "rewards/chosen": 0.21992187201976776,
      "rewards/margins": 0.10657958686351776,
      "rewards/rejected": 0.1134033203125,
      "step": 2970
    },
    {
      "epoch": 0.7659629248197735,
      "grad_norm": 115.5,
      "learning_rate": 1.1701853759011328e-07,
      "logits/chosen": -0.28857421875,
      "logits/rejected": -0.2978515625,
      "logps/chosen": -300.3999938964844,
      "logps/rejected": -263.6000061035156,
      "loss": 0.6719,
      "rewards/accuracies": 0.4958333373069763,
      "rewards/chosen": 0.23691406846046448,
      "rewards/margins": 0.07182617485523224,
      "rewards/rejected": 0.16562500596046448,
      "step": 2975
    },
    {
      "epoch": 0.7672502574665293,
      "grad_norm": 95.0,
      "learning_rate": 1.1637487126673531e-07,
      "logits/chosen": -0.36040037870407104,
      "logits/rejected": -0.39960938692092896,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -242.39999389648438,
      "loss": 0.6367,
      "rewards/accuracies": 0.59416663646698,
      "rewards/chosen": 0.22065429389476776,
      "rewards/margins": 0.14101561903953552,
      "rewards/rejected": 0.07956542819738388,
      "step": 2980
    },
    {
      "epoch": 0.7685375901132853,
      "grad_norm": 110.5,
      "learning_rate": 1.1573120494335737e-07,
      "logits/chosen": -0.25056153535842896,
      "logits/rejected": -0.3736328184604645,
      "logps/chosen": -277.6000061035156,
      "logps/rejected": -257.20001220703125,
      "loss": 0.6602,
      "rewards/accuracies": 0.4266666769981384,
      "rewards/chosen": 0.14716796576976776,
      "rewards/margins": 0.0927734375,
      "rewards/rejected": 0.05433959886431694,
      "step": 2985
    },
    {
      "epoch": 0.7698249227600412,
      "grad_norm": 118.5,
      "learning_rate": 1.150875386199794e-07,
      "logits/chosen": -0.3514648377895355,
      "logits/rejected": -0.35761719942092896,
      "logps/chosen": -353.6000061035156,
      "logps/rejected": -289.20001220703125,
      "loss": 0.6633,
      "rewards/accuracies": 0.44564104080200195,
      "rewards/chosen": 0.11152343451976776,
      "rewards/margins": 0.07426758110523224,
      "rewards/rejected": 0.03732910007238388,
      "step": 2990
    },
    {
      "epoch": 0.7711122554067971,
      "grad_norm": 133.0,
      "learning_rate": 1.1444387229660144e-07,
      "logits/chosen": -0.3935546875,
      "logits/rejected": -0.31562501192092896,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -253.60000610351562,
      "loss": 0.657,
      "rewards/accuracies": 0.5543939471244812,
      "rewards/chosen": 0.15122070908546448,
      "rewards/margins": 0.08808593451976776,
      "rewards/rejected": 0.06317138671875,
      "step": 2995
    },
    {
      "epoch": 0.772399588053553,
      "grad_norm": 129.0,
      "learning_rate": 1.1380020597322347e-07,
      "logits/chosen": -0.36406248807907104,
      "logits/rejected": -0.3095703125,
      "logps/chosen": -362.3999938964844,
      "logps/rejected": -310.20001220703125,
      "loss": 0.6508,
      "rewards/accuracies": 0.5279762148857117,
      "rewards/chosen": 0.20156249403953552,
      "rewards/margins": 0.10678710788488388,
      "rewards/rejected": 0.09477539360523224,
      "step": 3000
    },
    {
      "epoch": 0.773686920700309,
      "grad_norm": 129.0,
      "learning_rate": 1.1315653964984552e-07,
      "logits/chosen": -0.36796873807907104,
      "logits/rejected": -0.35234373807907104,
      "logps/chosen": -332.3999938964844,
      "logps/rejected": -270.79998779296875,
      "loss": 0.6281,
      "rewards/accuracies": 0.6366666555404663,
      "rewards/chosen": 0.23125000298023224,
      "rewards/margins": 0.13520507514476776,
      "rewards/rejected": 0.0958099365234375,
      "step": 3005
    },
    {
      "epoch": 0.7749742533470649,
      "grad_norm": 1120.0,
      "learning_rate": 1.1251287332646755e-07,
      "logits/chosen": -0.22382812201976776,
      "logits/rejected": -0.07597656548023224,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -223.1999969482422,
      "loss": 0.6742,
      "rewards/accuracies": 0.5951923131942749,
      "rewards/chosen": 0.14404296875,
      "rewards/margins": 0.07480468600988388,
      "rewards/rejected": 0.06926269829273224,
      "step": 3010
    },
    {
      "epoch": 0.7762615859938208,
      "grad_norm": 149.0,
      "learning_rate": 1.118692070030896e-07,
      "logits/chosen": -0.38749998807907104,
      "logits/rejected": -0.304443359375,
      "logps/chosen": -307.79998779296875,
      "logps/rejected": -278.20001220703125,
      "loss": 0.6656,
      "rewards/accuracies": 0.5049999952316284,
      "rewards/chosen": 0.18818359076976776,
      "rewards/margins": 0.07924804836511612,
      "rewards/rejected": 0.10893554985523224,
      "step": 3015
    },
    {
      "epoch": 0.7775489186405767,
      "grad_norm": 103.5,
      "learning_rate": 1.1122554067971163e-07,
      "logits/chosen": -0.5640624761581421,
      "logits/rejected": -0.4878906309604645,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -207.1999969482422,
      "loss": 0.6508,
      "rewards/accuracies": 0.5623809695243835,
      "rewards/chosen": 0.22832031548023224,
      "rewards/margins": 0.1070556640625,
      "rewards/rejected": 0.12131347507238388,
      "step": 3020
    },
    {
      "epoch": 0.7788362512873327,
      "grad_norm": 124.0,
      "learning_rate": 1.1058187435633368e-07,
      "logits/chosen": -0.3929687440395355,
      "logits/rejected": -0.3734374940395355,
      "logps/chosen": -304.20001220703125,
      "logps/rejected": -258.20001220703125,
      "loss": 0.65,
      "rewards/accuracies": 0.6206411123275757,
      "rewards/chosen": 0.20068359375,
      "rewards/margins": 0.11016845703125,
      "rewards/rejected": 0.09070968627929688,
      "step": 3025
    },
    {
      "epoch": 0.7801235839340885,
      "grad_norm": 111.0,
      "learning_rate": 1.0993820803295571e-07,
      "logits/chosen": -0.37578123807907104,
      "logits/rejected": -0.4097656309604645,
      "logps/chosen": -286.6000061035156,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6477,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.21987304091453552,
      "rewards/margins": 0.116943359375,
      "rewards/rejected": 0.10336913913488388,
      "step": 3030
    },
    {
      "epoch": 0.7814109165808445,
      "grad_norm": 156.0,
      "learning_rate": 1.0929454170957775e-07,
      "logits/chosen": -0.29609376192092896,
      "logits/rejected": -0.262939453125,
      "logps/chosen": -272.6000061035156,
      "logps/rejected": -224.39999389648438,
      "loss": 0.6305,
      "rewards/accuracies": 0.5850000381469727,
      "rewards/chosen": 0.25019532442092896,
      "rewards/margins": 0.16416016221046448,
      "rewards/rejected": 0.0859375,
      "step": 3035
    },
    {
      "epoch": 0.7826982492276005,
      "grad_norm": 98.5,
      "learning_rate": 1.0865087538619978e-07,
      "logits/chosen": -0.28813475370407104,
      "logits/rejected": -0.3785156309604645,
      "logps/chosen": -204.39999389648438,
      "logps/rejected": -231.1999969482422,
      "loss": 0.6578,
      "rewards/accuracies": 0.6758333444595337,
      "rewards/chosen": 0.2865234315395355,
      "rewards/margins": 0.09819336235523224,
      "rewards/rejected": 0.1884765625,
      "step": 3040
    },
    {
      "epoch": 0.7839855818743563,
      "grad_norm": 150.0,
      "learning_rate": 1.0800720906282184e-07,
      "logits/chosen": -0.39238280057907104,
      "logits/rejected": -0.4375,
      "logps/chosen": -254.60000610351562,
      "logps/rejected": -275.0,
      "loss": 0.6711,
      "rewards/accuracies": 0.5436111688613892,
      "rewards/chosen": 0.20405273139476776,
      "rewards/margins": 0.0579833984375,
      "rewards/rejected": 0.14626464247703552,
      "step": 3045
    },
    {
      "epoch": 0.7852729145211123,
      "grad_norm": 182.0,
      "learning_rate": 1.0736354273944387e-07,
      "logits/chosen": -0.3404296934604645,
      "logits/rejected": -0.18203124403953552,
      "logps/chosen": -218.0,
      "logps/rejected": -220.89999389648438,
      "loss": 0.6562,
      "rewards/accuracies": 0.6112662553787231,
      "rewards/chosen": 0.17529296875,
      "rewards/margins": 0.08476562798023224,
      "rewards/rejected": 0.09052734076976776,
      "step": 3050
    },
    {
      "epoch": 0.7865602471678682,
      "grad_norm": 102.0,
      "learning_rate": 1.067198764160659e-07,
      "logits/chosen": -0.3720703125,
      "logits/rejected": -0.421875,
      "logps/chosen": -332.0,
      "logps/rejected": -351.6000061035156,
      "loss": 0.6531,
      "rewards/accuracies": 0.5426281690597534,
      "rewards/chosen": 0.1591796875,
      "rewards/margins": 0.10122070461511612,
      "rewards/rejected": 0.05795898288488388,
      "step": 3055
    },
    {
      "epoch": 0.787847579814624,
      "grad_norm": 124.5,
      "learning_rate": 1.0607621009268794e-07,
      "logits/chosen": -0.5355468988418579,
      "logits/rejected": -0.431640625,
      "logps/chosen": -342.79998779296875,
      "logps/rejected": -294.6000061035156,
      "loss": 0.6555,
      "rewards/accuracies": 0.49988096952438354,
      "rewards/chosen": 0.2138671875,
      "rewards/margins": 0.10549316555261612,
      "rewards/rejected": 0.10771484673023224,
      "step": 3060
    },
    {
      "epoch": 0.78913491246138,
      "grad_norm": 162.0,
      "learning_rate": 1.0543254376930998e-07,
      "logits/chosen": -0.32499998807907104,
      "logits/rejected": -0.3521484434604645,
      "logps/chosen": -244.8000030517578,
      "logps/rejected": -236.60000610351562,
      "loss": 0.6469,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.29560548067092896,
      "rewards/margins": 0.10546875,
      "rewards/rejected": 0.18974609673023224,
      "step": 3065
    },
    {
      "epoch": 0.790422245108136,
      "grad_norm": 117.5,
      "learning_rate": 1.0478887744593203e-07,
      "logits/chosen": -0.35820311307907104,
      "logits/rejected": -0.3384765684604645,
      "logps/chosen": -307.20001220703125,
      "logps/rejected": -251.60000610351562,
      "loss": 0.657,
      "rewards/accuracies": 0.5307692289352417,
      "rewards/chosen": 0.13427734375,
      "rewards/margins": 0.08478393405675888,
      "rewards/rejected": 0.04948730394244194,
      "step": 3070
    },
    {
      "epoch": 0.7917095777548918,
      "grad_norm": 106.0,
      "learning_rate": 1.0414521112255406e-07,
      "logits/chosen": -0.2982421815395355,
      "logits/rejected": -0.2574218809604645,
      "logps/chosen": -272.6000061035156,
      "logps/rejected": -234.60000610351562,
      "loss": 0.6656,
      "rewards/accuracies": 0.49965038895606995,
      "rewards/chosen": 0.17387695610523224,
      "rewards/margins": 0.06660155951976776,
      "rewards/rejected": 0.107177734375,
      "step": 3075
    },
    {
      "epoch": 0.7929969104016478,
      "grad_norm": 209.0,
      "learning_rate": 1.035015447991761e-07,
      "logits/chosen": -0.2919921875,
      "logits/rejected": -0.23359374701976776,
      "logps/chosen": -238.39999389648438,
      "logps/rejected": -253.8000030517578,
      "loss": 0.6508,
      "rewards/accuracies": 0.6004945039749146,
      "rewards/chosen": 0.23906250298023224,
      "rewards/margins": 0.103515625,
      "rewards/rejected": 0.13579101860523224,
      "step": 3080
    },
    {
      "epoch": 0.7942842430484037,
      "grad_norm": 101.0,
      "learning_rate": 1.0285787847579814e-07,
      "logits/chosen": -0.2953124940395355,
      "logits/rejected": -0.34296876192092896,
      "logps/chosen": -291.3999938964844,
      "logps/rejected": -278.79998779296875,
      "loss": 0.6711,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.20512695610523224,
      "rewards/margins": 0.06186523288488388,
      "rewards/rejected": 0.14321288466453552,
      "step": 3085
    },
    {
      "epoch": 0.7955715756951597,
      "grad_norm": 149.0,
      "learning_rate": 1.0221421215242018e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.41914063692092896,
      "logps/chosen": -325.6000061035156,
      "logps/rejected": -282.0,
      "loss": 0.6297,
      "rewards/accuracies": 0.5727564096450806,
      "rewards/chosen": 0.30937498807907104,
      "rewards/margins": 0.15834960341453552,
      "rewards/rejected": 0.151123046875,
      "step": 3090
    },
    {
      "epoch": 0.7968589083419155,
      "grad_norm": 102.5,
      "learning_rate": 1.0157054582904221e-07,
      "logits/chosen": -0.21612548828125,
      "logits/rejected": -0.33574217557907104,
      "logps/chosen": -210.1999969482422,
      "logps/rejected": -213.60000610351562,
      "loss": 0.6367,
      "rewards/accuracies": 0.6491774320602417,
      "rewards/chosen": 0.15834960341453552,
      "rewards/margins": 0.13405761122703552,
      "rewards/rejected": 0.02457275427877903,
      "step": 3095
    },
    {
      "epoch": 0.7981462409886715,
      "grad_norm": 225.0,
      "learning_rate": 1.0092687950566426e-07,
      "logits/chosen": -0.2723632752895355,
      "logits/rejected": -0.28437501192092896,
      "logps/chosen": -248.1999969482422,
      "logps/rejected": -212.6999969482422,
      "loss": 0.6461,
      "rewards/accuracies": 0.5592857599258423,
      "rewards/chosen": 0.18447265028953552,
      "rewards/margins": 0.11406250298023224,
      "rewards/rejected": 0.0704193115234375,
      "step": 3100
    },
    {
      "epoch": 0.7994335736354274,
      "grad_norm": 408.0,
      "learning_rate": 1.002832131822863e-07,
      "logits/chosen": -0.13701172173023224,
      "logits/rejected": -0.15390625596046448,
      "logps/chosen": -250.1999969482422,
      "logps/rejected": -247.0,
      "loss": 0.7031,
      "rewards/accuracies": 0.41654762625694275,
      "rewards/chosen": 0.09672851860523224,
      "rewards/margins": 6.10351571594947e-06,
      "rewards/rejected": 0.09648437798023224,
      "step": 3105
    },
    {
      "epoch": 0.8007209062821833,
      "grad_norm": 135.0,
      "learning_rate": 9.963954685890834e-08,
      "logits/chosen": -0.27070313692092896,
      "logits/rejected": -0.29414063692092896,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -269.79998779296875,
      "loss": 0.6516,
      "rewards/accuracies": 0.6058333516120911,
      "rewards/chosen": 0.23554687201976776,
      "rewards/margins": 0.10454101860523224,
      "rewards/rejected": 0.13105468451976776,
      "step": 3110
    },
    {
      "epoch": 0.8020082389289392,
      "grad_norm": 122.5,
      "learning_rate": 9.899588053553037e-08,
      "logits/chosen": -0.21865233778953552,
      "logits/rejected": -0.14696045219898224,
      "logps/chosen": -224.0,
      "logps/rejected": -206.1999969482422,
      "loss": 0.6633,
      "rewards/accuracies": 0.5557359457015991,
      "rewards/chosen": 0.20498046278953552,
      "rewards/margins": 0.07805176079273224,
      "rewards/rejected": 0.1265869140625,
      "step": 3115
    },
    {
      "epoch": 0.8032955715756952,
      "grad_norm": 140.0,
      "learning_rate": 9.835221421215241e-08,
      "logits/chosen": -0.3294921815395355,
      "logits/rejected": -0.2509765625,
      "logps/chosen": -252.1999969482422,
      "logps/rejected": -246.0,
      "loss": 0.6672,
      "rewards/accuracies": 0.44083333015441895,
      "rewards/chosen": 0.16572265326976776,
      "rewards/margins": 0.05622558668255806,
      "rewards/rejected": 0.10915527492761612,
      "step": 3120
    },
    {
      "epoch": 0.804582904222451,
      "grad_norm": 274.0,
      "learning_rate": 9.770854788877446e-08,
      "logits/chosen": -0.3239753842353821,
      "logits/rejected": -0.24946899712085724,
      "logps/chosen": -258.6000061035156,
      "logps/rejected": -246.0,
      "loss": 0.6406,
      "rewards/accuracies": 0.6597222089767456,
      "rewards/chosen": 0.25108641386032104,
      "rewards/margins": 0.13857421278953552,
      "rewards/rejected": 0.11281738430261612,
      "step": 3125
    },
    {
      "epoch": 0.805870236869207,
      "grad_norm": 89.0,
      "learning_rate": 9.70648815653965e-08,
      "logits/chosen": -0.19892577826976776,
      "logits/rejected": -0.2998046875,
      "logps/chosen": -320.0,
      "logps/rejected": -281.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.5866667032241821,
      "rewards/chosen": 0.2962402403354645,
      "rewards/margins": 0.13193359971046448,
      "rewards/rejected": 0.16452637314796448,
      "step": 3130
    },
    {
      "epoch": 0.807157569515963,
      "grad_norm": 125.5,
      "learning_rate": 9.642121524201853e-08,
      "logits/chosen": -0.3603515625,
      "logits/rejected": -0.3228515684604645,
      "logps/chosen": -302.3999938964844,
      "logps/rejected": -279.79998779296875,
      "loss": 0.6523,
      "rewards/accuracies": 0.6183333396911621,
      "rewards/chosen": 0.2740234434604645,
      "rewards/margins": 0.120361328125,
      "rewards/rejected": 0.15349121391773224,
      "step": 3135
    },
    {
      "epoch": 0.8084449021627188,
      "grad_norm": 458.0,
      "learning_rate": 9.577754891864057e-08,
      "logits/chosen": -0.4312500059604645,
      "logits/rejected": -0.3841796815395355,
      "logps/chosen": -268.3999938964844,
      "logps/rejected": -257.3999938964844,
      "loss": 0.6695,
      "rewards/accuracies": 0.5883333683013916,
      "rewards/chosen": 0.213134765625,
      "rewards/margins": 0.06669006496667862,
      "rewards/rejected": 0.14667968451976776,
      "step": 3140
    },
    {
      "epoch": 0.8097322348094748,
      "grad_norm": 109.0,
      "learning_rate": 9.513388259526261e-08,
      "logits/chosen": -0.15500488877296448,
      "logits/rejected": -0.23720702528953552,
      "logps/chosen": -221.39999389648438,
      "logps/rejected": -196.1999969482422,
      "loss": 0.6602,
      "rewards/accuracies": 0.5046428442001343,
      "rewards/chosen": 0.18037109076976776,
      "rewards/margins": 0.07864685356616974,
      "rewards/rejected": 0.10197754204273224,
      "step": 3145
    },
    {
      "epoch": 0.8110195674562307,
      "grad_norm": 97.0,
      "learning_rate": 9.449021627188466e-08,
      "logits/chosen": -0.20878906548023224,
      "logits/rejected": -0.06484375149011612,
      "logps/chosen": -276.54998779296875,
      "logps/rejected": -217.8000030517578,
      "loss": 0.6594,
      "rewards/accuracies": 0.5641666650772095,
      "rewards/chosen": 0.20458984375,
      "rewards/margins": 0.09394530951976776,
      "rewards/rejected": 0.1107177734375,
      "step": 3150
    },
    {
      "epoch": 0.8123069001029866,
      "grad_norm": 150.0,
      "learning_rate": 9.384654994850669e-08,
      "logits/chosen": -0.3072265684604645,
      "logits/rejected": -0.3687500059604645,
      "logps/chosen": -282.20001220703125,
      "logps/rejected": -262.20001220703125,
      "loss": 0.6633,
      "rewards/accuracies": 0.5067948698997498,
      "rewards/chosen": 0.18222656846046448,
      "rewards/margins": 0.07912597805261612,
      "rewards/rejected": 0.103271484375,
      "step": 3155
    },
    {
      "epoch": 0.8135942327497425,
      "grad_norm": 448.0,
      "learning_rate": 9.320288362512873e-08,
      "logits/chosen": -0.42973631620407104,
      "logits/rejected": -0.462890625,
      "logps/chosen": -239.39999389648438,
      "logps/rejected": -273.20001220703125,
      "loss": 0.6641,
      "rewards/accuracies": 0.5366666913032532,
      "rewards/chosen": 0.1513671875,
      "rewards/margins": 0.07673339545726776,
      "rewards/rejected": 0.07447509467601776,
      "step": 3160
    },
    {
      "epoch": 0.8148815653964985,
      "grad_norm": 192.0,
      "learning_rate": 9.255921730175077e-08,
      "logits/chosen": -0.3626953065395355,
      "logits/rejected": -0.30546873807907104,
      "logps/chosen": -258.79998779296875,
      "logps/rejected": -242.8000030517578,
      "loss": 0.6625,
      "rewards/accuracies": 0.5674999952316284,
      "rewards/chosen": 0.18022461235523224,
      "rewards/margins": 0.08652343600988388,
      "rewards/rejected": 0.09369812160730362,
      "step": 3165
    },
    {
      "epoch": 0.8161688980432544,
      "grad_norm": 97.0,
      "learning_rate": 9.191555097837281e-08,
      "logits/chosen": -0.4710937440395355,
      "logits/rejected": -0.3773437440395355,
      "logps/chosen": -288.79998779296875,
      "logps/rejected": -232.0,
      "loss": 0.6773,
      "rewards/accuracies": 0.4914286136627197,
      "rewards/chosen": 0.157958984375,
      "rewards/margins": 0.04862060397863388,
      "rewards/rejected": 0.10917969048023224,
      "step": 3170
    },
    {
      "epoch": 0.8174562306900103,
      "grad_norm": 314.0,
      "learning_rate": 9.127188465499484e-08,
      "logits/chosen": -0.34882813692092896,
      "logits/rejected": -0.23886719346046448,
      "logps/chosen": -312.79998779296875,
      "logps/rejected": -222.1999969482422,
      "loss": 0.6336,
      "rewards/accuracies": 0.5958333015441895,
      "rewards/chosen": 0.2920898497104645,
      "rewards/margins": 0.16445311903953552,
      "rewards/rejected": 0.1274513304233551,
      "step": 3175
    },
    {
      "epoch": 0.8187435633367662,
      "grad_norm": 147.0,
      "learning_rate": 9.062821833161689e-08,
      "logits/chosen": -0.3511718809604645,
      "logits/rejected": -0.38701170682907104,
      "logps/chosen": -281.6000061035156,
      "logps/rejected": -271.3999938964844,
      "loss": 0.6984,
      "rewards/accuracies": 0.4413919448852539,
      "rewards/chosen": 0.15927734971046448,
      "rewards/margins": 0.01552734337747097,
      "rewards/rejected": 0.14374999701976776,
      "step": 3180
    },
    {
      "epoch": 0.8200308959835222,
      "grad_norm": 102.0,
      "learning_rate": 8.998455200823893e-08,
      "logits/chosen": -0.27363282442092896,
      "logits/rejected": -0.36860352754592896,
      "logps/chosen": -261.3999938964844,
      "logps/rejected": -208.60000610351562,
      "loss": 0.6477,
      "rewards/accuracies": 0.5429196357727051,
      "rewards/chosen": 0.21796874701976776,
      "rewards/margins": 0.116455078125,
      "rewards/rejected": 0.10129394382238388,
      "step": 3185
    },
    {
      "epoch": 0.821318228630278,
      "grad_norm": 114.0,
      "learning_rate": 8.934088568486097e-08,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.40839844942092896,
      "logps/chosen": -283.0,
      "logps/rejected": -233.39999389648438,
      "loss": 0.6672,
      "rewards/accuracies": 0.5275000333786011,
      "rewards/chosen": 0.19454345107078552,
      "rewards/margins": 0.06923828274011612,
      "rewards/rejected": 0.12543945014476776,
      "step": 3190
    },
    {
      "epoch": 0.822605561277034,
      "grad_norm": 131.0,
      "learning_rate": 8.8697219361483e-08,
      "logits/chosen": -0.435546875,
      "logits/rejected": -0.4820312559604645,
      "logps/chosen": -299.0,
      "logps/rejected": -254.0,
      "loss": 0.6641,
      "rewards/accuracies": 0.533214271068573,
      "rewards/chosen": 0.13037109375,
      "rewards/margins": 0.07563476264476776,
      "rewards/rejected": 0.05487060546875,
      "step": 3195
    },
    {
      "epoch": 0.82389289392379,
      "grad_norm": 152.0,
      "learning_rate": 8.805355303810503e-08,
      "logits/chosen": -0.24472656846046448,
      "logits/rejected": -0.36113280057907104,
      "logps/chosen": -279.0,
      "logps/rejected": -262.79998779296875,
      "loss": 0.6531,
      "rewards/accuracies": 0.5782575607299805,
      "rewards/chosen": 0.21875,
      "rewards/margins": 0.09072265774011612,
      "rewards/rejected": 0.12822265923023224,
      "step": 3200
    },
    {
      "epoch": 0.8251802265705458,
      "grad_norm": 118.5,
      "learning_rate": 8.740988671472709e-08,
      "logits/chosen": -0.38671875,
      "logits/rejected": -0.3095703125,
      "logps/chosen": -271.0,
      "logps/rejected": -262.79998779296875,
      "loss": 0.6727,
      "rewards/accuracies": 0.4898809492588043,
      "rewards/chosen": 0.24277344346046448,
      "rewards/margins": 0.06797180324792862,
      "rewards/rejected": 0.17509765923023224,
      "step": 3205
    },
    {
      "epoch": 0.8264675592173018,
      "grad_norm": 106.5,
      "learning_rate": 8.676622039134912e-08,
      "logits/chosen": -0.3921875059604645,
      "logits/rejected": -0.36186522245407104,
      "logps/chosen": -332.0,
      "logps/rejected": -288.20001220703125,
      "loss": 0.6555,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.23549804091453552,
      "rewards/margins": 0.1004638671875,
      "rewards/rejected": 0.13564452528953552,
      "step": 3210
    },
    {
      "epoch": 0.8277548918640577,
      "grad_norm": 172.0,
      "learning_rate": 8.612255406797116e-08,
      "logits/chosen": -0.27021485567092896,
      "logits/rejected": -0.23515625298023224,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -226.0,
      "loss": 0.6359,
      "rewards/accuracies": 0.6408333778381348,
      "rewards/chosen": 0.23779296875,
      "rewards/margins": 0.137451171875,
      "rewards/rejected": 0.10017089545726776,
      "step": 3215
    },
    {
      "epoch": 0.8290422245108136,
      "grad_norm": 111.0,
      "learning_rate": 8.547888774459319e-08,
      "logits/chosen": -0.3309570252895355,
      "logits/rejected": -0.31640625,
      "logps/chosen": -251.1999969482422,
      "logps/rejected": -217.1999969482422,
      "loss": 0.675,
      "rewards/accuracies": 0.5150583386421204,
      "rewards/chosen": 0.12109375,
      "rewards/margins": 0.044097900390625,
      "rewards/rejected": 0.0770263671875,
      "step": 3220
    },
    {
      "epoch": 0.8303295571575695,
      "grad_norm": 153.0,
      "learning_rate": 8.483522142121524e-08,
      "logits/chosen": -0.3402343690395355,
      "logits/rejected": -0.2500976622104645,
      "logps/chosen": -269.0,
      "logps/rejected": -237.60000610351562,
      "loss": 0.6453,
      "rewards/accuracies": 0.5986905097961426,
      "rewards/chosen": 0.3492187559604645,
      "rewards/margins": 0.13105468451976776,
      "rewards/rejected": 0.21875,
      "step": 3225
    },
    {
      "epoch": 0.8316168898043255,
      "grad_norm": 225.0,
      "learning_rate": 8.419155509783727e-08,
      "logits/chosen": -0.35101318359375,
      "logits/rejected": -0.3466796875,
      "logps/chosen": -294.6000061035156,
      "logps/rejected": -269.0,
      "loss": 0.6352,
      "rewards/accuracies": 0.60916668176651,
      "rewards/chosen": 0.29277342557907104,
      "rewards/margins": 0.14873047173023224,
      "rewards/rejected": 0.14384765923023224,
      "step": 3230
    },
    {
      "epoch": 0.8329042224510813,
      "grad_norm": 114.5,
      "learning_rate": 8.354788877445932e-08,
      "logits/chosen": -0.42109376192092896,
      "logits/rejected": -0.3228515684604645,
      "logps/chosen": -266.20001220703125,
      "logps/rejected": -244.1999969482422,
      "loss": 0.6484,
      "rewards/accuracies": 0.6074999570846558,
      "rewards/chosen": 0.27519530057907104,
      "rewards/margins": 0.11181640625,
      "rewards/rejected": 0.16386719048023224,
      "step": 3235
    },
    {
      "epoch": 0.8341915550978373,
      "grad_norm": 82.5,
      "learning_rate": 8.290422245108136e-08,
      "logits/chosen": -0.44941407442092896,
      "logits/rejected": -0.39082032442092896,
      "logps/chosen": -253.60000610351562,
      "logps/rejected": -259.20001220703125,
      "loss": 0.643,
      "rewards/accuracies": 0.6664285659790039,
      "rewards/chosen": 0.23808594048023224,
      "rewards/margins": 0.12758788466453552,
      "rewards/rejected": 0.11052246391773224,
      "step": 3240
    },
    {
      "epoch": 0.8354788877445932,
      "grad_norm": 118.0,
      "learning_rate": 8.22605561277034e-08,
      "logits/chosen": -0.30195313692092896,
      "logits/rejected": -0.3193359375,
      "logps/chosen": -269.0,
      "logps/rejected": -279.6000061035156,
      "loss": 0.675,
      "rewards/accuracies": 0.5100000500679016,
      "rewards/chosen": 0.17908935248851776,
      "rewards/margins": 0.061553955078125,
      "rewards/rejected": 0.11733398586511612,
      "step": 3245
    },
    {
      "epoch": 0.8367662203913491,
      "grad_norm": 127.0,
      "learning_rate": 8.161688980432543e-08,
      "logits/chosen": -0.4111328125,
      "logits/rejected": -0.4212890565395355,
      "logps/chosen": -316.6000061035156,
      "logps/rejected": -241.1999969482422,
      "loss": 0.6367,
      "rewards/accuracies": 0.6229853630065918,
      "rewards/chosen": 0.24238280951976776,
      "rewards/margins": 0.14473572373390198,
      "rewards/rejected": 0.09755859524011612,
      "step": 3250
    },
    {
      "epoch": 0.838053553038105,
      "grad_norm": 117.0,
      "learning_rate": 8.097322348094747e-08,
      "logits/chosen": NaN,
      "logits/rejected": 0.11679687350988388,
      "logps/chosen": -221.39999389648438,
      "logps/rejected": -205.60000610351562,
      "loss": 0.6609,
      "rewards/accuracies": 0.4315476417541504,
      "rewards/chosen": 0.1302490234375,
      "rewards/margins": 0.0809326171875,
      "rewards/rejected": 0.04926757887005806,
      "step": 3255
    },
    {
      "epoch": 0.839340885684861,
      "grad_norm": 104.0,
      "learning_rate": 8.032955715756952e-08,
      "logits/chosen": -0.39082032442092896,
      "logits/rejected": -0.4287109375,
      "logps/chosen": -271.6000061035156,
      "logps/rejected": -317.20001220703125,
      "loss": 0.6602,
      "rewards/accuracies": 0.5201923251152039,
      "rewards/chosen": 0.23076172173023224,
      "rewards/margins": 0.08427734673023224,
      "rewards/rejected": 0.14699706435203552,
      "step": 3260
    },
    {
      "epoch": 0.8406282183316169,
      "grad_norm": 112.5,
      "learning_rate": 7.968589083419156e-08,
      "logits/chosen": -0.39531248807907104,
      "logits/rejected": -0.40703123807907104,
      "logps/chosen": -300.0,
      "logps/rejected": -253.0,
      "loss": 0.6469,
      "rewards/accuracies": 0.621666669845581,
      "rewards/chosen": 0.22573241591453552,
      "rewards/margins": 0.12020187079906464,
      "rewards/rejected": 0.10556640475988388,
      "step": 3265
    },
    {
      "epoch": 0.8419155509783728,
      "grad_norm": 105.0,
      "learning_rate": 7.904222451081359e-08,
      "logits/chosen": -0.3287109434604645,
      "logits/rejected": -0.4124999940395355,
      "logps/chosen": -352.0,
      "logps/rejected": -307.0,
      "loss": 0.6742,
      "rewards/accuracies": 0.48237180709838867,
      "rewards/chosen": 0.13339844346046448,
      "rewards/margins": 0.05827941745519638,
      "rewards/rejected": 0.07470703125,
      "step": 3270
    },
    {
      "epoch": 0.8432028836251287,
      "grad_norm": 120.5,
      "learning_rate": 7.839855818743563e-08,
      "logits/chosen": -0.26972657442092896,
      "logits/rejected": -0.29765623807907104,
      "logps/chosen": -295.3999938964844,
      "logps/rejected": -234.39999389648438,
      "loss": 0.6695,
      "rewards/accuracies": 0.5395238399505615,
      "rewards/chosen": 0.17661133408546448,
      "rewards/margins": 0.0723114013671875,
      "rewards/rejected": 0.10454101860523224,
      "step": 3275
    },
    {
      "epoch": 0.8444902162718847,
      "grad_norm": 118.0,
      "learning_rate": 7.775489186405767e-08,
      "logits/chosen": -0.3765625059604645,
      "logits/rejected": -0.23115234076976776,
      "logps/chosen": -299.79998779296875,
      "logps/rejected": -249.60000610351562,
      "loss": 0.6695,
      "rewards/accuracies": 0.5294444561004639,
      "rewards/chosen": 0.18828125298023224,
      "rewards/margins": 0.07080688327550888,
      "rewards/rejected": 0.117431640625,
      "step": 3280
    },
    {
      "epoch": 0.8457775489186405,
      "grad_norm": 124.0,
      "learning_rate": 7.711122554067972e-08,
      "logits/chosen": -0.4847656190395355,
      "logits/rejected": -0.4300781190395355,
      "logps/chosen": -274.6000061035156,
      "logps/rejected": -275.0,
      "loss": 0.6547,
      "rewards/accuracies": 0.5544444918632507,
      "rewards/chosen": 0.2652343809604645,
      "rewards/margins": 0.10844039916992188,
      "rewards/rejected": 0.157470703125,
      "step": 3285
    },
    {
      "epoch": 0.8470648815653965,
      "grad_norm": 133.0,
      "learning_rate": 7.646755921730175e-08,
      "logits/chosen": -0.48261719942092896,
      "logits/rejected": -0.46367186307907104,
      "logps/chosen": -294.0,
      "logps/rejected": -293.20001220703125,
      "loss": 0.6602,
      "rewards/accuracies": 0.5953571200370789,
      "rewards/chosen": 0.24111327528953552,
      "rewards/margins": 0.08762206882238388,
      "rewards/rejected": 0.15410156548023224,
      "step": 3290
    },
    {
      "epoch": 0.8483522142121525,
      "grad_norm": 117.0,
      "learning_rate": 7.582389289392379e-08,
      "logits/chosen": -0.47187501192092896,
      "logits/rejected": -0.524609386920929,
      "logps/chosen": -309.20001220703125,
      "logps/rejected": -302.3999938964844,
      "loss": 0.6383,
      "rewards/accuracies": 0.65583336353302,
      "rewards/chosen": 0.32207030057907104,
      "rewards/margins": 0.14335937798023224,
      "rewards/rejected": 0.17949219048023224,
      "step": 3295
    },
    {
      "epoch": 0.8496395468589083,
      "grad_norm": 268.0,
      "learning_rate": 7.518022657054583e-08,
      "logits/chosen": -0.3892578184604645,
      "logits/rejected": -0.344970703125,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -252.1999969482422,
      "loss": 0.6406,
      "rewards/accuracies": 0.5866667032241821,
      "rewards/chosen": 0.23916015028953552,
      "rewards/margins": 0.1201171875,
      "rewards/rejected": 0.11904297024011612,
      "step": 3300
    },
    {
      "epoch": 0.8509268795056643,
      "grad_norm": 159.0,
      "learning_rate": 7.453656024716787e-08,
      "logits/chosen": -0.5269531011581421,
      "logits/rejected": -0.4574218690395355,
      "logps/chosen": -333.20001220703125,
      "logps/rejected": -314.3999938964844,
      "loss": 0.668,
      "rewards/accuracies": 0.4640476107597351,
      "rewards/chosen": 0.13176269829273224,
      "rewards/margins": 0.07280273735523224,
      "rewards/rejected": 0.05888671800494194,
      "step": 3305
    },
    {
      "epoch": 0.8522142121524202,
      "grad_norm": 109.0,
      "learning_rate": 7.38928939237899e-08,
      "logits/chosen": -0.32867431640625,
      "logits/rejected": -0.2938232421875,
      "logps/chosen": -321.20001220703125,
      "logps/rejected": -270.3999938964844,
      "loss": 0.6305,
      "rewards/accuracies": 0.6689394116401672,
      "rewards/chosen": 0.22451171278953552,
      "rewards/margins": 0.14541015028953552,
      "rewards/rejected": 0.07927246391773224,
      "step": 3310
    },
    {
      "epoch": 0.8535015447991761,
      "grad_norm": 109.5,
      "learning_rate": 7.324922760041195e-08,
      "logits/chosen": -0.29082030057907104,
      "logits/rejected": -0.3207031190395355,
      "logps/chosen": -269.3999938964844,
      "logps/rejected": -282.79998779296875,
      "loss": 0.6633,
      "rewards/accuracies": 0.5709091424942017,
      "rewards/chosen": 0.22520752251148224,
      "rewards/margins": 0.08730468899011612,
      "rewards/rejected": 0.13740234076976776,
      "step": 3315
    },
    {
      "epoch": 0.854788877445932,
      "grad_norm": 112.0,
      "learning_rate": 7.260556127703399e-08,
      "logits/chosen": -0.343994140625,
      "logits/rejected": -0.29179686307907104,
      "logps/chosen": -255.8000030517578,
      "logps/rejected": -232.1999969482422,
      "loss": 0.6477,
      "rewards/accuracies": 0.5963889360427856,
      "rewards/chosen": 0.26513671875,
      "rewards/margins": 0.115234375,
      "rewards/rejected": 0.15032958984375,
      "step": 3320
    },
    {
      "epoch": 0.856076210092688,
      "grad_norm": 114.5,
      "learning_rate": 7.196189495365603e-08,
      "logits/chosen": -0.42851561307907104,
      "logits/rejected": -0.4189453125,
      "logps/chosen": -282.79998779296875,
      "logps/rejected": -267.79998779296875,
      "loss": 0.6531,
      "rewards/accuracies": 0.5726648569107056,
      "rewards/chosen": 0.26396483182907104,
      "rewards/margins": 0.110107421875,
      "rewards/rejected": 0.15358276665210724,
      "step": 3325
    },
    {
      "epoch": 0.8573635427394438,
      "grad_norm": 101.0,
      "learning_rate": 7.131822863027806e-08,
      "logits/chosen": -0.48750001192092896,
      "logits/rejected": -0.41425782442092896,
      "logps/chosen": -330.0,
      "logps/rejected": -280.3999938964844,
      "loss": 0.6477,
      "rewards/accuracies": 0.5745238065719604,
      "rewards/chosen": 0.29960936307907104,
      "rewards/margins": 0.114990234375,
      "rewards/rejected": 0.1845703125,
      "step": 3330
    },
    {
      "epoch": 0.8586508753861998,
      "grad_norm": 87.5,
      "learning_rate": 7.067456230690009e-08,
      "logits/chosen": -0.3082031309604645,
      "logits/rejected": -0.2896484434604645,
      "logps/chosen": -226.6999969482422,
      "logps/rejected": -203.1999969482422,
      "loss": 0.6539,
      "rewards/accuracies": 0.505476176738739,
      "rewards/chosen": 0.21401366591453552,
      "rewards/margins": 0.11005859076976776,
      "rewards/rejected": 0.10361327975988388,
      "step": 3335
    },
    {
      "epoch": 0.8599382080329557,
      "grad_norm": 101.5,
      "learning_rate": 7.003089598352215e-08,
      "logits/chosen": -0.3310546875,
      "logits/rejected": -0.3882812559604645,
      "logps/chosen": -298.20001220703125,
      "logps/rejected": -272.0,
      "loss": 0.6617,
      "rewards/accuracies": 0.4785073399543762,
      "rewards/chosen": 0.20292969048023224,
      "rewards/margins": 0.07672729343175888,
      "rewards/rejected": 0.12611083686351776,
      "step": 3340
    },
    {
      "epoch": 0.8612255406797117,
      "grad_norm": 146.0,
      "learning_rate": 6.938722966014417e-08,
      "logits/chosen": -0.267578125,
      "logits/rejected": -0.4036621153354645,
      "logps/chosen": -301.0,
      "logps/rejected": -262.20001220703125,
      "loss": 0.6539,
      "rewards/accuracies": 0.5758333206176758,
      "rewards/chosen": 0.22080078721046448,
      "rewards/margins": 0.10302734375,
      "rewards/rejected": 0.1177978515625,
      "step": 3345
    },
    {
      "epoch": 0.8625128733264675,
      "grad_norm": 98.5,
      "learning_rate": 6.874356333676622e-08,
      "logits/chosen": -0.3326171934604645,
      "logits/rejected": -0.23476561903953552,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -256.20001220703125,
      "loss": 0.6516,
      "rewards/accuracies": 0.4931318759918213,
      "rewards/chosen": 0.2606445252895355,
      "rewards/margins": 0.12089844048023224,
      "rewards/rejected": 0.14028319716453552,
      "step": 3350
    },
    {
      "epoch": 0.8638002059732235,
      "grad_norm": 95.0,
      "learning_rate": 6.809989701338825e-08,
      "logits/chosen": -0.27460938692092896,
      "logits/rejected": -0.20722655951976776,
      "logps/chosen": -291.3999938964844,
      "logps/rejected": -279.79998779296875,
      "loss": 0.6547,
      "rewards/accuracies": 0.5525000095367432,
      "rewards/chosen": 0.2392578125,
      "rewards/margins": 0.09943847358226776,
      "rewards/rejected": 0.13995361328125,
      "step": 3355
    },
    {
      "epoch": 0.8650875386199794,
      "grad_norm": 123.0,
      "learning_rate": 6.74562306900103e-08,
      "logits/chosen": -0.3428710997104645,
      "logits/rejected": -0.20097656548023224,
      "logps/chosen": -278.5,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6328,
      "rewards/accuracies": 0.5985714793205261,
      "rewards/chosen": 0.25859373807907104,
      "rewards/margins": 0.15281982719898224,
      "rewards/rejected": 0.10612793266773224,
      "step": 3360
    },
    {
      "epoch": 0.8663748712667353,
      "grad_norm": 121.0,
      "learning_rate": 6.681256436663233e-08,
      "logits/chosen": -0.4515624940395355,
      "logits/rejected": -0.2828125059604645,
      "logps/chosen": -286.20001220703125,
      "logps/rejected": -244.1999969482422,
      "loss": 0.6359,
      "rewards/accuracies": 0.6158334016799927,
      "rewards/chosen": 0.224609375,
      "rewards/margins": 0.13865967094898224,
      "rewards/rejected": 0.08566894382238388,
      "step": 3365
    },
    {
      "epoch": 0.8676622039134912,
      "grad_norm": 126.5,
      "learning_rate": 6.616889804325438e-08,
      "logits/chosen": -0.35498046875,
      "logits/rejected": -0.41816407442092896,
      "logps/chosen": -294.79998779296875,
      "logps/rejected": -226.39999389648438,
      "loss": 0.6359,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.19189453125,
      "rewards/margins": 0.13496093451976776,
      "rewards/rejected": 0.05686035007238388,
      "step": 3370
    },
    {
      "epoch": 0.8689495365602472,
      "grad_norm": 97.5,
      "learning_rate": 6.55252317198764e-08,
      "logits/chosen": -0.44414061307907104,
      "logits/rejected": -0.4124999940395355,
      "logps/chosen": -288.3999938964844,
      "logps/rejected": -256.79998779296875,
      "loss": 0.6594,
      "rewards/accuracies": 0.6241666674613953,
      "rewards/chosen": 0.18193359673023224,
      "rewards/margins": 0.10646972805261612,
      "rewards/rejected": 0.0750732421875,
      "step": 3375
    },
    {
      "epoch": 0.870236869207003,
      "grad_norm": 110.5,
      "learning_rate": 6.488156539649846e-08,
      "logits/chosen": -0.36054688692092896,
      "logits/rejected": -0.36406248807907104,
      "logps/chosen": -323.20001220703125,
      "logps/rejected": -249.1999969482422,
      "loss": 0.6734,
      "rewards/accuracies": 0.528333306312561,
      "rewards/chosen": 0.21220703423023224,
      "rewards/margins": 0.06538085639476776,
      "rewards/rejected": 0.14707031846046448,
      "step": 3380
    },
    {
      "epoch": 0.871524201853759,
      "grad_norm": 94.5,
      "learning_rate": 6.423789907312049e-08,
      "logits/chosen": -0.35664063692092896,
      "logits/rejected": -0.3433593809604645,
      "logps/chosen": -266.3999938964844,
      "logps/rejected": -257.3999938964844,
      "loss": 0.6875,
      "rewards/accuracies": 0.5227564573287964,
      "rewards/chosen": 0.16484375298023224,
      "rewards/margins": 0.03001098707318306,
      "rewards/rejected": 0.13508300483226776,
      "step": 3385
    },
    {
      "epoch": 0.872811534500515,
      "grad_norm": 129.0,
      "learning_rate": 6.359423274974253e-08,
      "logits/chosen": -0.26386719942092896,
      "logits/rejected": -0.3062988221645355,
      "logps/chosen": -278.3999938964844,
      "logps/rejected": -235.8000030517578,
      "loss": 0.6492,
      "rewards/accuracies": 0.5942857265472412,
      "rewards/chosen": 0.2490234375,
      "rewards/margins": 0.118408203125,
      "rewards/rejected": 0.13078613579273224,
      "step": 3390
    },
    {
      "epoch": 0.8740988671472708,
      "grad_norm": 2736.0,
      "learning_rate": 6.295056642636456e-08,
      "logits/chosen": -0.3433593809604645,
      "logits/rejected": -0.2515625059604645,
      "logps/chosen": -340.3999938964844,
      "logps/rejected": -292.0,
      "loss": 0.6578,
      "rewards/accuracies": 0.5534065961837769,
      "rewards/chosen": 0.24091796576976776,
      "rewards/margins": 0.1019287109375,
      "rewards/rejected": 0.13887938857078552,
      "step": 3395
    },
    {
      "epoch": 0.8753861997940268,
      "grad_norm": 112.0,
      "learning_rate": 6.23069001029866e-08,
      "logits/chosen": -0.2876953184604645,
      "logits/rejected": 0.0498046875,
      "logps/chosen": -242.1999969482422,
      "logps/rejected": -219.0,
      "loss": 0.6602,
      "rewards/accuracies": 0.5065476298332214,
      "rewards/chosen": 0.18217773735523224,
      "rewards/margins": 0.06406249850988388,
      "rewards/rejected": 0.118133544921875,
      "step": 3400
    },
    {
      "epoch": 0.8766735324407827,
      "grad_norm": 127.0,
      "learning_rate": 6.166323377960865e-08,
      "logits/chosen": -0.25800782442092896,
      "logits/rejected": -0.26972657442092896,
      "logps/chosen": -313.0,
      "logps/rejected": -292.20001220703125,
      "loss": 0.6414,
      "rewards/accuracies": 0.5633333921432495,
      "rewards/chosen": 0.19160155951976776,
      "rewards/margins": 0.13503417372703552,
      "rewards/rejected": 0.05693359300494194,
      "step": 3405
    },
    {
      "epoch": 0.8779608650875386,
      "grad_norm": 91.5,
      "learning_rate": 6.101956745623069e-08,
      "logits/chosen": -0.3525390625,
      "logits/rejected": -0.4312500059604645,
      "logps/chosen": -291.79998779296875,
      "logps/rejected": -255.0,
      "loss": 0.6258,
      "rewards/accuracies": 0.6246429085731506,
      "rewards/chosen": 0.27900391817092896,
      "rewards/margins": 0.1806640625,
      "rewards/rejected": 0.09833984076976776,
      "step": 3410
    },
    {
      "epoch": 0.8792481977342945,
      "grad_norm": 228.0,
      "learning_rate": 6.037590113285273e-08,
      "logits/chosen": -0.30400389432907104,
      "logits/rejected": -0.21647949516773224,
      "logps/chosen": -243.39999389648438,
      "logps/rejected": -237.8000030517578,
      "loss": 0.6727,
      "rewards/accuracies": 0.5088278651237488,
      "rewards/chosen": 0.15371093153953552,
      "rewards/margins": 0.06523437798023224,
      "rewards/rejected": 0.08872070163488388,
      "step": 3415
    },
    {
      "epoch": 0.8805355303810505,
      "grad_norm": 100.5,
      "learning_rate": 5.973223480947476e-08,
      "logits/chosen": -0.25385743379592896,
      "logits/rejected": -0.11979980766773224,
      "logps/chosen": -319.20001220703125,
      "logps/rejected": -291.79998779296875,
      "loss": 0.657,
      "rewards/accuracies": 0.5576282143592834,
      "rewards/chosen": 0.078460693359375,
      "rewards/margins": 0.08112792670726776,
      "rewards/rejected": -0.00275421142578125,
      "step": 3420
    },
    {
      "epoch": 0.8818228630278064,
      "grad_norm": 184.0,
      "learning_rate": 5.9088568486096805e-08,
      "logits/chosen": -0.321044921875,
      "logits/rejected": -0.3158203065395355,
      "logps/chosen": -326.0,
      "logps/rejected": -248.0,
      "loss": 0.6727,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.15937499701976776,
      "rewards/margins": 0.08110351860523224,
      "rewards/rejected": 0.07844237983226776,
      "step": 3425
    },
    {
      "epoch": 0.8831101956745623,
      "grad_norm": 414.0,
      "learning_rate": 5.844490216271884e-08,
      "logits/chosen": -0.33955079317092896,
      "logits/rejected": -0.3677734434604645,
      "logps/chosen": -321.0,
      "logps/rejected": -295.3999938964844,
      "loss": 0.6305,
      "rewards/accuracies": 0.6475000381469727,
      "rewards/chosen": 0.2867187559604645,
      "rewards/margins": 0.15292969346046448,
      "rewards/rejected": 0.13374023139476776,
      "step": 3430
    },
    {
      "epoch": 0.8843975283213182,
      "grad_norm": 130.0,
      "learning_rate": 5.7801235839340884e-08,
      "logits/chosen": -0.10029296576976776,
      "logits/rejected": -0.22382812201976776,
      "logps/chosen": -245.8000030517578,
      "logps/rejected": -235.60000610351562,
      "loss": 0.6641,
      "rewards/accuracies": 0.4220454692840576,
      "rewards/chosen": 0.13916015625,
      "rewards/margins": 0.08012695610523224,
      "rewards/rejected": 0.05896606296300888,
      "step": 3435
    },
    {
      "epoch": 0.8856848609680742,
      "grad_norm": 264.0,
      "learning_rate": 5.715756951596292e-08,
      "logits/chosen": -0.3377441465854645,
      "logits/rejected": -0.35917967557907104,
      "logps/chosen": -282.20001220703125,
      "logps/rejected": -310.79998779296875,
      "loss": 0.6375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.29716795682907104,
      "rewards/margins": 0.14422607421875,
      "rewards/rejected": 0.15263672173023224,
      "step": 3440
    },
    {
      "epoch": 0.88697219361483,
      "grad_norm": 99.0,
      "learning_rate": 5.651390319258496e-08,
      "logits/chosen": -0.30585938692092896,
      "logits/rejected": -0.3814453184604645,
      "logps/chosen": -291.0,
      "logps/rejected": -324.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.5769230723381042,
      "rewards/chosen": 0.21943359076976776,
      "rewards/margins": 0.12014160305261612,
      "rewards/rejected": 0.09968261420726776,
      "step": 3445
    },
    {
      "epoch": 0.888259526261586,
      "grad_norm": 112.5,
      "learning_rate": 5.5870236869207e-08,
      "logits/chosen": -0.31005859375,
      "logits/rejected": -0.3296875059604645,
      "logps/chosen": -285.6000061035156,
      "logps/rejected": -216.1999969482422,
      "loss": 0.6617,
      "rewards/accuracies": 0.6117424368858337,
      "rewards/chosen": 0.19321289658546448,
      "rewards/margins": 0.07813720405101776,
      "rewards/rejected": 0.11464843899011612,
      "step": 3450
    },
    {
      "epoch": 0.889546858908342,
      "grad_norm": 119.0,
      "learning_rate": 5.522657054582904e-08,
      "logits/chosen": -0.3857421875,
      "logits/rejected": -0.39277344942092896,
      "logps/chosen": -247.8000030517578,
      "logps/rejected": -250.8000030517578,
      "loss": 0.6641,
      "rewards/accuracies": 0.5408333539962769,
      "rewards/chosen": 0.19316406548023224,
      "rewards/margins": 0.07636718451976776,
      "rewards/rejected": 0.11679687350988388,
      "step": 3455
    },
    {
      "epoch": 0.8908341915550978,
      "grad_norm": 103.5,
      "learning_rate": 5.458290422245108e-08,
      "logits/chosen": -0.3882812559604645,
      "logits/rejected": -0.45429688692092896,
      "logps/chosen": -283.79998779296875,
      "logps/rejected": -236.1999969482422,
      "loss": 0.6602,
      "rewards/accuracies": 0.5076923370361328,
      "rewards/chosen": 0.23481445014476776,
      "rewards/margins": 0.09404297173023224,
      "rewards/rejected": 0.14096680283546448,
      "step": 3460
    },
    {
      "epoch": 0.8921215242018538,
      "grad_norm": 161.0,
      "learning_rate": 5.393923789907312e-08,
      "logits/chosen": -0.4019531309604645,
      "logits/rejected": -0.26611328125,
      "logps/chosen": -229.8000030517578,
      "logps/rejected": -246.0,
      "loss": 0.6813,
      "rewards/accuracies": 0.5347619652748108,
      "rewards/chosen": 0.19990234076976776,
      "rewards/margins": 0.04997558519244194,
      "rewards/rejected": 0.15004882216453552,
      "step": 3465
    },
    {
      "epoch": 0.8934088568486097,
      "grad_norm": 84.5,
      "learning_rate": 5.3295571575695156e-08,
      "logits/chosen": -0.3785156309604645,
      "logits/rejected": -0.22182616591453552,
      "logps/chosen": -241.39999389648438,
      "logps/rejected": -210.8000030517578,
      "loss": 0.6523,
      "rewards/accuracies": 0.5334091186523438,
      "rewards/chosen": 0.25068360567092896,
      "rewards/margins": 0.09906616061925888,
      "rewards/rejected": 0.1513671875,
      "step": 3470
    },
    {
      "epoch": 0.8946961894953656,
      "grad_norm": 133.0,
      "learning_rate": 5.26519052523172e-08,
      "logits/chosen": -0.34355467557907104,
      "logits/rejected": -0.2718749940395355,
      "logps/chosen": -318.0,
      "logps/rejected": -294.3999938964844,
      "loss": 0.657,
      "rewards/accuracies": 0.555961549282074,
      "rewards/chosen": 0.21171875298023224,
      "rewards/margins": 0.095947265625,
      "rewards/rejected": 0.11538086086511612,
      "step": 3475
    },
    {
      "epoch": 0.8959835221421215,
      "grad_norm": 143.0,
      "learning_rate": 5.2008238928939235e-08,
      "logits/chosen": -0.42021483182907104,
      "logits/rejected": -0.3955322206020355,
      "logps/chosen": -309.79998779296875,
      "logps/rejected": -303.20001220703125,
      "loss": 0.6664,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.16728515923023224,
      "rewards/margins": 0.07470703125,
      "rewards/rejected": 0.09268798679113388,
      "step": 3480
    },
    {
      "epoch": 0.8972708547888775,
      "grad_norm": 151.0,
      "learning_rate": 5.136457260556128e-08,
      "logits/chosen": -0.11602173000574112,
      "logits/rejected": -0.14033202826976776,
      "logps/chosen": -262.6000061035156,
      "logps/rejected": -264.79998779296875,
      "loss": 0.6664,
      "rewards/accuracies": 0.5759615302085876,
      "rewards/chosen": 0.16884765028953552,
      "rewards/margins": 0.09379883110523224,
      "rewards/rejected": 0.07490234076976776,
      "step": 3485
    },
    {
      "epoch": 0.8985581874356333,
      "grad_norm": 146.0,
      "learning_rate": 5.0720906282183313e-08,
      "logits/chosen": -0.25849610567092896,
      "logits/rejected": -0.22465820610523224,
      "logps/chosen": -274.0,
      "logps/rejected": -268.3999938964844,
      "loss": 0.6656,
      "rewards/accuracies": 0.5214177370071411,
      "rewards/chosen": 0.23095703125,
      "rewards/margins": 0.08879394829273224,
      "rewards/rejected": 0.142333984375,
      "step": 3490
    },
    {
      "epoch": 0.8998455200823893,
      "grad_norm": 159.0,
      "learning_rate": 5.0077239958805356e-08,
      "logits/chosen": -0.4580078125,
      "logits/rejected": -0.46875,
      "logps/chosen": -345.6000061035156,
      "logps/rejected": -273.20001220703125,
      "loss": 0.6461,
      "rewards/accuracies": 0.5490385293960571,
      "rewards/chosen": 0.19355468451976776,
      "rewards/margins": 0.11923827975988388,
      "rewards/rejected": 0.07441405951976776,
      "step": 3495
    },
    {
      "epoch": 0.9011328527291452,
      "grad_norm": 121.0,
      "learning_rate": 4.943357363542739e-08,
      "logits/chosen": -0.380859375,
      "logits/rejected": -0.19179686903953552,
      "logps/chosen": -253.60000610351562,
      "logps/rejected": -204.8000030517578,
      "loss": 0.6367,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.3070312440395355,
      "rewards/margins": 0.14443358778953552,
      "rewards/rejected": 0.16249999403953552,
      "step": 3500
    },
    {
      "epoch": 0.9024201853759012,
      "grad_norm": 100.5,
      "learning_rate": 4.8789907312049435e-08,
      "logits/chosen": -0.3934570252895355,
      "logits/rejected": -0.39958494901657104,
      "logps/chosen": -314.79998779296875,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6422,
      "rewards/accuracies": 0.6415293216705322,
      "rewards/chosen": 0.27519530057907104,
      "rewards/margins": 0.13222655653953552,
      "rewards/rejected": 0.14284667372703552,
      "step": 3505
    },
    {
      "epoch": 0.903707518022657,
      "grad_norm": 122.0,
      "learning_rate": 4.814624098867147e-08,
      "logits/chosen": -0.3296875059604645,
      "logits/rejected": -0.39335936307907104,
      "logps/chosen": -299.20001220703125,
      "logps/rejected": -266.0,
      "loss": 0.6672,
      "rewards/accuracies": 0.5401923060417175,
      "rewards/chosen": 0.18386229872703552,
      "rewards/margins": 0.07167968899011612,
      "rewards/rejected": 0.11281738430261612,
      "step": 3510
    },
    {
      "epoch": 0.904994850669413,
      "grad_norm": 108.0,
      "learning_rate": 4.7502574665293514e-08,
      "logits/chosen": -0.3785156309604645,
      "logits/rejected": -0.3902343809604645,
      "logps/chosen": -324.0,
      "logps/rejected": -270.3999938964844,
      "loss": 0.6594,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2611328065395355,
      "rewards/margins": 0.09169922024011612,
      "rewards/rejected": 0.16923828423023224,
      "step": 3515
    },
    {
      "epoch": 0.9062821833161689,
      "grad_norm": 98.0,
      "learning_rate": 4.685890834191555e-08,
      "logits/chosen": -0.37158203125,
      "logits/rejected": -0.2914794981479645,
      "logps/chosen": -260.79998779296875,
      "logps/rejected": -251.60000610351562,
      "loss": 0.6234,
      "rewards/accuracies": 0.7075000405311584,
      "rewards/chosen": 0.3304687440395355,
      "rewards/margins": 0.173828125,
      "rewards/rejected": 0.15615233778953552,
      "step": 3520
    },
    {
      "epoch": 0.9075695159629248,
      "grad_norm": 106.0,
      "learning_rate": 4.621524201853759e-08,
      "logits/chosen": -0.33769530057907104,
      "logits/rejected": -0.416015625,
      "logps/chosen": -278.0,
      "logps/rejected": -292.79998779296875,
      "loss": 0.6602,
      "rewards/accuracies": 0.49115386605262756,
      "rewards/chosen": 0.25877684354782104,
      "rewards/margins": 0.08701171725988388,
      "rewards/rejected": 0.171875,
      "step": 3525
    },
    {
      "epoch": 0.9088568486096807,
      "grad_norm": 97.0,
      "learning_rate": 4.557157569515963e-08,
      "logits/chosen": -0.3765625059604645,
      "logits/rejected": -0.38408201932907104,
      "logps/chosen": -288.20001220703125,
      "logps/rejected": -253.39999389648438,
      "loss": 0.65,
      "rewards/accuracies": 0.5508333444595337,
      "rewards/chosen": 0.25908201932907104,
      "rewards/margins": 0.10942383110523224,
      "rewards/rejected": 0.14921875298023224,
      "step": 3530
    },
    {
      "epoch": 0.9101441812564367,
      "grad_norm": 93.5,
      "learning_rate": 4.492790937178167e-08,
      "logits/chosen": -0.31914061307907104,
      "logits/rejected": -0.3355468809604645,
      "logps/chosen": -236.0,
      "logps/rejected": -237.8000030517578,
      "loss": 0.6547,
      "rewards/accuracies": 0.5476190447807312,
      "rewards/chosen": 0.217041015625,
      "rewards/margins": 0.0947265625,
      "rewards/rejected": 0.12236328423023224,
      "step": 3535
    },
    {
      "epoch": 0.9114315139031925,
      "grad_norm": 106.0,
      "learning_rate": 4.42842430484037e-08,
      "logits/chosen": -0.271484375,
      "logits/rejected": -0.26704102754592896,
      "logps/chosen": -279.3999938964844,
      "logps/rejected": -258.20001220703125,
      "loss": 0.65,
      "rewards/accuracies": 0.5285714864730835,
      "rewards/chosen": 0.21342773735523224,
      "rewards/margins": 0.1021728515625,
      "rewards/rejected": 0.11126098781824112,
      "step": 3540
    },
    {
      "epoch": 0.9127188465499485,
      "grad_norm": 201.0,
      "learning_rate": 4.364057672502574e-08,
      "logits/chosen": -0.32597655057907104,
      "logits/rejected": -0.41328126192092896,
      "logps/chosen": -309.6000061035156,
      "logps/rejected": -311.0,
      "loss": 0.6352,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.24580077826976776,
      "rewards/margins": 0.13193359971046448,
      "rewards/rejected": 0.11381836235523224,
      "step": 3545
    },
    {
      "epoch": 0.9140061791967045,
      "grad_norm": 112.5,
      "learning_rate": 4.299691040164778e-08,
      "logits/chosen": -0.3985351622104645,
      "logits/rejected": -0.4320312440395355,
      "logps/chosen": -285.0,
      "logps/rejected": -286.79998779296875,
      "loss": 0.657,
      "rewards/accuracies": 0.5175000429153442,
      "rewards/chosen": 0.207763671875,
      "rewards/margins": 0.09326171875,
      "rewards/rejected": 0.11464843899011612,
      "step": 3550
    },
    {
      "epoch": 0.9152935118434603,
      "grad_norm": 102.5,
      "learning_rate": 4.235324407826982e-08,
      "logits/chosen": -0.34843748807907104,
      "logits/rejected": -0.2935546934604645,
      "logps/chosen": -279.79998779296875,
      "logps/rejected": -273.79998779296875,
      "loss": 0.6734,
      "rewards/accuracies": 0.5576923489570618,
      "rewards/chosen": 0.22382812201976776,
      "rewards/margins": 0.053466796875,
      "rewards/rejected": 0.17011718451976776,
      "step": 3555
    },
    {
      "epoch": 0.9165808444902163,
      "grad_norm": 127.5,
      "learning_rate": 4.170957775489186e-08,
      "logits/chosen": -0.4341796934604645,
      "logits/rejected": -0.4769531190395355,
      "logps/chosen": -328.3999938964844,
      "logps/rejected": -270.3999938964844,
      "loss": 0.668,
      "rewards/accuracies": 0.5172619223594666,
      "rewards/chosen": 0.20805664360523224,
      "rewards/margins": 0.083740234375,
      "rewards/rejected": 0.12422408908605576,
      "step": 3560
    },
    {
      "epoch": 0.9178681771369722,
      "grad_norm": 113.5,
      "learning_rate": 4.10659114315139e-08,
      "logits/chosen": -0.26982420682907104,
      "logits/rejected": -0.2891601622104645,
      "logps/chosen": -297.20001220703125,
      "logps/rejected": -292.3999938964844,
      "loss": 0.6664,
      "rewards/accuracies": 0.5091666579246521,
      "rewards/chosen": 0.16335448622703552,
      "rewards/margins": 0.08332519233226776,
      "rewards/rejected": 0.08002929389476776,
      "step": 3565
    },
    {
      "epoch": 0.9191555097837281,
      "grad_norm": 120.5,
      "learning_rate": 4.042224510813594e-08,
      "logits/chosen": -0.2470703125,
      "logits/rejected": -0.16933593153953552,
      "logps/chosen": -271.6000061035156,
      "logps/rejected": -241.10000610351562,
      "loss": 0.65,
      "rewards/accuracies": 0.53083336353302,
      "rewards/chosen": 0.22499999403953552,
      "rewards/margins": 0.10747070610523224,
      "rewards/rejected": 0.11773681640625,
      "step": 3570
    },
    {
      "epoch": 0.920442842430484,
      "grad_norm": 127.5,
      "learning_rate": 3.977857878475798e-08,
      "logits/chosen": -0.31269532442092896,
      "logits/rejected": -0.2987304627895355,
      "logps/chosen": -285.0,
      "logps/rejected": -207.8000030517578,
      "loss": 0.6461,
      "rewards/accuracies": 0.6483333706855774,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 0.11540527641773224,
      "rewards/rejected": 0.1405029296875,
      "step": 3575
    },
    {
      "epoch": 0.92173017507724,
      "grad_norm": 116.0,
      "learning_rate": 3.9134912461380015e-08,
      "logits/chosen": -0.3701171875,
      "logits/rejected": -0.39531248807907104,
      "logps/chosen": -287.6000061035156,
      "logps/rejected": -283.6000061035156,
      "loss": 0.6445,
      "rewards/accuracies": 0.6066666841506958,
      "rewards/chosen": 0.22001953423023224,
      "rewards/margins": 0.11088867485523224,
      "rewards/rejected": 0.10908202826976776,
      "step": 3580
    },
    {
      "epoch": 0.9230175077239959,
      "grad_norm": 180.0,
      "learning_rate": 3.849124613800206e-08,
      "logits/chosen": -0.35917967557907104,
      "logits/rejected": -0.4185546934604645,
      "logps/chosen": -302.6000061035156,
      "logps/rejected": -237.1999969482422,
      "loss": 0.6484,
      "rewards/accuracies": 0.5841667056083679,
      "rewards/chosen": 0.20878906548023224,
      "rewards/margins": 0.11699219048023224,
      "rewards/rejected": 0.09218750149011612,
      "step": 3585
    },
    {
      "epoch": 0.9243048403707518,
      "grad_norm": 101.5,
      "learning_rate": 3.7847579814624094e-08,
      "logits/chosen": -0.44169920682907104,
      "logits/rejected": -0.16015625,
      "logps/chosen": -270.20001220703125,
      "logps/rejected": -244.1999969482422,
      "loss": 0.6547,
      "rewards/accuracies": 0.4997802674770355,
      "rewards/chosen": 0.15625,
      "rewards/margins": 0.09763183444738388,
      "rewards/rejected": 0.05861816555261612,
      "step": 3590
    },
    {
      "epoch": 0.9255921730175077,
      "grad_norm": 204.0,
      "learning_rate": 3.720391349124614e-08,
      "logits/chosen": -0.3594726622104645,
      "logits/rejected": -0.21567383408546448,
      "logps/chosen": -293.20001220703125,
      "logps/rejected": -280.6000061035156,
      "loss": 0.6711,
      "rewards/accuracies": 0.5789394378662109,
      "rewards/chosen": 0.20429687201976776,
      "rewards/margins": 0.07294921576976776,
      "rewards/rejected": 0.13144531846046448,
      "step": 3595
    },
    {
      "epoch": 0.9268795056642637,
      "grad_norm": 103.0,
      "learning_rate": 3.656024716786817e-08,
      "logits/chosen": -0.3326171934604645,
      "logits/rejected": -0.24296875298023224,
      "logps/chosen": -259.79998779296875,
      "logps/rejected": -250.0,
      "loss": 0.6344,
      "rewards/accuracies": 0.5786508321762085,
      "rewards/chosen": 0.19362792372703552,
      "rewards/margins": 0.13900145888328552,
      "rewards/rejected": 0.054718017578125,
      "step": 3600
    },
    {
      "epoch": 0.9281668383110195,
      "grad_norm": 92.0,
      "learning_rate": 3.5916580844490216e-08,
      "logits/chosen": -0.3402343690395355,
      "logits/rejected": -0.20761719346046448,
      "logps/chosen": -241.1999969482422,
      "logps/rejected": -235.39999389648438,
      "loss": 0.6477,
      "rewards/accuracies": 0.5097527503967285,
      "rewards/chosen": 0.25114744901657104,
      "rewards/margins": 0.10893554985523224,
      "rewards/rejected": 0.14272460341453552,
      "step": 3605
    },
    {
      "epoch": 0.9294541709577755,
      "grad_norm": 115.0,
      "learning_rate": 3.527291452111225e-08,
      "logits/chosen": -0.314453125,
      "logits/rejected": -0.28828126192092896,
      "logps/chosen": -279.0,
      "logps/rejected": -226.8000030517578,
      "loss": 0.6406,
      "rewards/accuracies": 0.5808333158493042,
      "rewards/chosen": 0.2865234315395355,
      "rewards/margins": 0.12617187201976776,
      "rewards/rejected": 0.160400390625,
      "step": 3610
    },
    {
      "epoch": 0.9307415036045315,
      "grad_norm": 141.0,
      "learning_rate": 3.4629248197734294e-08,
      "logits/chosen": -0.37714844942092896,
      "logits/rejected": -0.4115234315395355,
      "logps/chosen": -311.20001220703125,
      "logps/rejected": -281.0,
      "loss": 0.668,
      "rewards/accuracies": 0.5511904954910278,
      "rewards/chosen": 0.2943359315395355,
      "rewards/margins": 0.08925781399011612,
      "rewards/rejected": 0.20488281548023224,
      "step": 3615
    },
    {
      "epoch": 0.9320288362512873,
      "grad_norm": 118.5,
      "learning_rate": 3.398558187435633e-08,
      "logits/chosen": -0.4234375059604645,
      "logits/rejected": -0.27470701932907104,
      "logps/chosen": -329.6000061035156,
      "logps/rejected": -262.3999938964844,
      "loss": 0.6578,
      "rewards/accuracies": 0.43047624826431274,
      "rewards/chosen": 0.20732422173023224,
      "rewards/margins": 0.07822265475988388,
      "rewards/rejected": 0.12949219346046448,
      "step": 3620
    },
    {
      "epoch": 0.9333161688980433,
      "grad_norm": 106.5,
      "learning_rate": 3.334191555097837e-08,
      "logits/chosen": -0.4033203125,
      "logits/rejected": -0.327880859375,
      "logps/chosen": -290.79998779296875,
      "logps/rejected": -248.8000030517578,
      "loss": 0.6539,
      "rewards/accuracies": 0.5010256767272949,
      "rewards/chosen": 0.13193359971046448,
      "rewards/margins": 0.09816894680261612,
      "rewards/rejected": 0.033935546875,
      "step": 3625
    },
    {
      "epoch": 0.9346035015447992,
      "grad_norm": 103.5,
      "learning_rate": 3.269824922760041e-08,
      "logits/chosen": -0.31718748807907104,
      "logits/rejected": -0.29414063692092896,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -297.3999938964844,
      "loss": 0.6695,
      "rewards/accuracies": 0.49666666984558105,
      "rewards/chosen": 0.19677734375,
      "rewards/margins": 0.078369140625,
      "rewards/rejected": 0.11837158352136612,
      "step": 3630
    },
    {
      "epoch": 0.935890834191555,
      "grad_norm": 139.0,
      "learning_rate": 3.205458290422245e-08,
      "logits/chosen": -0.3563232421875,
      "logits/rejected": -0.39482420682907104,
      "logps/chosen": -289.79998779296875,
      "logps/rejected": -268.0,
      "loss": 0.6687,
      "rewards/accuracies": 0.5464377999305725,
      "rewards/chosen": 0.17583008110523224,
      "rewards/margins": 0.06051330640912056,
      "rewards/rejected": 0.1153564453125,
      "step": 3635
    },
    {
      "epoch": 0.937178166838311,
      "grad_norm": 107.0,
      "learning_rate": 3.141091658084449e-08,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.41796875,
      "logps/chosen": -303.3999938964844,
      "logps/rejected": -262.6000061035156,
      "loss": 0.6578,
      "rewards/accuracies": 0.5320346355438232,
      "rewards/chosen": 0.11757812649011612,
      "rewards/margins": 0.0992431640625,
      "rewards/rejected": 0.01816406287252903,
      "step": 3640
    },
    {
      "epoch": 0.938465499485067,
      "grad_norm": 116.5,
      "learning_rate": 3.076725025746653e-08,
      "logits/chosen": -0.4535156190395355,
      "logits/rejected": -0.4375,
      "logps/chosen": -349.0,
      "logps/rejected": -268.0,
      "loss": 0.6484,
      "rewards/accuracies": 0.5446795225143433,
      "rewards/chosen": 0.24287109076976776,
      "rewards/margins": 0.101318359375,
      "rewards/rejected": 0.14169922471046448,
      "step": 3645
    },
    {
      "epoch": 0.9397528321318228,
      "grad_norm": 146.0,
      "learning_rate": 3.0123583934088567e-08,
      "logits/chosen": -0.16689452528953552,
      "logits/rejected": -0.3453125059604645,
      "logps/chosen": -305.6000061035156,
      "logps/rejected": -264.6000061035156,
      "loss": 0.6406,
      "rewards/accuracies": 0.5807143449783325,
      "rewards/chosen": 0.25762939453125,
      "rewards/margins": 0.11888428032398224,
      "rewards/rejected": 0.13916626572608948,
      "step": 3650
    },
    {
      "epoch": 0.9410401647785788,
      "grad_norm": 89.5,
      "learning_rate": 2.9479917610710606e-08,
      "logits/chosen": -0.2696289122104645,
      "logits/rejected": -0.3072265684604645,
      "logps/chosen": -284.3999938964844,
      "logps/rejected": -261.79998779296875,
      "loss": 0.6289,
      "rewards/accuracies": 0.6525000333786011,
      "rewards/chosen": 0.29667967557907104,
      "rewards/margins": 0.16416016221046448,
      "rewards/rejected": 0.13222655653953552,
      "step": 3655
    },
    {
      "epoch": 0.9423274974253347,
      "grad_norm": 109.0,
      "learning_rate": 2.8836251287332645e-08,
      "logits/chosen": -0.39921873807907104,
      "logits/rejected": -0.3187499940395355,
      "logps/chosen": -296.20001220703125,
      "logps/rejected": -258.20001220703125,
      "loss": 0.6461,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.28217774629592896,
      "rewards/margins": 0.12514647841453552,
      "rewards/rejected": 0.1573486328125,
      "step": 3660
    },
    {
      "epoch": 0.9436148300720907,
      "grad_norm": 99.5,
      "learning_rate": 2.8192584963954685e-08,
      "logits/chosen": -0.42070311307907104,
      "logits/rejected": -0.46601563692092896,
      "logps/chosen": -287.6000061035156,
      "logps/rejected": -275.20001220703125,
      "loss": 0.6453,
      "rewards/accuracies": 0.5391666889190674,
      "rewards/chosen": 0.233154296875,
      "rewards/margins": 0.124755859375,
      "rewards/rejected": 0.10867004096508026,
      "step": 3665
    },
    {
      "epoch": 0.9449021627188465,
      "grad_norm": 410.0,
      "learning_rate": 2.7548918640576724e-08,
      "logits/chosen": -0.28925782442092896,
      "logits/rejected": -0.35664063692092896,
      "logps/chosen": -261.79998779296875,
      "logps/rejected": -204.39999389648438,
      "loss": 0.6625,
      "rewards/accuracies": 0.5351190567016602,
      "rewards/chosen": 0.163330078125,
      "rewards/margins": 0.07346191257238388,
      "rewards/rejected": 0.09010009467601776,
      "step": 3670
    },
    {
      "epoch": 0.9461894953656025,
      "grad_norm": 140.0,
      "learning_rate": 2.6905252317198764e-08,
      "logits/chosen": -0.25898438692092896,
      "logits/rejected": -0.27226561307907104,
      "logps/chosen": -273.6000061035156,
      "logps/rejected": -213.60000610351562,
      "loss": 0.6703,
      "rewards/accuracies": 0.49935898184776306,
      "rewards/chosen": 0.12631836533546448,
      "rewards/margins": 0.06418456882238388,
      "rewards/rejected": 0.06227416917681694,
      "step": 3675
    },
    {
      "epoch": 0.9474768280123584,
      "grad_norm": 239.0,
      "learning_rate": 2.6261585993820803e-08,
      "logits/chosen": -0.19362792372703552,
      "logits/rejected": -0.19199219346046448,
      "logps/chosen": -240.0,
      "logps/rejected": -217.39999389648438,
      "loss": 0.6336,
      "rewards/accuracies": 0.6940934062004089,
      "rewards/chosen": 0.21757812798023224,
      "rewards/margins": 0.14174804091453552,
      "rewards/rejected": 0.07626952975988388,
      "step": 3680
    },
    {
      "epoch": 0.9487641606591143,
      "grad_norm": 162.0,
      "learning_rate": 2.5617919670442842e-08,
      "logits/chosen": -0.23764649033546448,
      "logits/rejected": -0.24594727158546448,
      "logps/chosen": -262.79998779296875,
      "logps/rejected": -272.20001220703125,
      "loss": 0.6594,
      "rewards/accuracies": 0.6183333396911621,
      "rewards/chosen": 0.20244140923023224,
      "rewards/margins": 0.10400390625,
      "rewards/rejected": 0.09848632663488388,
      "step": 3685
    },
    {
      "epoch": 0.9500514933058702,
      "grad_norm": 109.0,
      "learning_rate": 2.497425334706488e-08,
      "logits/chosen": -0.2876953184604645,
      "logits/rejected": -0.42265623807907104,
      "logps/chosen": -296.0,
      "logps/rejected": -245.1999969482422,
      "loss": 0.643,
      "rewards/accuracies": 0.6389102339744568,
      "rewards/chosen": 0.19177857041358948,
      "rewards/margins": 0.11767578125,
      "rewards/rejected": 0.07449951022863388,
      "step": 3690
    },
    {
      "epoch": 0.9513388259526262,
      "grad_norm": 94.5,
      "learning_rate": 2.433058702368692e-08,
      "logits/chosen": -0.30195313692092896,
      "logits/rejected": -0.408203125,
      "logps/chosen": -268.20001220703125,
      "logps/rejected": -217.39999389648438,
      "loss": 0.6258,
      "rewards/accuracies": 0.6596428751945496,
      "rewards/chosen": 0.29414063692092896,
      "rewards/margins": 0.15810546278953552,
      "rewards/rejected": 0.13613280653953552,
      "step": 3695
    },
    {
      "epoch": 0.952626158599382,
      "grad_norm": 116.5,
      "learning_rate": 2.368692070030896e-08,
      "logits/chosen": -0.34296876192092896,
      "logits/rejected": -0.3003906309604645,
      "logps/chosen": -311.0,
      "logps/rejected": -279.0,
      "loss": 0.6562,
      "rewards/accuracies": 0.5616666674613953,
      "rewards/chosen": 0.17197266221046448,
      "rewards/margins": 0.08916015923023224,
      "rewards/rejected": 0.08299560844898224,
      "step": 3700
    },
    {
      "epoch": 0.953913491246138,
      "grad_norm": 108.0,
      "learning_rate": 2.3043254376931e-08,
      "logits/chosen": -0.4302734434604645,
      "logits/rejected": -0.4068359434604645,
      "logps/chosen": -333.20001220703125,
      "logps/rejected": -246.6999969482422,
      "loss": 0.6484,
      "rewards/accuracies": 0.5714285969734192,
      "rewards/chosen": 0.19814452528953552,
      "rewards/margins": 0.11650390923023224,
      "rewards/rejected": 0.08146514743566513,
      "step": 3705
    },
    {
      "epoch": 0.955200823892894,
      "grad_norm": 131.0,
      "learning_rate": 2.239958805355304e-08,
      "logits/chosen": -0.28925782442092896,
      "logits/rejected": -0.3580078184604645,
      "logps/chosen": -289.3999938964844,
      "logps/rejected": -255.1999969482422,
      "loss": 0.6813,
      "rewards/accuracies": 0.44630956649780273,
      "rewards/chosen": 0.12045898288488388,
      "rewards/margins": 0.04615974426269531,
      "rewards/rejected": 0.07442627102136612,
      "step": 3710
    },
    {
      "epoch": 0.9564881565396498,
      "grad_norm": 129.0,
      "learning_rate": 2.1755921730175075e-08,
      "logits/chosen": -0.30156248807907104,
      "logits/rejected": -0.2689453065395355,
      "logps/chosen": -283.6000061035156,
      "logps/rejected": -231.5,
      "loss": 0.6758,
      "rewards/accuracies": 0.5777778029441833,
      "rewards/chosen": 0.17143554985523224,
      "rewards/margins": 0.0526123046875,
      "rewards/rejected": 0.119140625,
      "step": 3715
    },
    {
      "epoch": 0.9577754891864058,
      "grad_norm": 362.0,
      "learning_rate": 2.1112255406797115e-08,
      "logits/chosen": -0.4072265625,
      "logits/rejected": -0.3076171875,
      "logps/chosen": -320.20001220703125,
      "logps/rejected": -242.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.5208333134651184,
      "rewards/chosen": 0.217529296875,
      "rewards/margins": 0.10898437350988388,
      "rewards/rejected": 0.108642578125,
      "step": 3720
    },
    {
      "epoch": 0.9590628218331617,
      "grad_norm": 109.5,
      "learning_rate": 2.0468589083419154e-08,
      "logits/chosen": -0.29374998807907104,
      "logits/rejected": -0.294921875,
      "logps/chosen": -248.39999389648438,
      "logps/rejected": -226.39999389648438,
      "loss": 0.6695,
      "rewards/accuracies": 0.5567857027053833,
      "rewards/chosen": 0.18994140625,
      "rewards/margins": 0.06611327826976776,
      "rewards/rejected": 0.12387695163488388,
      "step": 3725
    },
    {
      "epoch": 0.9603501544799176,
      "grad_norm": 117.5,
      "learning_rate": 1.9824922760041193e-08,
      "logits/chosen": -0.40864259004592896,
      "logits/rejected": -0.4736328125,
      "logps/chosen": -292.6000061035156,
      "logps/rejected": -240.0,
      "loss": 0.6391,
      "rewards/accuracies": 0.5766667127609253,
      "rewards/chosen": 0.24465331435203552,
      "rewards/margins": 0.13740234076976776,
      "rewards/rejected": 0.10756836086511612,
      "step": 3730
    },
    {
      "epoch": 0.9616374871266735,
      "grad_norm": 237.0,
      "learning_rate": 1.9181256436663233e-08,
      "logits/chosen": -0.42412108182907104,
      "logits/rejected": -0.338134765625,
      "logps/chosen": -298.20001220703125,
      "logps/rejected": -273.79998779296875,
      "loss": 0.6672,
      "rewards/accuracies": 0.4450758099555969,
      "rewards/chosen": 0.11982421576976776,
      "rewards/margins": 0.07280273735523224,
      "rewards/rejected": 0.04720459133386612,
      "step": 3735
    },
    {
      "epoch": 0.9629248197734295,
      "grad_norm": 85.5,
      "learning_rate": 1.8537590113285272e-08,
      "logits/chosen": -0.22993163764476776,
      "logits/rejected": -0.24892577528953552,
      "logps/chosen": -190.0,
      "logps/rejected": -194.10000610351562,
      "loss": 0.6305,
      "rewards/accuracies": 0.6691666841506958,
      "rewards/chosen": 0.24892577528953552,
      "rewards/margins": 0.14443358778953552,
      "rewards/rejected": 0.10488281399011612,
      "step": 3740
    },
    {
      "epoch": 0.9642121524201854,
      "grad_norm": 179.0,
      "learning_rate": 1.789392378990731e-08,
      "logits/chosen": -0.29316407442092896,
      "logits/rejected": -0.24433593451976776,
      "logps/chosen": -275.79998779296875,
      "logps/rejected": -241.8000030517578,
      "loss": 0.6438,
      "rewards/accuracies": 0.5934065580368042,
      "rewards/chosen": 0.20864257216453552,
      "rewards/margins": 0.13957519829273224,
      "rewards/rejected": 0.0693359375,
      "step": 3745
    },
    {
      "epoch": 0.9654994850669413,
      "grad_norm": 104.5,
      "learning_rate": 1.725025746652935e-08,
      "logits/chosen": -0.470703125,
      "logits/rejected": -0.45820313692092896,
      "logps/chosen": -218.0,
      "logps/rejected": -237.3000030517578,
      "loss": 0.657,
      "rewards/accuracies": 0.4958333373069763,
      "rewards/chosen": 0.16452637314796448,
      "rewards/margins": 0.08818359673023224,
      "rewards/rejected": 0.07651367038488388,
      "step": 3750
    },
    {
      "epoch": 0.9667868177136972,
      "grad_norm": 169.0,
      "learning_rate": 1.660659114315139e-08,
      "logits/chosen": -0.3697265684604645,
      "logits/rejected": -0.3832031190395355,
      "logps/chosen": -252.60000610351562,
      "logps/rejected": -238.60000610351562,
      "loss": 0.6453,
      "rewards/accuracies": 0.6529762148857117,
      "rewards/chosen": 0.24531249701976776,
      "rewards/margins": 0.11318359524011612,
      "rewards/rejected": 0.1318359375,
      "step": 3755
    },
    {
      "epoch": 0.9680741503604532,
      "grad_norm": 102.0,
      "learning_rate": 1.596292481977343e-08,
      "logits/chosen": -0.361907958984375,
      "logits/rejected": -0.28076171875,
      "logps/chosen": -252.60000610351562,
      "logps/rejected": -221.60000610351562,
      "loss": 0.6531,
      "rewards/accuracies": 0.5608333349227905,
      "rewards/chosen": 0.2535156309604645,
      "rewards/margins": 0.09726562350988388,
      "rewards/rejected": 0.156005859375,
      "step": 3760
    },
    {
      "epoch": 0.969361483007209,
      "grad_norm": 166.0,
      "learning_rate": 1.531925849639547e-08,
      "logits/chosen": -0.35185545682907104,
      "logits/rejected": -0.25861817598342896,
      "logps/chosen": -275.3999938964844,
      "logps/rejected": -241.0,
      "loss": 0.6789,
      "rewards/accuracies": 0.4507051408290863,
      "rewards/chosen": 0.11318054050207138,
      "rewards/margins": 0.04807128757238388,
      "rewards/rejected": 0.06531982123851776,
      "step": 3765
    },
    {
      "epoch": 0.970648815653965,
      "grad_norm": 107.5,
      "learning_rate": 1.4675592173017507e-08,
      "logits/chosen": -0.380859375,
      "logits/rejected": 0.0072265625931322575,
      "logps/chosen": -244.89999389648438,
      "logps/rejected": -230.0,
      "loss": 0.6445,
      "rewards/accuracies": 0.6844780445098877,
      "rewards/chosen": 0.24863281846046448,
      "rewards/margins": 0.13447265326976776,
      "rewards/rejected": 0.11435546725988388,
      "step": 3770
    },
    {
      "epoch": 0.971936148300721,
      "grad_norm": 308.0,
      "learning_rate": 1.4031925849639546e-08,
      "logits/chosen": -0.3451171815395355,
      "logits/rejected": -0.4261718690395355,
      "logps/chosen": -297.6000061035156,
      "logps/rejected": -283.0,
      "loss": 0.632,
      "rewards/accuracies": 0.6212301850318909,
      "rewards/chosen": 0.25312501192092896,
      "rewards/margins": 0.146484375,
      "rewards/rejected": 0.10693359375,
      "step": 3775
    },
    {
      "epoch": 0.9732234809474768,
      "grad_norm": 114.5,
      "learning_rate": 1.3388259526261585e-08,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.27812498807907104,
      "logps/chosen": -272.0,
      "logps/rejected": -271.6000061035156,
      "loss": 0.6375,
      "rewards/accuracies": 0.5497435927391052,
      "rewards/chosen": 0.2562499940395355,
      "rewards/margins": 0.12873534858226776,
      "rewards/rejected": 0.12734679877758026,
      "step": 3780
    },
    {
      "epoch": 0.9745108135942327,
      "grad_norm": 113.0,
      "learning_rate": 1.2744593202883625e-08,
      "logits/chosen": -0.22412109375,
      "logits/rejected": -0.39042967557907104,
      "logps/chosen": -308.79998779296875,
      "logps/rejected": -272.20001220703125,
      "loss": 0.6586,
      "rewards/accuracies": 0.5600000619888306,
      "rewards/chosen": 0.17402343451976776,
      "rewards/margins": 0.08906249701976776,
      "rewards/rejected": 0.08476562798023224,
      "step": 3785
    },
    {
      "epoch": 0.9757981462409887,
      "grad_norm": 121.5,
      "learning_rate": 1.2100926879505664e-08,
      "logits/chosen": -0.3115234375,
      "logits/rejected": -0.49882811307907104,
      "logps/chosen": -257.79998779296875,
      "logps/rejected": -254.10000610351562,
      "loss": 0.6461,
      "rewards/accuracies": 0.5847222208976746,
      "rewards/chosen": 0.2001953125,
      "rewards/margins": 0.123291015625,
      "rewards/rejected": 0.07698364555835724,
      "step": 3790
    },
    {
      "epoch": 0.9770854788877446,
      "grad_norm": 141.0,
      "learning_rate": 1.1457260556127703e-08,
      "logits/chosen": -0.49687498807907104,
      "logits/rejected": -0.4078125059604645,
      "logps/chosen": -233.60000610351562,
      "logps/rejected": -252.60000610351562,
      "loss": 0.6562,
      "rewards/accuracies": 0.5839393734931946,
      "rewards/chosen": 0.21142578125,
      "rewards/margins": 0.09462890774011612,
      "rewards/rejected": 0.116943359375,
      "step": 3795
    },
    {
      "epoch": 0.9783728115345005,
      "grad_norm": 100.0,
      "learning_rate": 1.0813594232749741e-08,
      "logits/chosen": -0.291015625,
      "logits/rejected": -0.33427733182907104,
      "logps/chosen": -268.0,
      "logps/rejected": -246.60000610351562,
      "loss": 0.6523,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.20048828423023224,
      "rewards/margins": 0.10117187350988388,
      "rewards/rejected": 0.09967041015625,
      "step": 3800
    },
    {
      "epoch": 0.9796601441812565,
      "grad_norm": 109.5,
      "learning_rate": 1.016992790937178e-08,
      "logits/chosen": -0.34160155057907104,
      "logits/rejected": -0.291015625,
      "logps/chosen": -216.8000030517578,
      "logps/rejected": -226.3000030517578,
      "loss": 0.6578,
      "rewards/accuracies": 0.5654761791229248,
      "rewards/chosen": 0.20556640625,
      "rewards/margins": 0.08869323879480362,
      "rewards/rejected": 0.11689452826976776,
      "step": 3805
    },
    {
      "epoch": 0.9809474768280123,
      "grad_norm": 112.5,
      "learning_rate": 9.52626158599382e-09,
      "logits/chosen": -0.33544921875,
      "logits/rejected": -0.3285156190395355,
      "logps/chosen": -253.8000030517578,
      "logps/rejected": -229.39999389648438,
      "loss": 0.6555,
      "rewards/accuracies": 0.5708333253860474,
      "rewards/chosen": 0.17714843153953552,
      "rewards/margins": 0.11213378608226776,
      "rewards/rejected": 0.06533203274011612,
      "step": 3810
    },
    {
      "epoch": 0.9822348094747683,
      "grad_norm": 95.0,
      "learning_rate": 8.88259526261586e-09,
      "logits/chosen": -0.22470703721046448,
      "logits/rejected": -0.2662109434604645,
      "logps/chosen": -218.8000030517578,
      "logps/rejected": -226.0,
      "loss": 0.6414,
      "rewards/accuracies": 0.6274999976158142,
      "rewards/chosen": 0.2666015625,
      "rewards/margins": 0.12456054985523224,
      "rewards/rejected": 0.1416015625,
      "step": 3815
    },
    {
      "epoch": 0.9835221421215242,
      "grad_norm": 103.0,
      "learning_rate": 8.238928939237899e-09,
      "logits/chosen": -0.223876953125,
      "logits/rejected": -0.2962890565395355,
      "logps/chosen": -274.79998779296875,
      "logps/rejected": -300.0,
      "loss": 0.6789,
      "rewards/accuracies": 0.5077564120292664,
      "rewards/chosen": 0.2138671875,
      "rewards/margins": 0.04931640625,
      "rewards/rejected": 0.16441650688648224,
      "step": 3820
    },
    {
      "epoch": 0.9848094747682801,
      "grad_norm": 180.0,
      "learning_rate": 7.595262615859938e-09,
      "logits/chosen": -0.20878295600414276,
      "logits/rejected": -0.22724609076976776,
      "logps/chosen": -235.1999969482422,
      "logps/rejected": -265.20001220703125,
      "loss": 0.6664,
      "rewards/accuracies": 0.5024999976158142,
      "rewards/chosen": 0.24492187798023224,
      "rewards/margins": 0.06142578274011612,
      "rewards/rejected": 0.18320313096046448,
      "step": 3825
    },
    {
      "epoch": 0.986096807415036,
      "grad_norm": 306.0,
      "learning_rate": 6.951596292481977e-09,
      "logits/chosen": -0.2686523497104645,
      "logits/rejected": -0.3158203065395355,
      "logps/chosen": -323.20001220703125,
      "logps/rejected": -276.3999938964844,
      "loss": 0.6523,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.2621093690395355,
      "rewards/margins": 0.11728515475988388,
      "rewards/rejected": 0.14511719346046448,
      "step": 3830
    },
    {
      "epoch": 0.987384140061792,
      "grad_norm": 159.0,
      "learning_rate": 6.307929969104016e-09,
      "logits/chosen": -0.3714843690395355,
      "logits/rejected": -0.24423828721046448,
      "logps/chosen": -280.3999938964844,
      "logps/rejected": -268.79998779296875,
      "loss": 0.6359,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": 0.3070312440395355,
      "rewards/margins": 0.14681395888328552,
      "rewards/rejected": 0.16035155951976776,
      "step": 3835
    },
    {
      "epoch": 0.9886714727085479,
      "grad_norm": 98.5,
      "learning_rate": 5.664263645726055e-09,
      "logits/chosen": -0.30732423067092896,
      "logits/rejected": -0.3603149354457855,
      "logps/chosen": -336.6000061035156,
      "logps/rejected": -288.0,
      "loss": 0.657,
      "rewards/accuracies": 0.5617948770523071,
      "rewards/chosen": 0.21181640028953552,
      "rewards/margins": 0.09604492038488388,
      "rewards/rejected": 0.11601562798023224,
      "step": 3840
    },
    {
      "epoch": 0.9899588053553038,
      "grad_norm": 120.0,
      "learning_rate": 5.020597322348095e-09,
      "logits/chosen": -0.2505859434604645,
      "logits/rejected": -0.33232420682907104,
      "logps/chosen": -349.79998779296875,
      "logps/rejected": -253.60000610351562,
      "loss": 0.6336,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 0.29960936307907104,
      "rewards/margins": 0.14399413764476776,
      "rewards/rejected": 0.15549317002296448,
      "step": 3845
    },
    {
      "epoch": 0.9912461380020597,
      "grad_norm": 117.0,
      "learning_rate": 4.376930998970134e-09,
      "logits/chosen": -0.36503905057907104,
      "logits/rejected": -0.37187498807907104,
      "logps/chosen": -349.6000061035156,
      "logps/rejected": -277.3999938964844,
      "loss": 0.6422,
      "rewards/accuracies": 0.5425000190734863,
      "rewards/chosen": 0.26933592557907104,
      "rewards/margins": 0.12939453125,
      "rewards/rejected": 0.14003905653953552,
      "step": 3850
    },
    {
      "epoch": 0.9925334706488157,
      "grad_norm": 96.0,
      "learning_rate": 3.733264675592173e-09,
      "logits/chosen": -0.26689451932907104,
      "logits/rejected": -0.19121094048023224,
      "logps/chosen": -249.0,
      "logps/rejected": -223.0,
      "loss": 0.65,
      "rewards/accuracies": 0.5708333253860474,
      "rewards/chosen": 0.21633300185203552,
      "rewards/margins": 0.09624023735523224,
      "rewards/rejected": 0.11970214545726776,
      "step": 3855
    },
    {
      "epoch": 0.9938208032955715,
      "grad_norm": 125.5,
      "learning_rate": 3.089598352214212e-09,
      "logits/chosen": -0.29570311307907104,
      "logits/rejected": -0.3775390684604645,
      "logps/chosen": -298.79998779296875,
      "logps/rejected": -265.6000061035156,
      "loss": 0.6594,
      "rewards/accuracies": 0.6208333373069763,
      "rewards/chosen": 0.24296875298023224,
      "rewards/margins": 0.10043945163488388,
      "rewards/rejected": 0.14252929389476776,
      "step": 3860
    },
    {
      "epoch": 0.9951081359423275,
      "grad_norm": 105.0,
      "learning_rate": 2.445932028836251e-09,
      "logits/chosen": -0.38945311307907104,
      "logits/rejected": -0.3915039002895355,
      "logps/chosen": -303.20001220703125,
      "logps/rejected": -292.0,
      "loss": 0.657,
      "rewards/accuracies": 0.5517857074737549,
      "rewards/chosen": 0.20009765028953552,
      "rewards/margins": 0.09306640923023224,
      "rewards/rejected": 0.10688476264476776,
      "step": 3865
    },
    {
      "epoch": 0.9963954685890835,
      "grad_norm": 106.5,
      "learning_rate": 1.8022657054582903e-09,
      "logits/chosen": -0.26289063692092896,
      "logits/rejected": -0.2105712890625,
      "logps/chosen": -281.3999938964844,
      "logps/rejected": -266.6000061035156,
      "loss": 0.6406,
      "rewards/accuracies": 0.5525000095367432,
      "rewards/chosen": 0.3265624940395355,
      "rewards/margins": 0.13715820014476776,
      "rewards/rejected": 0.18974609673023224,
      "step": 3870
    },
    {
      "epoch": 0.9976828012358393,
      "grad_norm": 94.0,
      "learning_rate": 1.1585993820803295e-09,
      "logits/chosen": -0.29705810546875,
      "logits/rejected": -0.35624998807907104,
      "logps/chosen": -306.6000061035156,
      "logps/rejected": -263.79998779296875,
      "loss": 0.6625,
      "rewards/accuracies": 0.48249998688697815,
      "rewards/chosen": 0.21440429985523224,
      "rewards/margins": 0.08741454780101776,
      "rewards/rejected": 0.126953125,
      "step": 3875
    },
    {
      "epoch": 0.9989701338825953,
      "grad_norm": 460.0,
      "learning_rate": 5.149330587023687e-10,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.36699217557907104,
      "logps/chosen": -294.6000061035156,
      "logps/rejected": -216.39999389648438,
      "loss": 0.6844,
      "rewards/accuracies": 0.41138115525245667,
      "rewards/chosen": 0.13852539658546448,
      "rewards/margins": 0.03542480617761612,
      "rewards/rejected": 0.1026611328125,
      "step": 3880
    }
  ],
  "logging_steps": 5,
  "max_steps": 3884,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 4000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}