{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.0458550453186035,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.1378610134124756,
      "logits/rejected": -1.1131585836410522,
      "logps/chosen": -220.4049530029297,
      "logps/rejected": -234.8083953857422,
      "loss": 0.6929,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.0011935376096516848,
      "rewards/margins": 0.0005433893529698253,
      "rewards/rejected": 0.0006501483730971813,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 2.8662004470825195,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.122249722480774,
      "logits/rejected": -1.0652912855148315,
      "logps/chosen": -223.3093719482422,
      "logps/rejected": -215.4896697998047,
      "loss": 0.6878,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.010744137689471245,
      "rewards/margins": 0.010764424689114094,
      "rewards/rejected": -2.028709241130855e-05,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.1073992252349854,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.103829026222229,
      "logits/rejected": -1.021059274673462,
      "logps/chosen": -230.15866088867188,
      "logps/rejected": -228.9940948486328,
      "loss": 0.6777,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02945018745958805,
      "rewards/margins": 0.032092902809381485,
      "rewards/rejected": -0.002642714884132147,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 2.8836920261383057,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.131155252456665,
      "logits/rejected": -1.1667766571044922,
      "logps/chosen": -224.6150665283203,
      "logps/rejected": -239.52145385742188,
      "loss": 0.6578,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.06944439560174942,
      "rewards/margins": 0.07668272405862808,
      "rewards/rejected": -0.0072383270598948,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 2.933941602706909,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.182265043258667,
      "logits/rejected": -1.1075844764709473,
      "logps/chosen": -224.00808715820312,
      "logps/rejected": -253.9156951904297,
      "loss": 0.6349,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.11455950886011124,
      "rewards/margins": 0.13011686503887177,
      "rewards/rejected": -0.015557368285953999,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.208721399307251,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.1079695224761963,
      "logits/rejected": -1.1115219593048096,
      "logps/chosen": -214.77395629882812,
      "logps/rejected": -233.01913452148438,
      "loss": 0.6115,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15914058685302734,
      "rewards/margins": 0.19624650478363037,
      "rewards/rejected": -0.03710593655705452,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.0986759662628174,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.2299989461898804,
      "logits/rejected": -1.0735408067703247,
      "logps/chosen": -226.4742889404297,
      "logps/rejected": -225.53738403320312,
      "loss": 0.5806,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.2632412314414978,
      "rewards/margins": 0.3024232089519501,
      "rewards/rejected": -0.03918201103806496,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.538855791091919,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.2544300556182861,
      "logits/rejected": -1.1668885946273804,
      "logps/chosen": -217.96572875976562,
      "logps/rejected": -240.23434448242188,
      "loss": 0.543,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.3376604914665222,
      "rewards/margins": 0.44354838132858276,
      "rewards/rejected": -0.10588787496089935,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 2.8498013019561768,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.205204963684082,
      "logits/rejected": -1.1860382556915283,
      "logps/chosen": -222.1924591064453,
      "logps/rejected": -227.74472045898438,
      "loss": 0.5477,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.33458584547042847,
      "rewards/margins": 0.4299747347831726,
      "rewards/rejected": -0.09538889676332474,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 2.4346165657043457,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.158801794052124,
      "logits/rejected": -1.1945750713348389,
      "logps/chosen": -219.235107421875,
      "logps/rejected": -242.387451171875,
      "loss": 0.5045,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.4660046100616455,
      "rewards/margins": 0.6082803010940552,
      "rewards/rejected": -0.14227569103240967,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 2.311579704284668,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.2277817726135254,
      "logits/rejected": -1.1361002922058105,
      "logps/chosen": -204.9770050048828,
      "logps/rejected": -241.1087188720703,
      "loss": 0.4901,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.47566550970077515,
      "rewards/margins": 0.7237527966499329,
      "rewards/rejected": -0.24808719754219055,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.3512964248657227,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.2030401229858398,
      "logits/rejected": -1.1904528141021729,
      "logps/chosen": -228.5006561279297,
      "logps/rejected": -251.4223175048828,
      "loss": 0.4376,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.6135213971138,
      "rewards/margins": 0.9777547717094421,
      "rewards/rejected": -0.36423343420028687,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.3212287425994873,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.2506873607635498,
      "logits/rejected": -1.196251392364502,
      "logps/chosen": -240.0591583251953,
      "logps/rejected": -230.9004669189453,
      "loss": 0.4769,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.5166982412338257,
      "rewards/margins": 0.8954504728317261,
      "rewards/rejected": -0.3787522315979004,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 2.683375120162964,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.383887529373169,
      "logits/rejected": -1.2323031425476074,
      "logps/chosen": -209.1799774169922,
      "logps/rejected": -238.6314239501953,
      "loss": 0.4814,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.5157665014266968,
      "rewards/margins": 0.9230127334594727,
      "rewards/rejected": -0.4072461724281311,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.596590280532837,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.3620352745056152,
      "logits/rejected": -1.3067903518676758,
      "logps/chosen": -200.5843505859375,
      "logps/rejected": -233.67300415039062,
      "loss": 0.4318,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.5101022124290466,
      "rewards/margins": 0.9610568284988403,
      "rewards/rejected": -0.4509546160697937,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 2.6825714111328125,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.312425136566162,
      "logits/rejected": -1.2870181798934937,
      "logps/chosen": -230.2716064453125,
      "logps/rejected": -238.5307159423828,
      "loss": 0.4464,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.5834903717041016,
      "rewards/margins": 1.1369092464447021,
      "rewards/rejected": -0.5534188747406006,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 3.115396738052368,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.2816749811172485,
      "logits/rejected": -1.2261050939559937,
      "logps/chosen": -227.169189453125,
      "logps/rejected": -247.8328857421875,
      "loss": 0.3964,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.7111604809761047,
      "rewards/margins": 1.3225681781768799,
      "rewards/rejected": -0.6114078164100647,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.20292329788208,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.2855154275894165,
      "logits/rejected": -1.2214223146438599,
      "logps/chosen": -207.7841796875,
      "logps/rejected": -247.1141815185547,
      "loss": 0.3909,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6553568243980408,
      "rewards/margins": 1.384448766708374,
      "rewards/rejected": -0.729092001914978,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 3.6834919452667236,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.3217604160308838,
      "logits/rejected": -1.3131191730499268,
      "logps/chosen": -211.2960662841797,
      "logps/rejected": -242.7036590576172,
      "loss": 0.459,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.3602835536003113,
      "rewards/margins": 1.075905680656433,
      "rewards/rejected": -0.7156221866607666,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.337843179702759,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.4003416299819946,
      "logits/rejected": -1.3594415187835693,
      "logps/chosen": -263.1855773925781,
      "logps/rejected": -250.721435546875,
      "loss": 0.3688,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.5528009533882141,
      "rewards/margins": 1.4668482542037964,
      "rewards/rejected": -0.9140474200248718,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.303995132446289,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.319817066192627,
      "logits/rejected": -1.2342548370361328,
      "logps/chosen": -213.7783966064453,
      "logps/rejected": -256.16680908203125,
      "loss": 0.3626,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.6676830053329468,
      "rewards/margins": 1.5440034866333008,
      "rewards/rejected": -0.876320481300354,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 2.2961652278900146,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.34084153175354,
      "logits/rejected": -1.3188228607177734,
      "logps/chosen": -226.8044891357422,
      "logps/rejected": -235.49752807617188,
      "loss": 0.4056,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.4525774419307709,
      "rewards/margins": 1.4416332244873047,
      "rewards/rejected": -0.9890557527542114,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 2.7958457469940186,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.3018507957458496,
      "logits/rejected": -1.2472938299179077,
      "logps/chosen": -214.3036651611328,
      "logps/rejected": -240.1237030029297,
      "loss": 0.3764,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.5986575484275818,
      "rewards/margins": 1.6609241962432861,
      "rewards/rejected": -1.0622665882110596,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 3.2483537197113037,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.2724711894989014,
      "logits/rejected": -1.3170382976531982,
      "logps/chosen": -205.23342895507812,
      "logps/rejected": -252.8673095703125,
      "loss": 0.3533,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.7337419390678406,
      "rewards/margins": 1.7917591333389282,
      "rewards/rejected": -1.0580171346664429,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 2.846489191055298,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.4343109130859375,
      "logits/rejected": -1.3631373643875122,
      "logps/chosen": -198.54171752929688,
      "logps/rejected": -262.36981201171875,
      "loss": 0.37,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.6829321384429932,
      "rewards/margins": 1.8867518901824951,
      "rewards/rejected": -1.2038198709487915,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 1.6181107759475708,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.4376757144927979,
      "logits/rejected": -1.4121671915054321,
      "logps/chosen": -207.24801635742188,
      "logps/rejected": -277.3598327636719,
      "loss": 0.3248,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.5676002502441406,
      "rewards/margins": 1.9428207874298096,
      "rewards/rejected": -1.375220537185669,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 3.0397274494171143,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.4408540725708008,
      "logits/rejected": -1.3692197799682617,
      "logps/chosen": -222.3744659423828,
      "logps/rejected": -261.72308349609375,
      "loss": 0.41,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.29623842239379883,
      "rewards/margins": 1.554736614227295,
      "rewards/rejected": -1.2584980726242065,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 3.019113779067993,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.4328057765960693,
      "logits/rejected": -1.3377001285552979,
      "logps/chosen": -203.7570343017578,
      "logps/rejected": -245.04714965820312,
      "loss": 0.3437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4516337513923645,
      "rewards/margins": 1.7678340673446655,
      "rewards/rejected": -1.3162004947662354,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 2.6861343383789062,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.4812839031219482,
      "logits/rejected": -1.4032024145126343,
      "logps/chosen": -213.575439453125,
      "logps/rejected": -232.4373016357422,
      "loss": 0.3738,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.5343422293663025,
      "rewards/margins": 1.5936319828033447,
      "rewards/rejected": -1.0592896938323975,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.3076558113098145,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.5180320739746094,
      "logits/rejected": -1.362023115158081,
      "logps/chosen": -237.8686065673828,
      "logps/rejected": -263.8629455566406,
      "loss": 0.3664,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.5287350416183472,
      "rewards/margins": 2.087376356124878,
      "rewards/rejected": -1.5586414337158203,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 3.5192840099334717,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.5185763835906982,
      "logits/rejected": -1.4548755884170532,
      "logps/chosen": -215.61984252929688,
      "logps/rejected": -258.0099792480469,
      "loss": 0.385,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.38530558347702026,
      "rewards/margins": 1.934962511062622,
      "rewards/rejected": -1.5496571063995361,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 4.284816741943359,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.4868816137313843,
      "logits/rejected": -1.4484461545944214,
      "logps/chosen": -203.9530487060547,
      "logps/rejected": -259.6980895996094,
      "loss": 0.321,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.456205278635025,
      "rewards/margins": 2.133808135986328,
      "rewards/rejected": -1.6776031255722046,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.7927303314208984,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.3723137378692627,
      "logits/rejected": -1.3202406167984009,
      "logps/chosen": -216.36062622070312,
      "logps/rejected": -239.44772338867188,
      "loss": 0.3563,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.6298683881759644,
      "rewards/margins": 2.070138692855835,
      "rewards/rejected": -1.4402703046798706,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 3.5610151290893555,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.4355902671813965,
      "logits/rejected": -1.4587870836257935,
      "logps/chosen": -225.1851806640625,
      "logps/rejected": -241.0689239501953,
      "loss": 0.3507,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.5874021053314209,
      "rewards/margins": 1.9395253658294678,
      "rewards/rejected": -1.352123498916626,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 2.739126443862915,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.4393802881240845,
      "logits/rejected": -1.340178370475769,
      "logps/chosen": -218.3380584716797,
      "logps/rejected": -254.26895141601562,
      "loss": 0.3761,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.48171377182006836,
      "rewards/margins": 1.7582696676254272,
      "rewards/rejected": -1.2765560150146484,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 3.747058391571045,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.4342701435089111,
      "logits/rejected": -1.4054113626480103,
      "logps/chosen": -238.86172485351562,
      "logps/rejected": -251.49441528320312,
      "loss": 0.3594,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.5655156373977661,
      "rewards/margins": 2.102660655975342,
      "rewards/rejected": -1.5371451377868652,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 2.4808998107910156,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.4115904569625854,
      "logits/rejected": -1.3571182489395142,
      "logps/chosen": -208.3723602294922,
      "logps/rejected": -235.1692657470703,
      "loss": 0.3642,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.47888022661209106,
      "rewards/margins": 1.8599073886871338,
      "rewards/rejected": -1.3810272216796875,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 3.6028048992156982,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.3889703750610352,
      "logits/rejected": -1.3901216983795166,
      "logps/chosen": -204.32888793945312,
      "logps/rejected": -242.7096710205078,
      "loss": 0.3413,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.5300948023796082,
      "rewards/margins": 2.064892530441284,
      "rewards/rejected": -1.5347976684570312,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 2.390690326690674,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.3985008001327515,
      "logits/rejected": -1.384414553642273,
      "logps/chosen": -201.96835327148438,
      "logps/rejected": -274.15191650390625,
      "loss": 0.2784,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9769827127456665,
      "rewards/margins": 2.9426965713500977,
      "rewards/rejected": -1.9657137393951416,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 3.0497686862945557,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.3238798379898071,
      "logits/rejected": -1.3403199911117554,
      "logps/chosen": -263.2410583496094,
      "logps/rejected": -245.2187957763672,
      "loss": 0.3321,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.7236034870147705,
      "rewards/margins": 2.1606106758117676,
      "rewards/rejected": -1.4370070695877075,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 3.0012412071228027,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.5537317991256714,
      "logits/rejected": -1.4608463048934937,
      "logps/chosen": -200.6154022216797,
      "logps/rejected": -248.28836059570312,
      "loss": 0.3128,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.4444311261177063,
      "rewards/margins": 2.3631651401519775,
      "rewards/rejected": -1.918733835220337,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.235403060913086,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.5298961400985718,
      "logits/rejected": -1.4516569375991821,
      "logps/chosen": -218.44970703125,
      "logps/rejected": -259.55633544921875,
      "loss": 0.2824,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.8634427785873413,
      "rewards/margins": 2.612311840057373,
      "rewards/rejected": -1.7488689422607422,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 2.519658088684082,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.4698635339736938,
      "logits/rejected": -1.380286455154419,
      "logps/chosen": -204.63861083984375,
      "logps/rejected": -256.0403137207031,
      "loss": 0.2968,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7937697172164917,
      "rewards/margins": 2.663573980331421,
      "rewards/rejected": -1.8698041439056396,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 4.095766544342041,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.463175892829895,
      "logits/rejected": -1.4339282512664795,
      "logps/chosen": -244.7183837890625,
      "logps/rejected": -261.1028747558594,
      "loss": 0.3653,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.38948220014572144,
      "rewards/margins": 2.2226202487945557,
      "rewards/rejected": -1.8331377506256104,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.851942300796509,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.3653357028961182,
      "logits/rejected": -1.330588936805725,
      "logps/chosen": -217.13760375976562,
      "logps/rejected": -240.1661376953125,
      "loss": 0.2874,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.732596755027771,
      "rewards/margins": 2.40860915184021,
      "rewards/rejected": -1.676012396812439,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 3.4217729568481445,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.4823631048202515,
      "logits/rejected": -1.495508074760437,
      "logps/chosen": -212.22323608398438,
      "logps/rejected": -263.6670837402344,
      "loss": 0.2746,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.5829408168792725,
      "rewards/margins": 2.398766040802002,
      "rewards/rejected": -1.8158252239227295,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 2.7907536029815674,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.5069305896759033,
      "logits/rejected": -1.464449167251587,
      "logps/chosen": -193.39453125,
      "logps/rejected": -249.33828735351562,
      "loss": 0.3033,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.5029724836349487,
      "rewards/margins": 2.535928726196289,
      "rewards/rejected": -2.03295636177063,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.6872572898864746,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.4873557090759277,
      "logits/rejected": -1.4141255617141724,
      "logps/chosen": -218.90084838867188,
      "logps/rejected": -248.9842071533203,
      "loss": 0.2922,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.41843342781066895,
      "rewards/margins": 2.512903928756714,
      "rewards/rejected": -2.094470739364624,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 2.2702150344848633,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.500101923942566,
      "logits/rejected": -1.3862520456314087,
      "logps/chosen": -211.1094970703125,
      "logps/rejected": -270.4104919433594,
      "loss": 0.2471,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5343133211135864,
      "rewards/margins": 2.90330171585083,
      "rewards/rejected": -2.368988513946533,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 3.828376054763794,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.538053274154663,
      "logits/rejected": -1.4550931453704834,
      "logps/chosen": -237.7978973388672,
      "logps/rejected": -277.4515686035156,
      "loss": 0.2882,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.445631206035614,
      "rewards/margins": 3.2756295204162598,
      "rewards/rejected": -2.829998254776001,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 7.073206901550293,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.6128956079483032,
      "logits/rejected": -1.5589414834976196,
      "logps/chosen": -218.8146514892578,
      "logps/rejected": -247.97824096679688,
      "loss": 0.2747,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.433823823928833,
      "rewards/margins": 2.5846328735351562,
      "rewards/rejected": -2.150808811187744,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 3.671692132949829,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.5363198518753052,
      "logits/rejected": -1.432539939880371,
      "logps/chosen": -194.21780395507812,
      "logps/rejected": -259.4532470703125,
      "loss": 0.2959,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.29416778683662415,
      "rewards/margins": 2.8601794242858887,
      "rewards/rejected": -2.566011905670166,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 4.124833583831787,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.5194003582000732,
      "logits/rejected": -1.4758837223052979,
      "logps/chosen": -222.5775146484375,
      "logps/rejected": -270.2529296875,
      "loss": 0.2939,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.4511001706123352,
      "rewards/margins": 2.7838187217712402,
      "rewards/rejected": -2.3327183723449707,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 3.6710762977600098,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.5958110094070435,
      "logits/rejected": -1.4789632558822632,
      "logps/chosen": -229.09042358398438,
      "logps/rejected": -255.7130889892578,
      "loss": 0.2994,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.42495328187942505,
      "rewards/margins": 2.7729554176330566,
      "rewards/rejected": -2.3480019569396973,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 2.481130838394165,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.3712882995605469,
      "logits/rejected": -1.3942594528198242,
      "logps/chosen": -221.78616333007812,
      "logps/rejected": -271.4743347167969,
      "loss": 0.2772,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.5858339071273804,
      "rewards/margins": 2.6469860076904297,
      "rewards/rejected": -2.0611519813537598,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 3.2142515182495117,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.5062901973724365,
      "logits/rejected": -1.4870845079421997,
      "logps/chosen": -219.1357421875,
      "logps/rejected": -263.96307373046875,
      "loss": 0.2826,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.28349483013153076,
      "rewards/margins": 2.623389482498169,
      "rewards/rejected": -2.3398945331573486,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.7220044136047363,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.4841105937957764,
      "logits/rejected": -1.4876569509506226,
      "logps/chosen": -227.7067108154297,
      "logps/rejected": -277.1427917480469,
      "loss": 0.2772,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.5623529553413391,
      "rewards/margins": 2.996119260787964,
      "rewards/rejected": -2.4337663650512695,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 3.9186594486236572,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.5781495571136475,
      "logits/rejected": -1.5577192306518555,
      "logps/chosen": -215.00234985351562,
      "logps/rejected": -272.5433044433594,
      "loss": 0.3279,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.29643192887306213,
      "rewards/margins": 2.8020668029785156,
      "rewards/rejected": -2.5056347846984863,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 10.114188194274902,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.5302282571792603,
      "logits/rejected": -1.520778775215149,
      "logps/chosen": -238.6151123046875,
      "logps/rejected": -278.87847900390625,
      "loss": 0.3095,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.2077063024044037,
      "rewards/margins": 2.5661838054656982,
      "rewards/rejected": -2.3584771156311035,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 2.399756669998169,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.4977320432662964,
      "logits/rejected": -1.389096975326538,
      "logps/chosen": -215.07699584960938,
      "logps/rejected": -257.64459228515625,
      "loss": 0.267,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.5605412125587463,
      "rewards/margins": 3.175250291824341,
      "rewards/rejected": -2.614708662033081,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 2.632978916168213,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.5880491733551025,
      "logits/rejected": -1.4870240688323975,
      "logps/chosen": -235.69680786132812,
      "logps/rejected": -260.10711669921875,
      "loss": 0.3318,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.39901071786880493,
      "rewards/margins": 2.604020118713379,
      "rewards/rejected": -2.2050094604492188,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 2.3961174488067627,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.4467146396636963,
      "logits/rejected": -1.3576264381408691,
      "logps/chosen": -224.96505737304688,
      "logps/rejected": -259.876953125,
      "loss": 0.2734,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.3828810751438141,
      "rewards/margins": 2.681600570678711,
      "rewards/rejected": -2.298719644546509,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 3.1757161617279053,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.527538537979126,
      "logits/rejected": -1.528403639793396,
      "logps/chosen": -222.27877807617188,
      "logps/rejected": -258.99517822265625,
      "loss": 0.3181,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3306906521320343,
      "rewards/margins": 2.67979097366333,
      "rewards/rejected": -2.349100351333618,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 2.3359498977661133,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.5962450504302979,
      "logits/rejected": -1.4786200523376465,
      "logps/chosen": -220.0579376220703,
      "logps/rejected": -273.56915283203125,
      "loss": 0.2359,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.6050397157669067,
      "rewards/margins": 3.208775758743286,
      "rewards/rejected": -2.60373592376709,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 4.228616237640381,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.4205467700958252,
      "logits/rejected": -1.4639966487884521,
      "logps/chosen": -235.7682647705078,
      "logps/rejected": -263.72576904296875,
      "loss": 0.2699,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.4260522723197937,
      "rewards/margins": 2.8981242179870605,
      "rewards/rejected": -2.472072124481201,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 3.994901180267334,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.6341301202774048,
      "logits/rejected": -1.5283631086349487,
      "logps/chosen": -211.6797332763672,
      "logps/rejected": -266.39544677734375,
      "loss": 0.2507,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.3721843361854553,
      "rewards/margins": 3.0854640007019043,
      "rewards/rejected": -2.7132792472839355,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 2.890618085861206,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.5078494548797607,
      "logits/rejected": -1.4805063009262085,
      "logps/chosen": -214.09115600585938,
      "logps/rejected": -252.56002807617188,
      "loss": 0.2518,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.32611769437789917,
      "rewards/margins": 2.888943910598755,
      "rewards/rejected": -2.562826633453369,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 4.099746227264404,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.5950417518615723,
      "logits/rejected": -1.467929482460022,
      "logps/chosen": -230.2890167236328,
      "logps/rejected": -252.35507202148438,
      "loss": 0.3413,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.19502702355384827,
      "rewards/margins": 2.6124227046966553,
      "rewards/rejected": -2.417395830154419,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 2.731096029281616,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.4848092794418335,
      "logits/rejected": -1.4913749694824219,
      "logps/chosen": -222.9447784423828,
      "logps/rejected": -275.4829406738281,
      "loss": 0.2407,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.09350375831127167,
      "rewards/margins": 3.030643939971924,
      "rewards/rejected": -2.9371399879455566,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 2.716784954071045,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.5931785106658936,
      "logits/rejected": -1.445983648300171,
      "logps/chosen": -231.0179443359375,
      "logps/rejected": -247.4789276123047,
      "loss": 0.2612,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.10627200454473495,
      "rewards/margins": 2.9888815879821777,
      "rewards/rejected": -2.8826098442077637,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 4.22883939743042,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.6081180572509766,
      "logits/rejected": -1.557462215423584,
      "logps/chosen": -217.7015838623047,
      "logps/rejected": -258.1307678222656,
      "loss": 0.2923,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.25779181718826294,
      "rewards/margins": 2.9773926734924316,
      "rewards/rejected": -2.7196006774902344,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.020744562149048,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.454756259918213,
      "logits/rejected": -1.4177753925323486,
      "logps/chosen": -209.04257202148438,
      "logps/rejected": -252.439208984375,
      "loss": 0.2649,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.3628079891204834,
      "rewards/margins": 2.9099552631378174,
      "rewards/rejected": -2.547147274017334,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 2.0658082962036133,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.460191011428833,
      "logits/rejected": -1.4749881029129028,
      "logps/chosen": -223.18679809570312,
      "logps/rejected": -285.80413818359375,
      "loss": 0.1976,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.4657130241394043,
      "rewards/margins": 3.4711146354675293,
      "rewards/rejected": -3.005401611328125,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 3.1469435691833496,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.5563991069793701,
      "logits/rejected": -1.5385240316390991,
      "logps/chosen": -225.7266387939453,
      "logps/rejected": -277.9988708496094,
      "loss": 0.2847,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.30648013949394226,
      "rewards/margins": 2.885929822921753,
      "rewards/rejected": -2.5794498920440674,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.111003875732422,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.6107202768325806,
      "logits/rejected": -1.537246823310852,
      "logps/chosen": -238.1480255126953,
      "logps/rejected": -271.89373779296875,
      "loss": 0.2056,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.4036385118961334,
      "rewards/margins": 3.371934413909912,
      "rewards/rejected": -2.9682960510253906,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 3.8910281658172607,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.5961246490478516,
      "logits/rejected": -1.5346485376358032,
      "logps/chosen": -212.40737915039062,
      "logps/rejected": -242.66879272460938,
      "loss": 0.2987,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.1651727855205536,
      "rewards/margins": 2.9011754989624023,
      "rewards/rejected": -2.7360026836395264,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.2986905574798584,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.5123292207717896,
      "logits/rejected": -1.4317362308502197,
      "logps/chosen": -215.16653442382812,
      "logps/rejected": -250.494873046875,
      "loss": 0.297,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.2533227205276489,
      "rewards/margins": 3.0472066402435303,
      "rewards/rejected": -2.793883800506592,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 2.390202283859253,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.4423052072525024,
      "logits/rejected": -1.4100033044815063,
      "logps/chosen": -220.01803588867188,
      "logps/rejected": -267.92913818359375,
      "loss": 0.273,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.3290650248527527,
      "rewards/margins": 3.225865125656128,
      "rewards/rejected": -2.8968000411987305,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.8150224685668945,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.522770643234253,
      "logits/rejected": -1.5387157201766968,
      "logps/chosen": -221.63082885742188,
      "logps/rejected": -273.49322509765625,
      "loss": 0.2344,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.5276051163673401,
      "rewards/margins": 3.0984864234924316,
      "rewards/rejected": -2.5708811283111572,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 3.9126734733581543,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.5304027795791626,
      "logits/rejected": -1.4529463052749634,
      "logps/chosen": -212.8784942626953,
      "logps/rejected": -272.4510803222656,
      "loss": 0.2645,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 0.4774355888366699,
      "rewards/margins": 3.302903413772583,
      "rewards/rejected": -2.825467348098755,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 1.941848635673523,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.5869189500808716,
      "logits/rejected": -1.520146131515503,
      "logps/chosen": -206.26156616210938,
      "logps/rejected": -257.34173583984375,
      "loss": 0.1988,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.7204893827438354,
      "rewards/margins": 3.477137327194214,
      "rewards/rejected": -2.756648063659668,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 3.2625303268432617,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.5185470581054688,
      "logits/rejected": -1.4453086853027344,
      "logps/chosen": -204.9614715576172,
      "logps/rejected": -272.30706787109375,
      "loss": 0.2353,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.33646291494369507,
      "rewards/margins": 3.2520689964294434,
      "rewards/rejected": -2.9156060218811035,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 2.795285224914551,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.4114835262298584,
      "logits/rejected": -1.4512608051300049,
      "logps/chosen": -217.20480346679688,
      "logps/rejected": -261.76727294921875,
      "loss": 0.2568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.555108904838562,
      "rewards/margins": 3.5297584533691406,
      "rewards/rejected": -2.9746499061584473,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 4.065403461456299,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.4818923473358154,
      "logits/rejected": -1.4995462894439697,
      "logps/chosen": -209.0481719970703,
      "logps/rejected": -255.38623046875,
      "loss": 0.2527,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.5125702023506165,
      "rewards/margins": 2.971860647201538,
      "rewards/rejected": -2.459290027618408,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 4.261399745941162,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.6032196283340454,
      "logits/rejected": -1.5087716579437256,
      "logps/chosen": -229.303955078125,
      "logps/rejected": -273.9316711425781,
      "loss": 0.2773,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.056284308433532715,
      "rewards/margins": 2.942106246948242,
      "rewards/rejected": -2.885822057723999,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 1.7086869478225708,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.5353028774261475,
      "logits/rejected": -1.4393786191940308,
      "logps/chosen": -208.2356719970703,
      "logps/rejected": -272.1566467285156,
      "loss": 0.23,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.37963634729385376,
      "rewards/margins": 3.723524808883667,
      "rewards/rejected": -3.343888521194458,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 2.9886105060577393,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.6221126317977905,
      "logits/rejected": -1.5396804809570312,
      "logps/chosen": -219.42343139648438,
      "logps/rejected": -275.2545471191406,
      "loss": 0.2345,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.3662927746772766,
      "rewards/margins": 3.762205123901367,
      "rewards/rejected": -3.3959121704101562,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 2.806347370147705,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.5537915229797363,
      "logits/rejected": -1.4640367031097412,
      "logps/chosen": -222.9738006591797,
      "logps/rejected": -288.2947692871094,
      "loss": 0.244,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.5269616842269897,
      "rewards/margins": 3.6222381591796875,
      "rewards/rejected": -3.095276355743408,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 3.4051554203033447,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.5768224000930786,
      "logits/rejected": -1.4694640636444092,
      "logps/chosen": -222.8614959716797,
      "logps/rejected": -276.8932189941406,
      "loss": 0.2644,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.48596447706222534,
      "rewards/margins": 3.459862232208252,
      "rewards/rejected": -2.9738974571228027,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 1.5982437133789062,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.5265693664550781,
      "logits/rejected": -1.4454200267791748,
      "logps/chosen": -248.08175659179688,
      "logps/rejected": -263.53564453125,
      "loss": 0.2418,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.9819803237915039,
      "rewards/margins": 3.853539228439331,
      "rewards/rejected": -2.8715591430664062,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 3.993597984313965,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.6176679134368896,
      "logits/rejected": -1.5645582675933838,
      "logps/chosen": -229.76318359375,
      "logps/rejected": -262.1435852050781,
      "loss": 0.29,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.5866170525550842,
      "rewards/margins": 3.056516408920288,
      "rewards/rejected": -2.4698996543884277,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 2.713717222213745,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.4770532846450806,
      "logits/rejected": -1.4442005157470703,
      "logps/chosen": -208.3784942626953,
      "logps/rejected": -270.6184387207031,
      "loss": 0.2425,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.6007565855979919,
      "rewards/margins": 3.5067782402038574,
      "rewards/rejected": -2.9060218334198,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 2.6341986656188965,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.5352776050567627,
      "logits/rejected": -1.5059245824813843,
      "logps/chosen": -231.0626678466797,
      "logps/rejected": -271.16015625,
      "loss": 0.2402,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5198360681533813,
      "rewards/margins": 3.263561964035034,
      "rewards/rejected": -2.7437267303466797,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 2.800222396850586,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.507896900177002,
      "logits/rejected": -1.4542078971862793,
      "logps/chosen": -223.8075408935547,
      "logps/rejected": -264.8511657714844,
      "loss": 0.2531,
      "rewards/accuracies": 0.8881579041481018,
      "rewards/chosen": 0.7271106839179993,
      "rewards/margins": 3.1902239322662354,
      "rewards/rejected": -2.4631130695343018,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 1.8111785650253296,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.5138390064239502,
      "logits/rejected": -1.5100157260894775,
      "logps/chosen": -222.42367553710938,
      "logps/rejected": -270.0904235839844,
      "loss": 0.1977,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.9269617199897766,
      "rewards/margins": 3.785017728805542,
      "rewards/rejected": -2.8580563068389893,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 2.3462116718292236,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.4097557067871094,
      "logits/rejected": -1.2989996671676636,
      "logps/chosen": -242.18115234375,
      "logps/rejected": -254.24459838867188,
      "loss": 0.2003,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.9647325277328491,
      "rewards/margins": 3.434415102005005,
      "rewards/rejected": -2.4696826934814453,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 2.4273829460144043,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.6104650497436523,
      "logits/rejected": -1.5876798629760742,
      "logps/chosen": -199.7315673828125,
      "logps/rejected": -262.48309326171875,
      "loss": 0.2409,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.648854672908783,
      "rewards/margins": 3.4328675270080566,
      "rewards/rejected": -2.78401255607605,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 2.23551344871521,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.4702277183532715,
      "logits/rejected": -1.3995940685272217,
      "logps/chosen": -241.27835083007812,
      "logps/rejected": -275.19061279296875,
      "loss": 0.1657,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.8960878252983093,
      "rewards/margins": 3.913990020751953,
      "rewards/rejected": -3.01790189743042,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 2.930630922317505,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.5484654903411865,
      "logits/rejected": -1.5531933307647705,
      "logps/chosen": -212.1646728515625,
      "logps/rejected": -272.5528564453125,
      "loss": 0.2003,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.4871641993522644,
      "rewards/margins": 3.573843002319336,
      "rewards/rejected": -3.0866787433624268,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 1.9930554628372192,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.5165655612945557,
      "logits/rejected": -1.48264479637146,
      "logps/chosen": -225.77340698242188,
      "logps/rejected": -268.895751953125,
      "loss": 0.1915,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.714760959148407,
      "rewards/margins": 3.824763059616089,
      "rewards/rejected": -3.110002040863037,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 3.423259973526001,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.5718601942062378,
      "logits/rejected": -1.503844976425171,
      "logps/chosen": -208.6433563232422,
      "logps/rejected": -278.9520263671875,
      "loss": 0.2557,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4504665434360504,
      "rewards/margins": 3.5247490406036377,
      "rewards/rejected": -3.0742828845977783,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 2.5199198722839355,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.4899485111236572,
      "logits/rejected": -1.4122374057769775,
      "logps/chosen": -227.7731170654297,
      "logps/rejected": -277.1492004394531,
      "loss": 0.1901,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.5795149207115173,
      "rewards/margins": 4.235302925109863,
      "rewards/rejected": -3.6557884216308594,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 3.217991590499878,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.4657047986984253,
      "logits/rejected": -1.423705816268921,
      "logps/chosen": -208.4857177734375,
      "logps/rejected": -274.12799072265625,
      "loss": 0.1808,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.6760324239730835,
      "rewards/margins": 3.973620653152466,
      "rewards/rejected": -3.2975876331329346,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 2.835545301437378,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.5454612970352173,
      "logits/rejected": -1.5034112930297852,
      "logps/chosen": -229.1444854736328,
      "logps/rejected": -275.30047607421875,
      "loss": 0.1639,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3288564383983612,
      "rewards/margins": 3.9869868755340576,
      "rewards/rejected": -3.658130168914795,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 2.960411787033081,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.5239708423614502,
      "logits/rejected": -1.500842571258545,
      "logps/chosen": -209.4385528564453,
      "logps/rejected": -283.56854248046875,
      "loss": 0.1454,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5432895421981812,
      "rewards/margins": 4.694332599639893,
      "rewards/rejected": -4.15104341506958,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 2.7328364849090576,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.6704280376434326,
      "logits/rejected": -1.5336025953292847,
      "logps/chosen": -218.2663116455078,
      "logps/rejected": -287.0753479003906,
      "loss": 0.1882,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.1658170372247696,
      "rewards/margins": 4.253686428070068,
      "rewards/rejected": -4.087869167327881,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 3.2963521480560303,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.605804443359375,
      "logits/rejected": -1.5619615316390991,
      "logps/chosen": -203.71310424804688,
      "logps/rejected": -267.4145812988281,
      "loss": 0.2009,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.14755108952522278,
      "rewards/margins": 4.140759468078613,
      "rewards/rejected": -4.288310527801514,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 4.603735446929932,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.6345436573028564,
      "logits/rejected": -1.5837303400039673,
      "logps/chosen": -223.50537109375,
      "logps/rejected": -281.343017578125,
      "loss": 0.183,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.307804673910141,
      "rewards/margins": 4.121958255767822,
      "rewards/rejected": -4.429762840270996,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 3.5765788555145264,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.5483214855194092,
      "logits/rejected": -1.451832890510559,
      "logps/chosen": -228.17013549804688,
      "logps/rejected": -270.2801818847656,
      "loss": 0.2415,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.13486400246620178,
      "rewards/margins": 3.71563982963562,
      "rewards/rejected": -3.850503921508789,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 2.7986321449279785,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.5429253578186035,
      "logits/rejected": -1.5843229293823242,
      "logps/chosen": -225.6952362060547,
      "logps/rejected": -317.2574157714844,
      "loss": 0.1625,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.02160305716097355,
      "rewards/margins": 5.061923027038574,
      "rewards/rejected": -5.083526611328125,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 3.046375274658203,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.5679842233657837,
      "logits/rejected": -1.565364956855774,
      "logps/chosen": -221.58963012695312,
      "logps/rejected": -277.75091552734375,
      "loss": 0.2252,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.12510250508785248,
      "rewards/margins": 3.9496397972106934,
      "rewards/rejected": -4.074742317199707,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 2.593656301498413,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.3937714099884033,
      "logits/rejected": -1.4195549488067627,
      "logps/chosen": -221.0165252685547,
      "logps/rejected": -310.3327331542969,
      "loss": 0.1942,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.2540460228919983,
      "rewards/margins": 4.681498050689697,
      "rewards/rejected": -4.935544490814209,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 4.23615026473999,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.5223419666290283,
      "logits/rejected": -1.4661571979522705,
      "logps/chosen": -217.5615234375,
      "logps/rejected": -292.19683837890625,
      "loss": 0.1682,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.1407851129770279,
      "rewards/margins": 4.647849082946777,
      "rewards/rejected": -4.507063865661621,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 3.9983882904052734,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.5181207656860352,
      "logits/rejected": -1.4914724826812744,
      "logps/chosen": -210.10433959960938,
      "logps/rejected": -273.66107177734375,
      "loss": 0.1985,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.14912539720535278,
      "rewards/margins": 4.561871528625488,
      "rewards/rejected": -4.412745952606201,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 4.649803161621094,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.4583940505981445,
      "logits/rejected": -1.4670298099517822,
      "logps/chosen": -239.1728057861328,
      "logps/rejected": -291.33392333984375,
      "loss": 0.2066,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.08136308193206787,
      "rewards/margins": 4.34789514541626,
      "rewards/rejected": -4.429257869720459,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 2.3416860103607178,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.5698027610778809,
      "logits/rejected": -1.491105318069458,
      "logps/chosen": -220.2606658935547,
      "logps/rejected": -281.61572265625,
      "loss": 0.1992,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.2473183125257492,
      "rewards/margins": 4.019981384277344,
      "rewards/rejected": -4.267299175262451,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 2.0436906814575195,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.5137239694595337,
      "logits/rejected": -1.465408444404602,
      "logps/chosen": -221.51223754882812,
      "logps/rejected": -292.5458679199219,
      "loss": 0.1629,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.05579204484820366,
      "rewards/margins": 4.571835517883301,
      "rewards/rejected": -4.516043663024902,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 5.584434509277344,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.53299880027771,
      "logits/rejected": -1.477111577987671,
      "logps/chosen": -277.4779968261719,
      "logps/rejected": -267.1672668457031,
      "loss": 0.2395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.21413402259349823,
      "rewards/margins": 4.064342021942139,
      "rewards/rejected": -3.850208282470703,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 3.1149520874023438,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.569455862045288,
      "logits/rejected": -1.5134979486465454,
      "logps/chosen": -219.28079223632812,
      "logps/rejected": -292.9593811035156,
      "loss": 0.1917,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.11000633239746094,
      "rewards/margins": 4.244585037231445,
      "rewards/rejected": -4.354591369628906,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 3.55899977684021,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.4652760028839111,
      "logits/rejected": -1.428091287612915,
      "logps/chosen": -225.0402069091797,
      "logps/rejected": -285.8553466796875,
      "loss": 0.183,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.03203168511390686,
      "rewards/margins": 4.701508522033691,
      "rewards/rejected": -4.7335405349731445,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 3.1927192211151123,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.5257372856140137,
      "logits/rejected": -1.502838134765625,
      "logps/chosen": -232.27536010742188,
      "logps/rejected": -305.02679443359375,
      "loss": 0.155,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.1784450113773346,
      "rewards/margins": 5.102679252624512,
      "rewards/rejected": -5.281124114990234,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 2.593752861022949,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.5107977390289307,
      "logits/rejected": -1.5205315351486206,
      "logps/chosen": -229.0679473876953,
      "logps/rejected": -298.5066223144531,
      "loss": 0.1543,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.29753953218460083,
      "rewards/margins": 4.581008434295654,
      "rewards/rejected": -4.8785481452941895,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 3.3902955055236816,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.6671390533447266,
      "logits/rejected": -1.50319504737854,
      "logps/chosen": -244.96578979492188,
      "logps/rejected": -264.3321228027344,
      "loss": 0.2094,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.17131295800209045,
      "rewards/margins": 3.743359088897705,
      "rewards/rejected": -3.9146721363067627,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 1.5384914875030518,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.521968126296997,
      "logits/rejected": -1.4370750188827515,
      "logps/chosen": -223.0912628173828,
      "logps/rejected": -272.32012939453125,
      "loss": 0.1685,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.06771224737167358,
      "rewards/margins": 4.779266834259033,
      "rewards/rejected": -4.711554527282715,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 4.687516212463379,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.4829010963439941,
      "logits/rejected": -1.4064357280731201,
      "logps/chosen": -237.6384735107422,
      "logps/rejected": -287.4407043457031,
      "loss": 0.1972,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.37526464462280273,
      "rewards/margins": 4.6708197593688965,
      "rewards/rejected": -5.046084880828857,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 3.4469211101531982,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.5600812435150146,
      "logits/rejected": -1.5160925388336182,
      "logps/chosen": -219.6197967529297,
      "logps/rejected": -277.3929748535156,
      "loss": 0.1902,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6968265771865845,
      "rewards/margins": 5.039797782897949,
      "rewards/rejected": -5.736624240875244,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 2.8947951793670654,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.5564571619033813,
      "logits/rejected": -1.4424645900726318,
      "logps/chosen": -253.28347778320312,
      "logps/rejected": -278.94183349609375,
      "loss": 0.2277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6225299835205078,
      "rewards/margins": 4.102585792541504,
      "rewards/rejected": -4.725115776062012,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 1.9876494407653809,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.5709205865859985,
      "logits/rejected": -1.5445259809494019,
      "logps/chosen": -215.19046020507812,
      "logps/rejected": -280.66961669921875,
      "loss": 0.2121,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4023857116699219,
      "rewards/margins": 4.108145236968994,
      "rewards/rejected": -4.510531425476074,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 5.166397571563721,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.5972050428390503,
      "logits/rejected": -1.5269817113876343,
      "logps/chosen": -239.1528778076172,
      "logps/rejected": -287.1815490722656,
      "loss": 0.2613,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5880186557769775,
      "rewards/margins": 4.12716007232666,
      "rewards/rejected": -4.715178489685059,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 3.5479846000671387,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.521431803703308,
      "logits/rejected": -1.463780403137207,
      "logps/chosen": -232.4151611328125,
      "logps/rejected": -268.7264099121094,
      "loss": 0.2158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.15852993726730347,
      "rewards/margins": 4.376145839691162,
      "rewards/rejected": -4.534675598144531,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 4.420405387878418,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.6656224727630615,
      "logits/rejected": -1.5566421747207642,
      "logps/chosen": -215.8498992919922,
      "logps/rejected": -274.35357666015625,
      "loss": 0.2414,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.14214067161083221,
      "rewards/margins": 4.248762130737305,
      "rewards/rejected": -4.390902519226074,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 2.908594846725464,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.4593592882156372,
      "logits/rejected": -1.4189205169677734,
      "logps/chosen": -204.3805694580078,
      "logps/rejected": -258.9472351074219,
      "loss": 0.1519,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.021609745919704437,
      "rewards/margins": 4.725183963775635,
      "rewards/rejected": -4.746793270111084,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 5.427767276763916,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.5551220178604126,
      "logits/rejected": -1.5175118446350098,
      "logps/chosen": -219.32080078125,
      "logps/rejected": -286.12615966796875,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3829983174800873,
      "rewards/margins": 4.652920722961426,
      "rewards/rejected": -5.035919189453125,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 2.0883243083953857,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.5865761041641235,
      "logits/rejected": -1.5060150623321533,
      "logps/chosen": -211.5963592529297,
      "logps/rejected": -292.7962951660156,
      "loss": 0.1581,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3247772455215454,
      "rewards/margins": 5.109382152557373,
      "rewards/rejected": -5.434159278869629,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 4.16923189163208,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.5806046724319458,
      "logits/rejected": -1.4857282638549805,
      "logps/chosen": -213.30587768554688,
      "logps/rejected": -269.56365966796875,
      "loss": 0.1641,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5577279329299927,
      "rewards/margins": 4.701187610626221,
      "rewards/rejected": -5.258915901184082,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 4.474754810333252,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.4321134090423584,
      "logits/rejected": -1.477826714515686,
      "logps/chosen": -216.34506225585938,
      "logps/rejected": -296.2746887207031,
      "loss": 0.2041,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.43209022283554077,
      "rewards/margins": 4.764926433563232,
      "rewards/rejected": -5.197016716003418,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 3.623042583465576,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.5734399557113647,
      "logits/rejected": -1.5528452396392822,
      "logps/chosen": -245.42086791992188,
      "logps/rejected": -285.8833923339844,
      "loss": 0.2116,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6044676899909973,
      "rewards/margins": 4.195218563079834,
      "rewards/rejected": -4.799685955047607,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 3.855921506881714,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.385648488998413,
      "logits/rejected": -1.407254934310913,
      "logps/chosen": -238.00991821289062,
      "logps/rejected": -293.28619384765625,
      "loss": 0.2153,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.36002954840660095,
      "rewards/margins": 4.525365352630615,
      "rewards/rejected": -4.885395050048828,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 3.164611339569092,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.5446698665618896,
      "logits/rejected": -1.4527463912963867,
      "logps/chosen": -229.75595092773438,
      "logps/rejected": -275.39825439453125,
      "loss": 0.1813,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.625076413154602,
      "rewards/margins": 4.960640907287598,
      "rewards/rejected": -5.585717678070068,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 2.971756935119629,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.620855689048767,
      "logits/rejected": -1.5859421491622925,
      "logps/chosen": -232.5053253173828,
      "logps/rejected": -301.48516845703125,
      "loss": 0.1653,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6067575216293335,
      "rewards/margins": 4.746621131896973,
      "rewards/rejected": -5.353378772735596,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 5.307252883911133,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.542690396308899,
      "logits/rejected": -1.5111274719238281,
      "logps/chosen": -229.7939453125,
      "logps/rejected": -281.5523986816406,
      "loss": 0.2582,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8791757822036743,
      "rewards/margins": 4.178490161895752,
      "rewards/rejected": -5.057665824890137,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 3.5124833583831787,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.54134202003479,
      "logits/rejected": -1.527388572692871,
      "logps/chosen": -221.4246368408203,
      "logps/rejected": -286.0513000488281,
      "loss": 0.1538,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.41137829422950745,
      "rewards/margins": 5.438130855560303,
      "rewards/rejected": -5.849509239196777,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 5.473639011383057,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.5290729999542236,
      "logits/rejected": -1.4904533624649048,
      "logps/chosen": -212.76895141601562,
      "logps/rejected": -281.6219482421875,
      "loss": 0.1965,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6981285810470581,
      "rewards/margins": 4.48247766494751,
      "rewards/rejected": -5.180605888366699,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 5.93931770324707,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.6086270809173584,
      "logits/rejected": -1.5720148086547852,
      "logps/chosen": -228.1181182861328,
      "logps/rejected": -307.33563232421875,
      "loss": 0.1537,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6191433668136597,
      "rewards/margins": 4.839142322540283,
      "rewards/rejected": -5.458285331726074,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 2.0792360305786133,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.6214183568954468,
      "logits/rejected": -1.5774202346801758,
      "logps/chosen": -213.54150390625,
      "logps/rejected": -274.8948974609375,
      "loss": 0.1982,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.48994430899620056,
      "rewards/margins": 4.390619277954102,
      "rewards/rejected": -4.880563259124756,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 3.505744218826294,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.6032994985580444,
      "logits/rejected": -1.5804897546768188,
      "logps/chosen": -221.6490020751953,
      "logps/rejected": -285.50909423828125,
      "loss": 0.2008,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7238569259643555,
      "rewards/margins": 4.416735649108887,
      "rewards/rejected": -5.1405930519104,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 2.8359549045562744,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.519239068031311,
      "logits/rejected": -1.4421329498291016,
      "logps/chosen": -217.1107177734375,
      "logps/rejected": -258.64642333984375,
      "loss": 0.1843,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.4088948667049408,
      "rewards/margins": 4.174046516418457,
      "rewards/rejected": -4.582940101623535,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 4.682807922363281,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.4647693634033203,
      "logits/rejected": -1.4171350002288818,
      "logps/chosen": -225.9655303955078,
      "logps/rejected": -299.84405517578125,
      "loss": 0.1921,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.16728904843330383,
      "rewards/margins": 5.242563724517822,
      "rewards/rejected": -5.409852981567383,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 5.701550006866455,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.5880542993545532,
      "logits/rejected": -1.547324776649475,
      "logps/chosen": -235.06039428710938,
      "logps/rejected": -297.7355651855469,
      "loss": 0.1892,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6305644512176514,
      "rewards/margins": 5.0092668533325195,
      "rewards/rejected": -5.63983154296875,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 4.782139301300049,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.5697643756866455,
      "logits/rejected": -1.510378122329712,
      "logps/chosen": -224.08065795898438,
      "logps/rejected": -284.9275207519531,
      "loss": 0.1929,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6623788475990295,
      "rewards/margins": 4.567178726196289,
      "rewards/rejected": -5.229557514190674,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 4.09246826171875,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.6691592931747437,
      "logits/rejected": -1.568692922592163,
      "logps/chosen": -226.2360382080078,
      "logps/rejected": -297.37799072265625,
      "loss": 0.1745,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5370773077011108,
      "rewards/margins": 4.940179347991943,
      "rewards/rejected": -5.47725772857666,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 4.379272937774658,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.4959796667099,
      "logits/rejected": -1.51832115650177,
      "logps/chosen": -216.44430541992188,
      "logps/rejected": -308.85015869140625,
      "loss": 0.1948,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.460176557302475,
      "rewards/margins": 4.9603471755981445,
      "rewards/rejected": -5.420523166656494,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 2.3544809818267822,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.636636734008789,
      "logits/rejected": -1.5793819427490234,
      "logps/chosen": -216.9700927734375,
      "logps/rejected": -287.65386962890625,
      "loss": 0.1965,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.5221351385116577,
      "rewards/margins": 4.529210090637207,
      "rewards/rejected": -5.0513458251953125,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 3.5676753520965576,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.660280466079712,
      "logits/rejected": -1.5980972051620483,
      "logps/chosen": -239.9665985107422,
      "logps/rejected": -295.49224853515625,
      "loss": 0.2126,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6387029886245728,
      "rewards/margins": 4.475255966186523,
      "rewards/rejected": -5.113959312438965,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 4.009433746337891,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.5477678775787354,
      "logits/rejected": -1.5038559436798096,
      "logps/chosen": -233.2089080810547,
      "logps/rejected": -275.3874816894531,
      "loss": 0.1616,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.1705014854669571,
      "rewards/margins": 4.591641426086426,
      "rewards/rejected": -4.762143135070801,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 4.829162120819092,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.6277942657470703,
      "logits/rejected": -1.541800856590271,
      "logps/chosen": -264.6813659667969,
      "logps/rejected": -298.3739318847656,
      "loss": 0.1844,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.132328063249588,
      "rewards/margins": 5.129775047302246,
      "rewards/rejected": -4.997447490692139,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 2.8724241256713867,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.5807163715362549,
      "logits/rejected": -1.5272471904754639,
      "logps/chosen": -212.9057159423828,
      "logps/rejected": -302.0938720703125,
      "loss": 0.1395,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.03779119253158569,
      "rewards/margins": 5.289394378662109,
      "rewards/rejected": -5.251603603363037,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 4.156414985656738,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.6008613109588623,
      "logits/rejected": -1.5462461709976196,
      "logps/chosen": -221.5519256591797,
      "logps/rejected": -276.48651123046875,
      "loss": 0.2132,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.037129394710063934,
      "rewards/margins": 4.2190446853637695,
      "rewards/rejected": -4.256174087524414,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 4.604877948760986,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.4992469549179077,
      "logits/rejected": -1.588596224784851,
      "logps/chosen": -237.80368041992188,
      "logps/rejected": -276.7908630371094,
      "loss": 0.2331,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.19210034608840942,
      "rewards/margins": 4.0293097496032715,
      "rewards/rejected": -4.221410274505615,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 4.77518367767334,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.5367120504379272,
      "logits/rejected": -1.518836259841919,
      "logps/chosen": -216.94961547851562,
      "logps/rejected": -297.82269287109375,
      "loss": 0.1641,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.05688552185893059,
      "rewards/margins": 4.8048095703125,
      "rewards/rejected": -4.747923851013184,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 2.7294278144836426,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.5205106735229492,
      "logits/rejected": -1.4168894290924072,
      "logps/chosen": -214.5801239013672,
      "logps/rejected": -279.34320068359375,
      "loss": 0.164,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3186998963356018,
      "rewards/margins": 4.727145195007324,
      "rewards/rejected": -5.045844078063965,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 3.494635820388794,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.668074369430542,
      "logits/rejected": -1.6056506633758545,
      "logps/chosen": -226.056396484375,
      "logps/rejected": -283.64312744140625,
      "loss": 0.1962,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.24879607558250427,
      "rewards/margins": 4.313138961791992,
      "rewards/rejected": -4.561934947967529,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 4.093262195587158,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.6196622848510742,
      "logits/rejected": -1.5892953872680664,
      "logps/chosen": -211.94473266601562,
      "logps/rejected": -296.87890625,
      "loss": 0.1735,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.1287282407283783,
      "rewards/margins": 4.94359827041626,
      "rewards/rejected": -4.814869403839111,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 4.43931770324707,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.5054540634155273,
      "logits/rejected": -1.5589126348495483,
      "logps/chosen": -200.23751831054688,
      "logps/rejected": -271.1064453125,
      "loss": 0.2093,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.019808894023299217,
      "rewards/margins": 4.327120780944824,
      "rewards/rejected": -4.34692907333374,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 4.095510959625244,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.5728521347045898,
      "logits/rejected": -1.4718952178955078,
      "logps/chosen": -210.8157196044922,
      "logps/rejected": -272.39410400390625,
      "loss": 0.1988,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.16639301180839539,
      "rewards/margins": 4.377795219421387,
      "rewards/rejected": -4.21140193939209,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 4.527327060699463,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.5970168113708496,
      "logits/rejected": -1.4815723896026611,
      "logps/chosen": -223.53250122070312,
      "logps/rejected": -286.47918701171875,
      "loss": 0.1972,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.09205605089664459,
      "rewards/margins": 4.707822322845459,
      "rewards/rejected": -4.615766525268555,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 2.798776388168335,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.5575376749038696,
      "logits/rejected": -1.4680263996124268,
      "logps/chosen": -229.95352172851562,
      "logps/rejected": -277.1444396972656,
      "loss": 0.1885,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.11531858146190643,
      "rewards/margins": 4.542990684509277,
      "rewards/rejected": -4.427672386169434,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 2.6367621421813965,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.5009130239486694,
      "logits/rejected": -1.52145254611969,
      "logps/chosen": -233.2731170654297,
      "logps/rejected": -285.61663818359375,
      "loss": 0.1508,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.020412441343069077,
      "rewards/margins": 4.928297996520996,
      "rewards/rejected": -4.9487104415893555,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 3.9212377071380615,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.6089658737182617,
      "logits/rejected": -1.5349115133285522,
      "logps/chosen": -217.51895141601562,
      "logps/rejected": -281.0561218261719,
      "loss": 0.1702,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.24485504627227783,
      "rewards/margins": 4.494772434234619,
      "rewards/rejected": -4.739627361297607,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 2.247929334640503,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.560617208480835,
      "logits/rejected": -1.529205083847046,
      "logps/chosen": -226.0928192138672,
      "logps/rejected": -268.5011901855469,
      "loss": 0.1853,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20438656210899353,
      "rewards/margins": 4.4868550300598145,
      "rewards/rejected": -4.691241264343262,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 3.934013843536377,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.6019423007965088,
      "logits/rejected": -1.5354797840118408,
      "logps/chosen": -223.7849578857422,
      "logps/rejected": -286.2995300292969,
      "loss": 0.2091,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.15956279635429382,
      "rewards/margins": 4.305161952972412,
      "rewards/rejected": -4.464724540710449,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 3.362903594970703,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.541973352432251,
      "logits/rejected": -1.5166317224502563,
      "logps/chosen": -221.7763214111328,
      "logps/rejected": -293.17388916015625,
      "loss": 0.2248,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.11231374740600586,
      "rewards/margins": 4.609333038330078,
      "rewards/rejected": -4.721646308898926,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 1.669443964958191,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.6339985132217407,
      "logits/rejected": -1.5085113048553467,
      "logps/chosen": -234.7068328857422,
      "logps/rejected": -285.39642333984375,
      "loss": 0.1391,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.4002878665924072,
      "rewards/margins": 4.612551689147949,
      "rewards/rejected": -4.212264060974121,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 4.4884934425354,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.591619610786438,
      "logits/rejected": -1.495373010635376,
      "logps/chosen": -237.3191375732422,
      "logps/rejected": -295.4523010253906,
      "loss": 0.1871,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.10502872616052628,
      "rewards/margins": 5.0281901359558105,
      "rewards/rejected": -4.923161506652832,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 3.515310049057007,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.6238123178482056,
      "logits/rejected": -1.586525321006775,
      "logps/chosen": -206.5242156982422,
      "logps/rejected": -278.3238220214844,
      "loss": 0.1752,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.16240589320659637,
      "rewards/margins": 4.887397766113281,
      "rewards/rejected": -4.724991798400879,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 5.287433624267578,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.540462851524353,
      "logits/rejected": -1.5053167343139648,
      "logps/chosen": -222.8974609375,
      "logps/rejected": -283.7711486816406,
      "loss": 0.231,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.02352920174598694,
      "rewards/margins": 4.16703987121582,
      "rewards/rejected": -4.143510341644287,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 4.371232509613037,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.4189904928207397,
      "logits/rejected": -1.4196317195892334,
      "logps/chosen": -238.34396362304688,
      "logps/rejected": -286.05682373046875,
      "loss": 0.2134,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.4680125117301941,
      "rewards/margins": 5.1263604164123535,
      "rewards/rejected": -4.6583476066589355,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 4.04213285446167,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.5660136938095093,
      "logits/rejected": -1.5414934158325195,
      "logps/chosen": -226.266845703125,
      "logps/rejected": -281.00823974609375,
      "loss": 0.1493,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.38566893339157104,
      "rewards/margins": 4.783738136291504,
      "rewards/rejected": -4.398069858551025,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 2.9223861694335938,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.53744637966156,
      "logits/rejected": -1.5588454008102417,
      "logps/chosen": -236.10806274414062,
      "logps/rejected": -289.128173828125,
      "loss": 0.1751,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.26428985595703125,
      "rewards/margins": 4.615136623382568,
      "rewards/rejected": -4.350847244262695,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 4.002009391784668,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.439851999282837,
      "logits/rejected": -1.4446804523468018,
      "logps/chosen": -225.57601928710938,
      "logps/rejected": -275.29644775390625,
      "loss": 0.1965,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.3654697835445404,
      "rewards/margins": 4.579410076141357,
      "rewards/rejected": -4.213940143585205,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 3.070713996887207,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.5961030721664429,
      "logits/rejected": -1.5069525241851807,
      "logps/chosen": -216.4861297607422,
      "logps/rejected": -267.57098388671875,
      "loss": 0.1618,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.414588987827301,
      "rewards/margins": 4.7993268966674805,
      "rewards/rejected": -4.384737491607666,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 3.228595495223999,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.6398388147354126,
      "logits/rejected": -1.5302976369857788,
      "logps/chosen": -241.460693359375,
      "logps/rejected": -305.2004699707031,
      "loss": 0.2167,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.2526926100254059,
      "rewards/margins": 4.873023509979248,
      "rewards/rejected": -4.620330810546875,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 4.4869794845581055,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.5678088665008545,
      "logits/rejected": -1.4262385368347168,
      "logps/chosen": -232.7917022705078,
      "logps/rejected": -277.34649658203125,
      "loss": 0.174,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.3906717002391815,
      "rewards/margins": 4.650841236114502,
      "rewards/rejected": -4.26016902923584,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 4.238879203796387,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.5569572448730469,
      "logits/rejected": -1.573560118675232,
      "logps/chosen": -209.18460083007812,
      "logps/rejected": -279.4632263183594,
      "loss": 0.1466,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6271043419837952,
      "rewards/margins": 4.994255065917969,
      "rewards/rejected": -4.367150783538818,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 3.870699882507324,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.6715972423553467,
      "logits/rejected": -1.5435243844985962,
      "logps/chosen": -223.7204132080078,
      "logps/rejected": -261.2884826660156,
      "loss": 0.1914,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.28375479578971863,
      "rewards/margins": 4.4150848388671875,
      "rewards/rejected": -4.131329536437988,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 3.798652172088623,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.6140682697296143,
      "logits/rejected": -1.5190401077270508,
      "logps/chosen": -224.6747283935547,
      "logps/rejected": -268.4491882324219,
      "loss": 0.202,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.05197082832455635,
      "rewards/margins": 4.320896148681641,
      "rewards/rejected": -4.268925666809082,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 2.9648468494415283,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.473844289779663,
      "logits/rejected": -1.479618787765503,
      "logps/chosen": -232.2730712890625,
      "logps/rejected": -298.5815124511719,
      "loss": 0.1892,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.10708019882440567,
      "rewards/margins": 4.55968713760376,
      "rewards/rejected": -4.452606678009033,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 3.558039665222168,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.501671314239502,
      "logits/rejected": -1.4494259357452393,
      "logps/chosen": -232.77017211914062,
      "logps/rejected": -291.8146057128906,
      "loss": 0.1702,
      "rewards/accuracies": 0.9473684430122375,
      "rewards/chosen": 0.30579400062561035,
      "rewards/margins": 4.907108306884766,
      "rewards/rejected": -4.601314544677734,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 1.816678762435913,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.5644961595535278,
      "logits/rejected": -1.4842809438705444,
      "logps/chosen": -237.2871551513672,
      "logps/rejected": -296.8790588378906,
      "loss": 0.13,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.1960347443819046,
      "rewards/margins": 4.91409969329834,
      "rewards/rejected": -4.7180657386779785,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 3.761232376098633,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.4631893634796143,
      "logits/rejected": -1.4054863452911377,
      "logps/chosen": -219.4536590576172,
      "logps/rejected": -298.73504638671875,
      "loss": 0.1734,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.10820744186639786,
      "rewards/margins": 4.628209114074707,
      "rewards/rejected": -4.520001411437988,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 3.467888832092285,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.4637609720230103,
      "logits/rejected": -1.4368250370025635,
      "logps/chosen": -214.6084442138672,
      "logps/rejected": -276.13189697265625,
      "loss": 0.1424,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2766164243221283,
      "rewards/margins": 4.959458351135254,
      "rewards/rejected": -4.682842254638672,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 2.6543424129486084,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.6459102630615234,
      "logits/rejected": -1.6207349300384521,
      "logps/chosen": -210.13259887695312,
      "logps/rejected": -287.6831359863281,
      "loss": 0.1244,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.011042666621506214,
      "rewards/margins": 5.0912251472473145,
      "rewards/rejected": -5.080182075500488,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 3.120337963104248,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.600803017616272,
      "logits/rejected": -1.5503169298171997,
      "logps/chosen": -233.22439575195312,
      "logps/rejected": -283.6503601074219,
      "loss": 0.1614,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.14875979721546173,
      "rewards/margins": 4.319775581359863,
      "rewards/rejected": -4.171015739440918,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 3.577122211456299,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.6564123630523682,
      "logits/rejected": -1.5476027727127075,
      "logps/chosen": -229.00778198242188,
      "logps/rejected": -290.4595642089844,
      "loss": 0.1642,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.0114586828276515,
      "rewards/margins": 5.043142795562744,
      "rewards/rejected": -5.031683921813965,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 3.0231740474700928,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.5095252990722656,
      "logits/rejected": -1.5051429271697998,
      "logps/chosen": -215.7372589111328,
      "logps/rejected": -279.8727722167969,
      "loss": 0.1494,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.09145411103963852,
      "rewards/margins": 4.958231449127197,
      "rewards/rejected": -4.8667778968811035,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 2.384561538696289,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.5030790567398071,
      "logits/rejected": -1.5039005279541016,
      "logps/chosen": -223.53341674804688,
      "logps/rejected": -268.07305908203125,
      "loss": 0.1692,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.3087685704231262,
      "rewards/margins": 4.652369499206543,
      "rewards/rejected": -4.343601226806641,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 2.826698064804077,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.5461299419403076,
      "logits/rejected": -1.5500050783157349,
      "logps/chosen": -219.19961547851562,
      "logps/rejected": -297.0096740722656,
      "loss": 0.1482,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.03134971857070923,
      "rewards/margins": 4.71282958984375,
      "rewards/rejected": -4.744178771972656,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 4.161277770996094,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.6473701000213623,
      "logits/rejected": -1.5392096042633057,
      "logps/chosen": -237.41329956054688,
      "logps/rejected": -287.1451110839844,
      "loss": 0.1731,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3596492409706116,
      "rewards/margins": 4.376633644104004,
      "rewards/rejected": -4.7362823486328125,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 2.4758927822113037,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.5916746854782104,
      "logits/rejected": -1.5499130487442017,
      "logps/chosen": -200.91793823242188,
      "logps/rejected": -293.1402282714844,
      "loss": 0.1315,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.13916119933128357,
      "rewards/margins": 5.298827648162842,
      "rewards/rejected": -5.437989234924316,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 2.656404972076416,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.6521761417388916,
      "logits/rejected": -1.4956820011138916,
      "logps/chosen": -221.94546508789062,
      "logps/rejected": -280.0649719238281,
      "loss": 0.1546,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.021595388650894165,
      "rewards/margins": 4.696625709533691,
      "rewards/rejected": -4.718221187591553,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 3.276386260986328,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.5643367767333984,
      "logits/rejected": -1.5389833450317383,
      "logps/chosen": -232.3899383544922,
      "logps/rejected": -296.09600830078125,
      "loss": 0.1472,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.1014956459403038,
      "rewards/margins": 4.947221755981445,
      "rewards/rejected": -5.048717021942139,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 4.7639923095703125,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.6109592914581299,
      "logits/rejected": -1.52823007106781,
      "logps/chosen": -238.28677368164062,
      "logps/rejected": -277.99713134765625,
      "loss": 0.1226,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.02549382485449314,
      "rewards/margins": 4.942860126495361,
      "rewards/rejected": -4.968354225158691,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 2.7891688346862793,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.5980685949325562,
      "logits/rejected": -1.5289149284362793,
      "logps/chosen": -217.5404510498047,
      "logps/rejected": -278.1165771484375,
      "loss": 0.1493,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06327895075082779,
      "rewards/margins": 4.962241172790527,
      "rewards/rejected": -4.898962020874023,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 3.5160892009735107,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.5579856634140015,
      "logits/rejected": -1.478334665298462,
      "logps/chosen": -227.519287109375,
      "logps/rejected": -287.3228454589844,
      "loss": 0.1968,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.20410621166229248,
      "rewards/margins": 4.94499397277832,
      "rewards/rejected": -5.149099826812744,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 3.181624174118042,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.5951579809188843,
      "logits/rejected": -1.5237982273101807,
      "logps/chosen": -208.18765258789062,
      "logps/rejected": -279.5474548339844,
      "loss": 0.1626,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.01243979949504137,
      "rewards/margins": 5.2108306884765625,
      "rewards/rejected": -5.198391437530518,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 3.6655056476593018,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.4442431926727295,
      "logits/rejected": -1.4688308238983154,
      "logps/chosen": -217.04135131835938,
      "logps/rejected": -294.37237548828125,
      "loss": 0.1154,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.18168587982654572,
      "rewards/margins": 4.986029624938965,
      "rewards/rejected": -5.167714595794678,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 2.8012073040008545,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.598311424255371,
      "logits/rejected": -1.5263886451721191,
      "logps/chosen": -225.3389892578125,
      "logps/rejected": -280.2146911621094,
      "loss": 0.1635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08202177286148071,
      "rewards/margins": 4.779571056365967,
      "rewards/rejected": -4.861592769622803,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 1.4678078889846802,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.5057955980300903,
      "logits/rejected": -1.5607259273529053,
      "logps/chosen": -228.93185424804688,
      "logps/rejected": -289.180419921875,
      "loss": 0.1137,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.21716734766960144,
      "rewards/margins": 5.549811363220215,
      "rewards/rejected": -5.7669782638549805,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 3.0782907009124756,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.5692188739776611,
      "logits/rejected": -1.4577603340148926,
      "logps/chosen": -224.8596649169922,
      "logps/rejected": -299.32440185546875,
      "loss": 0.1343,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03337126225233078,
      "rewards/margins": 5.490138053894043,
      "rewards/rejected": -5.5235090255737305,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 2.948732376098633,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.5691108703613281,
      "logits/rejected": -1.5023016929626465,
      "logps/chosen": -221.1350555419922,
      "logps/rejected": -278.23333740234375,
      "loss": 0.1583,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0295710451900959,
      "rewards/margins": 5.049849510192871,
      "rewards/rejected": -5.020279407501221,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 3.2476890087127686,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.5953521728515625,
      "logits/rejected": -1.5542570352554321,
      "logps/chosen": -232.19564819335938,
      "logps/rejected": -282.22723388671875,
      "loss": 0.1667,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.024515807628631592,
      "rewards/margins": 5.146022319793701,
      "rewards/rejected": -5.170538425445557,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 2.909899950027466,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.6917043924331665,
      "logits/rejected": -1.5641858577728271,
      "logps/chosen": -229.15185546875,
      "logps/rejected": -308.77288818359375,
      "loss": 0.1287,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.030151885002851486,
      "rewards/margins": 5.283860206604004,
      "rewards/rejected": -5.31401252746582,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 3.3673746585845947,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.5426510572433472,
      "logits/rejected": -1.4823468923568726,
      "logps/chosen": -231.9100341796875,
      "logps/rejected": -267.9788818359375,
      "loss": 0.1257,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08240224421024323,
      "rewards/margins": 4.7699127197265625,
      "rewards/rejected": -4.8523149490356445,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 1.933462142944336,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.5066808462142944,
      "logits/rejected": -1.4589513540267944,
      "logps/chosen": -220.457763671875,
      "logps/rejected": -277.82281494140625,
      "loss": 0.1569,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.21246495842933655,
      "rewards/margins": 5.227828025817871,
      "rewards/rejected": -5.015363693237305,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 3.799119710922241,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.4951709508895874,
      "logits/rejected": -1.448192834854126,
      "logps/chosen": -224.44320678710938,
      "logps/rejected": -273.17864990234375,
      "loss": 0.1707,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.21202436089515686,
      "rewards/margins": 4.784801483154297,
      "rewards/rejected": -4.996825218200684,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 3.391683340072632,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.4559767246246338,
      "logits/rejected": -1.3820351362228394,
      "logps/chosen": -224.79928588867188,
      "logps/rejected": -291.6627197265625,
      "loss": 0.1311,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.032825786620378494,
      "rewards/margins": 5.525381565093994,
      "rewards/rejected": -5.492555618286133,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 2.596304416656494,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.6544069051742554,
      "logits/rejected": -1.7071607112884521,
      "logps/chosen": -212.39675903320312,
      "logps/rejected": -295.53192138671875,
      "loss": 0.1484,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.38105496764183044,
      "rewards/margins": 5.09371280670166,
      "rewards/rejected": -5.474766731262207,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 3.5445754528045654,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.6285476684570312,
      "logits/rejected": -1.4937007427215576,
      "logps/chosen": -231.6444091796875,
      "logps/rejected": -286.3141174316406,
      "loss": 0.1976,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.24693647027015686,
      "rewards/margins": 4.544675827026367,
      "rewards/rejected": -4.791612148284912,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 6.524362087249756,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.4855129718780518,
      "logits/rejected": -1.4172195196151733,
      "logps/chosen": -240.4997100830078,
      "logps/rejected": -289.3425598144531,
      "loss": 0.1824,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5645937323570251,
      "rewards/margins": 4.782386302947998,
      "rewards/rejected": -5.346980094909668,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 2.2315993309020996,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.5639643669128418,
      "logits/rejected": -1.477234125137329,
      "logps/chosen": -211.5214080810547,
      "logps/rejected": -280.4677429199219,
      "loss": 0.1117,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.0826307162642479,
      "rewards/margins": 5.693845748901367,
      "rewards/rejected": -5.776476860046387,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 2.92634654045105,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.5262413024902344,
      "logits/rejected": -1.5272846221923828,
      "logps/chosen": -213.702880859375,
      "logps/rejected": -303.1562194824219,
      "loss": 0.1217,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.05086277052760124,
      "rewards/margins": 5.797705173492432,
      "rewards/rejected": -5.848567485809326,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 6.668047904968262,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.673953652381897,
      "logits/rejected": -1.703916311264038,
      "logps/chosen": -208.412109375,
      "logps/rejected": -313.27484130859375,
      "loss": 0.1454,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5052535533905029,
      "rewards/margins": 5.835505485534668,
      "rewards/rejected": -6.34075927734375,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 3.2992238998413086,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.5450654029846191,
      "logits/rejected": -1.51485276222229,
      "logps/chosen": -216.7915802001953,
      "logps/rejected": -273.28741455078125,
      "loss": 0.1854,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.23610785603523254,
      "rewards/margins": 4.69701623916626,
      "rewards/rejected": -4.933123588562012,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 2.530855417251587,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.5250041484832764,
      "logits/rejected": -1.4885377883911133,
      "logps/chosen": -264.8410339355469,
      "logps/rejected": -289.5926818847656,
      "loss": 0.1345,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.18493278324604034,
      "rewards/margins": 5.43831205368042,
      "rewards/rejected": -5.253379821777344,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 2.140273332595825,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.499107837677002,
      "logits/rejected": -1.499398946762085,
      "logps/chosen": -222.88674926757812,
      "logps/rejected": -291.7001037597656,
      "loss": 0.1099,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3723798394203186,
      "rewards/margins": 5.258582592010498,
      "rewards/rejected": -5.630963325500488,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 1.6813573837280273,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.64093017578125,
      "logits/rejected": -1.5066547393798828,
      "logps/chosen": -230.73257446289062,
      "logps/rejected": -288.6161804199219,
      "loss": 0.1134,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4509183466434479,
      "rewards/margins": 5.539261817932129,
      "rewards/rejected": -5.990180015563965,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 3.6021177768707275,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.5829105377197266,
      "logits/rejected": -1.5672820806503296,
      "logps/chosen": -222.74728393554688,
      "logps/rejected": -302.9520263671875,
      "loss": 0.1154,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.12925612926483154,
      "rewards/margins": 5.923072814941406,
      "rewards/rejected": -6.052329063415527,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 4.091590881347656,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.4938812255859375,
      "logits/rejected": -1.484735131263733,
      "logps/chosen": -238.08212280273438,
      "logps/rejected": -304.36090087890625,
      "loss": 0.1727,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.22507795691490173,
      "rewards/margins": 5.176184177398682,
      "rewards/rejected": -5.401261806488037,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 4.568892002105713,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.6104847192764282,
      "logits/rejected": -1.5651620626449585,
      "logps/chosen": -220.1466827392578,
      "logps/rejected": -282.4197998046875,
      "loss": 0.1564,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.18282730877399445,
      "rewards/margins": 5.292811393737793,
      "rewards/rejected": -5.4756388664245605,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 3.958512306213379,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.5375100374221802,
      "logits/rejected": -1.5117011070251465,
      "logps/chosen": -217.5662078857422,
      "logps/rejected": -295.0578918457031,
      "loss": 0.141,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3425526022911072,
      "rewards/margins": 5.050057888031006,
      "rewards/rejected": -5.392610549926758,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 4.313801288604736,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.474077582359314,
      "logits/rejected": -1.488755702972412,
      "logps/chosen": -222.3016815185547,
      "logps/rejected": -293.5660095214844,
      "loss": 0.2337,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4544387459754944,
      "rewards/margins": 4.519167900085449,
      "rewards/rejected": -4.973607063293457,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 2.2440829277038574,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.5298980474472046,
      "logits/rejected": -1.526539921760559,
      "logps/chosen": -223.37890625,
      "logps/rejected": -300.4910583496094,
      "loss": 0.132,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.316823810338974,
      "rewards/margins": 5.748261451721191,
      "rewards/rejected": -6.065085411071777,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 3.3815062046051025,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.5895965099334717,
      "logits/rejected": -1.5554959774017334,
      "logps/chosen": -239.8270721435547,
      "logps/rejected": -320.05853271484375,
      "loss": 0.1482,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.015445577912032604,
      "rewards/margins": 5.61312198638916,
      "rewards/rejected": -5.597676753997803,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 2.022261381149292,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.5600824356079102,
      "logits/rejected": -1.5448620319366455,
      "logps/chosen": -228.0321044921875,
      "logps/rejected": -286.53924560546875,
      "loss": 0.1514,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.341894268989563,
      "rewards/margins": 4.940691947937012,
      "rewards/rejected": -5.282585620880127,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 3.8453714847564697,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.5347708463668823,
      "logits/rejected": -1.5214083194732666,
      "logps/chosen": -216.1919403076172,
      "logps/rejected": -290.57806396484375,
      "loss": 0.1768,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.2583093047142029,
      "rewards/margins": 4.864516735076904,
      "rewards/rejected": -5.122825622558594,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 3.6349446773529053,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.4968597888946533,
      "logits/rejected": -1.4725162982940674,
      "logps/chosen": -223.86300659179688,
      "logps/rejected": -276.0819396972656,
      "loss": 0.1417,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.19201938807964325,
      "rewards/margins": 5.1369218826293945,
      "rewards/rejected": -5.328940391540527,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 2.9113125801086426,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.5767369270324707,
      "logits/rejected": -1.5519667863845825,
      "logps/chosen": -227.7290496826172,
      "logps/rejected": -302.18341064453125,
      "loss": 0.1685,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.14696352183818817,
      "rewards/margins": 5.197046756744385,
      "rewards/rejected": -5.344009876251221,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.3454444408416748,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.6368871927261353,
      "logits/rejected": -1.610581398010254,
      "logps/chosen": -218.0592803955078,
      "logps/rejected": -311.0408630371094,
      "loss": 0.1269,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03764452040195465,
      "rewards/margins": 5.279090881347656,
      "rewards/rejected": -5.241446495056152,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 7.484287261962891,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.591990351676941,
      "logits/rejected": -1.5665239095687866,
      "logps/chosen": -212.2829132080078,
      "logps/rejected": -309.5106506347656,
      "loss": 0.1343,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.053197573870420456,
      "rewards/margins": 5.905511856079102,
      "rewards/rejected": -5.8523149490356445,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 3.2489264011383057,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.541517734527588,
      "logits/rejected": -1.4897912740707397,
      "logps/chosen": -216.9764862060547,
      "logps/rejected": -293.786865234375,
      "loss": 0.1538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.21131077408790588,
      "rewards/margins": 5.59000301361084,
      "rewards/rejected": -5.801313877105713,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 4.643258571624756,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.5824702978134155,
      "logits/rejected": -1.5110851526260376,
      "logps/chosen": -246.80264282226562,
      "logps/rejected": -279.2035217285156,
      "loss": 0.1596,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.21497675776481628,
      "rewards/margins": 4.839934349060059,
      "rewards/rejected": -5.054910659790039,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 5.287117004394531,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.4827041625976562,
      "logits/rejected": -1.4560563564300537,
      "logps/chosen": -254.863525390625,
      "logps/rejected": -300.84552001953125,
      "loss": 0.1416,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.15708236396312714,
      "rewards/margins": 5.223883152008057,
      "rewards/rejected": -5.380965232849121,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 6.302121162414551,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.6150802373886108,
      "logits/rejected": -1.5412870645523071,
      "logps/chosen": -223.01675415039062,
      "logps/rejected": -272.59735107421875,
      "loss": 0.1892,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3169715702533722,
      "rewards/margins": 4.87038516998291,
      "rewards/rejected": -5.187356948852539,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 4.08319091796875,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.5628572702407837,
      "logits/rejected": -1.4623311758041382,
      "logps/chosen": -210.0499267578125,
      "logps/rejected": -287.72686767578125,
      "loss": 0.1337,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2616520822048187,
      "rewards/margins": 5.518267631530762,
      "rewards/rejected": -5.779919624328613,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 3.5064926147460938,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.5222089290618896,
      "logits/rejected": -1.5433714389801025,
      "logps/chosen": -223.13699340820312,
      "logps/rejected": -277.22027587890625,
      "loss": 0.1881,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.1823982298374176,
      "rewards/margins": 4.619892120361328,
      "rewards/rejected": -4.802290916442871,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 5.202704906463623,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.495309591293335,
      "logits/rejected": -1.4390442371368408,
      "logps/chosen": -219.75357055664062,
      "logps/rejected": -292.7464599609375,
      "loss": 0.1962,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.33655446767807007,
      "rewards/margins": 4.839663982391357,
      "rewards/rejected": -5.1762189865112305,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 2.8011467456817627,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.6091105937957764,
      "logits/rejected": -1.5161319971084595,
      "logps/chosen": -237.4774627685547,
      "logps/rejected": -300.9270324707031,
      "loss": 0.1419,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3717946410179138,
      "rewards/margins": 5.390846252441406,
      "rewards/rejected": -5.762640953063965,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 3.398773193359375,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.5985854864120483,
      "logits/rejected": -1.5678365230560303,
      "logps/chosen": -239.23867797851562,
      "logps/rejected": -289.22503662109375,
      "loss": 0.1431,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.35332396626472473,
      "rewards/margins": 4.7726006507873535,
      "rewards/rejected": -5.125924587249756,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 4.891382217407227,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.581493616104126,
      "logits/rejected": -1.582097053527832,
      "logps/chosen": -207.20639038085938,
      "logps/rejected": -289.98748779296875,
      "loss": 0.1611,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.2472093105316162,
      "rewards/margins": 5.706404209136963,
      "rewards/rejected": -5.95361328125,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 2.183753728866577,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.5279576778411865,
      "logits/rejected": -1.4338546991348267,
      "logps/chosen": -278.4076232910156,
      "logps/rejected": -308.9873046875,
      "loss": 0.1213,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.06257566064596176,
      "rewards/margins": 6.102546691894531,
      "rewards/rejected": -6.165122032165527,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 4.505012035369873,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.498521089553833,
      "logits/rejected": -1.533311128616333,
      "logps/chosen": -235.9097900390625,
      "logps/rejected": -305.294677734375,
      "loss": 0.1317,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4602181017398834,
      "rewards/margins": 5.288244247436523,
      "rewards/rejected": -5.748462200164795,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 3.8435416221618652,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.635210633277893,
      "logits/rejected": -1.6011021137237549,
      "logps/chosen": -203.96682739257812,
      "logps/rejected": -292.73248291015625,
      "loss": 0.1212,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.29221805930137634,
      "rewards/margins": 5.458622932434082,
      "rewards/rejected": -5.7508416175842285,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 3.876904249191284,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.5189704895019531,
      "logits/rejected": -1.479864239692688,
      "logps/chosen": -234.113037109375,
      "logps/rejected": -301.35260009765625,
      "loss": 0.1127,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4382593035697937,
      "rewards/margins": 5.527853488922119,
      "rewards/rejected": -5.9661126136779785,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 3.5501556396484375,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.5354712009429932,
      "logits/rejected": -1.5287415981292725,
      "logps/chosen": -232.2896270751953,
      "logps/rejected": -295.0993957519531,
      "loss": 0.1816,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.33327555656433105,
      "rewards/margins": 5.117754936218262,
      "rewards/rejected": -5.451030731201172,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 4.883026599884033,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.546303391456604,
      "logits/rejected": -1.469744086265564,
      "logps/chosen": -218.6349639892578,
      "logps/rejected": -278.43115234375,
      "loss": 0.1493,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.04183761402964592,
      "rewards/margins": 5.151387691497803,
      "rewards/rejected": -5.193225383758545,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 3.7158358097076416,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.535130262374878,
      "logits/rejected": -1.5816833972930908,
      "logps/chosen": -226.7908172607422,
      "logps/rejected": -315.193603515625,
      "loss": 0.1752,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.836959958076477,
      "rewards/margins": 5.0551886558532715,
      "rewards/rejected": -5.892148971557617,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 3.4619786739349365,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.5055862665176392,
      "logits/rejected": -1.485164999961853,
      "logps/chosen": -214.0679473876953,
      "logps/rejected": -299.93524169921875,
      "loss": 0.1093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2596716284751892,
      "rewards/margins": 5.675719261169434,
      "rewards/rejected": -5.935390949249268,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 2.670748472213745,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.5897644758224487,
      "logits/rejected": -1.4919850826263428,
      "logps/chosen": -233.66592407226562,
      "logps/rejected": -283.1734924316406,
      "loss": 0.1356,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.1264813244342804,
      "rewards/margins": 5.175812721252441,
      "rewards/rejected": -5.302294731140137,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 3.714576482772827,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.508699655532837,
      "logits/rejected": -1.4295189380645752,
      "logps/chosen": -229.2381591796875,
      "logps/rejected": -296.42657470703125,
      "loss": 0.1774,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4729483127593994,
      "rewards/margins": 5.336666584014893,
      "rewards/rejected": -5.809614658355713,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 4.889886379241943,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.4413424730300903,
      "logits/rejected": -1.502366304397583,
      "logps/chosen": -234.2248992919922,
      "logps/rejected": -292.7245788574219,
      "loss": 0.1684,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6228765845298767,
      "rewards/margins": 4.261327743530273,
      "rewards/rejected": -4.884203910827637,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 3.7302002906799316,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.5013799667358398,
      "logits/rejected": -1.5552741289138794,
      "logps/chosen": -219.25308227539062,
      "logps/rejected": -306.8885803222656,
      "loss": 0.1487,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.20830953121185303,
      "rewards/margins": 5.794388771057129,
      "rewards/rejected": -6.002697467803955,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 4.609808444976807,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.4839293956756592,
      "logits/rejected": -1.5049326419830322,
      "logps/chosen": -248.7401885986328,
      "logps/rejected": -316.1900939941406,
      "loss": 0.1847,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.1252104490995407,
      "rewards/margins": 5.684288024902344,
      "rewards/rejected": -5.809498310089111,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 3.7130322456359863,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.5436820983886719,
      "logits/rejected": -1.5339748859405518,
      "logps/chosen": -221.01025390625,
      "logps/rejected": -291.3807373046875,
      "loss": 0.1641,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.1684941202402115,
      "rewards/margins": 5.483412742614746,
      "rewards/rejected": -5.651907444000244,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 4.641164302825928,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.4652761220932007,
      "logits/rejected": -1.3969151973724365,
      "logps/chosen": -221.5576629638672,
      "logps/rejected": -288.90411376953125,
      "loss": 0.1661,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.37317955493927,
      "rewards/margins": 5.2659220695495605,
      "rewards/rejected": -5.639101505279541,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 1.6908352375030518,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.651746392250061,
      "logits/rejected": -1.603569746017456,
      "logps/chosen": -239.42745971679688,
      "logps/rejected": -300.3352966308594,
      "loss": 0.1821,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.31848442554473877,
      "rewards/margins": 5.208474159240723,
      "rewards/rejected": -5.526959419250488,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 2.5582754611968994,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.669661283493042,
      "logits/rejected": -1.6277196407318115,
      "logps/chosen": -222.32205200195312,
      "logps/rejected": -291.809814453125,
      "loss": 0.1485,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6328510046005249,
      "rewards/margins": 4.7722649574279785,
      "rewards/rejected": -5.405115604400635,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 3.1268057823181152,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.5684897899627686,
      "logits/rejected": -1.5883140563964844,
      "logps/chosen": -231.85134887695312,
      "logps/rejected": -299.2911071777344,
      "loss": 0.1581,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.49852675199508667,
      "rewards/margins": 5.060816764831543,
      "rewards/rejected": -5.5593438148498535,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 1.8423947095870972,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.4214311838150024,
      "logits/rejected": -1.356693983078003,
      "logps/chosen": -239.08755493164062,
      "logps/rejected": -274.2549133300781,
      "loss": 0.1151,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.49795475602149963,
      "rewards/margins": 4.846909046173096,
      "rewards/rejected": -5.344862937927246,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 3.8253424167633057,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.5384304523468018,
      "logits/rejected": -1.509032964706421,
      "logps/chosen": -236.12686157226562,
      "logps/rejected": -284.77642822265625,
      "loss": 0.2041,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.509702742099762,
      "rewards/margins": 4.945359230041504,
      "rewards/rejected": -5.455061435699463,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 3.1354925632476807,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.6047881841659546,
      "logits/rejected": -1.5616092681884766,
      "logps/chosen": -213.57376098632812,
      "logps/rejected": -300.4778747558594,
      "loss": 0.1115,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4236835539340973,
      "rewards/margins": 5.589079856872559,
      "rewards/rejected": -6.012763023376465,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 2.7940657138824463,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.6039320230484009,
      "logits/rejected": -1.4985624551773071,
      "logps/chosen": -236.09060668945312,
      "logps/rejected": -281.80096435546875,
      "loss": 0.1656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3135233521461487,
      "rewards/margins": 4.965387344360352,
      "rewards/rejected": -5.2789106369018555,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 4.520009517669678,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.6661808490753174,
      "logits/rejected": -1.5840932130813599,
      "logps/chosen": -219.1964111328125,
      "logps/rejected": -289.63824462890625,
      "loss": 0.1875,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.30475789308547974,
      "rewards/margins": 5.524220943450928,
      "rewards/rejected": -5.828978538513184,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 3.332031726837158,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.5584747791290283,
      "logits/rejected": -1.5064611434936523,
      "logps/chosen": -241.1908721923828,
      "logps/rejected": -306.5281677246094,
      "loss": 0.1813,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.28189021348953247,
      "rewards/margins": 5.6129279136657715,
      "rewards/rejected": -5.8948187828063965,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 3.0526838302612305,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.555757999420166,
      "logits/rejected": -1.552303671836853,
      "logps/chosen": -229.64736938476562,
      "logps/rejected": -298.7793273925781,
      "loss": 0.1325,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7364352345466614,
      "rewards/margins": 5.4410200119018555,
      "rewards/rejected": -6.177455425262451,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 4.28289270401001,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6077375411987305,
      "logits/rejected": -1.6054372787475586,
      "logps/chosen": -244.438720703125,
      "logps/rejected": -294.0166320800781,
      "loss": 0.1795,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4182673394680023,
      "rewards/margins": 5.0655107498168945,
      "rewards/rejected": -5.483778476715088,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 3.2869319915771484,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.563001036643982,
      "logits/rejected": -1.467656135559082,
      "logps/chosen": -226.94686889648438,
      "logps/rejected": -281.23114013671875,
      "loss": 0.1337,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4600711464881897,
      "rewards/margins": 5.309510707855225,
      "rewards/rejected": -5.7695817947387695,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 2.482046365737915,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.5494192838668823,
      "logits/rejected": -1.5291993618011475,
      "logps/chosen": -215.4961395263672,
      "logps/rejected": -297.94671630859375,
      "loss": 0.1336,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3590046763420105,
      "rewards/margins": 5.251859188079834,
      "rewards/rejected": -5.610864162445068,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 2.1505300998687744,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.5560667514801025,
      "logits/rejected": -1.570675253868103,
      "logps/chosen": -232.88662719726562,
      "logps/rejected": -308.8785400390625,
      "loss": 0.1682,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.34640324115753174,
      "rewards/margins": 5.028754711151123,
      "rewards/rejected": -5.375157356262207,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 2.367377758026123,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.5260568857192993,
      "logits/rejected": -1.4501224756240845,
      "logps/chosen": -213.99169921875,
      "logps/rejected": -291.2949523925781,
      "loss": 0.1375,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.27664098143577576,
      "rewards/margins": 5.49239444732666,
      "rewards/rejected": -5.769036769866943,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 2.2598955631256104,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.4077026844024658,
      "logits/rejected": -1.4860719442367554,
      "logps/chosen": -228.6597137451172,
      "logps/rejected": -294.4121398925781,
      "loss": 0.138,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3458075523376465,
      "rewards/margins": 5.261725902557373,
      "rewards/rejected": -5.6075334548950195,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 3.635240077972412,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.5582494735717773,
      "logits/rejected": -1.4538967609405518,
      "logps/chosen": -232.40707397460938,
      "logps/rejected": -286.7071838378906,
      "loss": 0.1605,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3285108506679535,
      "rewards/margins": 5.033994674682617,
      "rewards/rejected": -5.362505912780762,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}