{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 182,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005494505494505495,
      "grad_norm": 2.0219836235046387,
      "learning_rate": 0.0,
      "logits/chosen": -0.29912787675857544,
      "logits/rejected": -0.26033276319503784,
      "logps/chosen": -1493.069580078125,
      "logps/rejected": -1464.670654296875,
      "loss": 0.6931473016738892,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01098901098901099,
      "grad_norm": 1.2694443464279175,
      "learning_rate": 1.3157894736842107e-07,
      "logits/chosen": -0.5480956435203552,
      "logits/rejected": -0.6893957853317261,
      "logps/chosen": -1354.66455078125,
      "logps/rejected": -1234.7755126953125,
      "loss": 0.6931473016738892,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.016483516483516484,
      "grad_norm": 1.7238407135009766,
      "learning_rate": 2.6315789473684213e-07,
      "logits/chosen": -0.27911028265953064,
      "logits/rejected": -0.146735280752182,
      "logps/chosen": -1381.031982421875,
      "logps/rejected": -1174.5887451171875,
      "loss": 0.6932641267776489,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002829666016623378,
      "rewards/margins": 0.001636037603020668,
      "rewards/rejected": 0.001193628297187388,
      "step": 3
    },
    {
      "epoch": 0.02197802197802198,
      "grad_norm": 1.3951466083526611,
      "learning_rate": 3.9473684210526315e-07,
      "logits/chosen": -0.168782040476799,
      "logits/rejected": -0.25047171115875244,
      "logps/chosen": -1239.2984619140625,
      "logps/rejected": -1150.9674072265625,
      "loss": 0.6944670081138611,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0013842176413163543,
      "rewards/margins": -0.0002266278606839478,
      "rewards/rejected": 0.0016108450945466757,
      "step": 4
    },
    {
      "epoch": 0.027472527472527472,
      "grad_norm": 1.1389007568359375,
      "learning_rate": 5.263157894736843e-07,
      "logits/chosen": -0.6456503868103027,
      "logits/rejected": -0.6371781229972839,
      "logps/chosen": -1409.6326904296875,
      "logps/rejected": -1272.1806640625,
      "loss": 0.6931549906730652,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00566960358992219,
      "rewards/margins": 0.0016335974214598536,
      "rewards/rejected": 0.004036006983369589,
      "step": 5
    },
    {
      "epoch": 0.03296703296703297,
      "grad_norm": 0.974063515663147,
      "learning_rate": 6.578947368421053e-07,
      "logits/chosen": -0.30573540925979614,
      "logits/rejected": -0.2751460373401642,
      "logps/chosen": -1334.066650390625,
      "logps/rejected": -1179.54248046875,
      "loss": 0.6933455467224121,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015271691605448723,
      "rewards/margins": 0.002894277684390545,
      "rewards/rejected": 0.012377413921058178,
      "step": 6
    },
    {
      "epoch": 0.038461538461538464,
      "grad_norm": 1.2145659923553467,
      "learning_rate": 7.894736842105263e-07,
      "logits/chosen": 0.048936627805233,
      "logits/rejected": -0.05937391147017479,
      "logps/chosen": -1329.2144775390625,
      "logps/rejected": -1263.015869140625,
      "loss": 0.6919921040534973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012406377121806145,
      "rewards/margins": 0.0037618777714669704,
      "rewards/rejected": 0.008644499816000462,
      "step": 7
    },
    {
      "epoch": 0.04395604395604396,
      "grad_norm": 1.1337213516235352,
      "learning_rate": 9.210526315789474e-07,
      "logits/chosen": -0.6898252964019775,
      "logits/rejected": -0.5947282314300537,
      "logps/chosen": -1531.9923095703125,
      "logps/rejected": -1458.8387451171875,
      "loss": 0.692238450050354,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010655415244400501,
      "rewards/margins": 0.006124755833297968,
      "rewards/rejected": 0.004530658945441246,
      "step": 8
    },
    {
      "epoch": 0.04945054945054945,
      "grad_norm": 1.4570075273513794,
      "learning_rate": 1.0526315789473685e-06,
      "logits/chosen": -0.6079831123352051,
      "logits/rejected": -0.6475998759269714,
      "logps/chosen": -1284.917236328125,
      "logps/rejected": -1239.587158203125,
      "loss": 0.6913315653800964,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008226280100643635,
      "rewards/margins": 0.00454014353454113,
      "rewards/rejected": 0.0036861367989331484,
      "step": 9
    },
    {
      "epoch": 0.054945054945054944,
      "grad_norm": 1.1168543100357056,
      "learning_rate": 1.1842105263157894e-06,
      "logits/chosen": -0.4832429587841034,
      "logits/rejected": -0.6784942150115967,
      "logps/chosen": -1411.264404296875,
      "logps/rejected": -1368.5498046875,
      "loss": 0.6921623349189758,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017078913748264313,
      "rewards/margins": 0.003607435617595911,
      "rewards/rejected": 0.013471478596329689,
      "step": 10
    },
    {
      "epoch": 0.06043956043956044,
      "grad_norm": 1.6089200973510742,
      "learning_rate": 1.3157894736842106e-06,
      "logits/chosen": -0.3044881820678711,
      "logits/rejected": -0.4217117428779602,
      "logps/chosen": -1790.7430419921875,
      "logps/rejected": -1541.77587890625,
      "loss": 0.6924711465835571,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017292898148298264,
      "rewards/margins": 0.0005286832456476986,
      "rewards/rejected": 0.016764217987656593,
      "step": 11
    },
    {
      "epoch": 0.06593406593406594,
      "grad_norm": 0.6998999714851379,
      "learning_rate": 1.4473684210526317e-06,
      "logits/chosen": -0.484161376953125,
      "logits/rejected": -0.4741247892379761,
      "logps/chosen": -1405.0679931640625,
      "logps/rejected": -1307.798095703125,
      "loss": 0.6923574209213257,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03834030032157898,
      "rewards/margins": 0.016229379922151566,
      "rewards/rejected": 0.022110918536782265,
      "step": 12
    },
    {
      "epoch": 0.07142857142857142,
      "grad_norm": 0.7959760427474976,
      "learning_rate": 1.5789473684210526e-06,
      "logits/chosen": -0.35663121938705444,
      "logits/rejected": -0.44265735149383545,
      "logps/chosen": -1244.1324462890625,
      "logps/rejected": -1132.490478515625,
      "loss": 0.6914532780647278,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0250022504478693,
      "rewards/margins": 0.010114078409969807,
      "rewards/rejected": 0.014888172969222069,
      "step": 13
    },
    {
      "epoch": 0.07692307692307693,
      "grad_norm": 0.8386003971099854,
      "learning_rate": 1.710526315789474e-06,
      "logits/chosen": -0.19147101044654846,
      "logits/rejected": -0.1299954354763031,
      "logps/chosen": -1534.652099609375,
      "logps/rejected": -1399.5792236328125,
      "loss": 0.6917384266853333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04474790394306183,
      "rewards/margins": 0.02114131860435009,
      "rewards/rejected": 0.023606587201356888,
      "step": 14
    },
    {
      "epoch": 0.08241758241758242,
      "grad_norm": 0.7331187725067139,
      "learning_rate": 1.8421052631578948e-06,
      "logits/chosen": -0.444730281829834,
      "logits/rejected": -0.5467236638069153,
      "logps/chosen": -1175.93017578125,
      "logps/rejected": -1152.0045166015625,
      "loss": 0.6896647214889526,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.01556220930069685,
      "rewards/margins": 0.008490944281220436,
      "rewards/rejected": 0.007071265950798988,
      "step": 15
    },
    {
      "epoch": 0.08791208791208792,
      "grad_norm": 0.7852977514266968,
      "learning_rate": 1.973684210526316e-06,
      "logits/chosen": -0.3976231515407562,
      "logits/rejected": -0.6108076572418213,
      "logps/chosen": -1739.89404296875,
      "logps/rejected": -1463.2957763671875,
      "loss": 0.6904494166374207,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.04959982633590698,
      "rewards/margins": 0.015834834426641464,
      "rewards/rejected": 0.03376498818397522,
      "step": 16
    },
    {
      "epoch": 0.09340659340659341,
      "grad_norm": 0.6691881418228149,
      "learning_rate": 2.105263157894737e-06,
      "logits/chosen": -0.35132962465286255,
      "logits/rejected": -0.4285813570022583,
      "logps/chosen": -1459.74755859375,
      "logps/rejected": -1310.574951171875,
      "loss": 0.6907796263694763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03758728504180908,
      "rewards/margins": 0.011371154338121414,
      "rewards/rejected": 0.026216136291623116,
      "step": 17
    },
    {
      "epoch": 0.0989010989010989,
      "grad_norm": 0.8749127984046936,
      "learning_rate": 2.236842105263158e-06,
      "logits/chosen": -0.4039854407310486,
      "logits/rejected": -0.5412226319313049,
      "logps/chosen": -1535.715087890625,
      "logps/rejected": -1592.76123046875,
      "loss": 0.6916483044624329,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.029426174238324165,
      "rewards/margins": 0.004588168114423752,
      "rewards/rejected": 0.024838006123900414,
      "step": 18
    },
    {
      "epoch": 0.1043956043956044,
      "grad_norm": 0.6890782713890076,
      "learning_rate": 2.368421052631579e-06,
      "logits/chosen": -0.5154158473014832,
      "logits/rejected": -0.4239461421966553,
      "logps/chosen": -1366.4837646484375,
      "logps/rejected": -1341.562744140625,
      "loss": 0.6925311088562012,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02481229603290558,
      "rewards/margins": 0.014975612983107567,
      "rewards/rejected": 0.009836683049798012,
      "step": 19
    },
    {
      "epoch": 0.10989010989010989,
      "grad_norm": 0.7121235728263855,
      "learning_rate": 2.5e-06,
      "logits/chosen": -0.24524424970149994,
      "logits/rejected": -0.1629214882850647,
      "logps/chosen": -1579.9754638671875,
      "logps/rejected": -1489.90625,
      "loss": 0.6927945613861084,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.008585358038544655,
      "rewards/margins": 0.009461937472224236,
      "rewards/rejected": -0.0008765789680182934,
      "step": 20
    },
    {
      "epoch": 0.11538461538461539,
      "grad_norm": 0.7799815535545349,
      "learning_rate": 2.484662576687117e-06,
      "logits/chosen": -0.7088998556137085,
      "logits/rejected": -0.6623613834381104,
      "logps/chosen": -1458.49853515625,
      "logps/rejected": -1424.711181640625,
      "loss": 0.6919652819633484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00381654710508883,
      "rewards/margins": 0.008333091624081135,
      "rewards/rejected": -0.0045165447518229485,
      "step": 21
    },
    {
      "epoch": 0.12087912087912088,
      "grad_norm": 0.9595739245414734,
      "learning_rate": 2.4693251533742334e-06,
      "logits/chosen": -0.18136945366859436,
      "logits/rejected": -0.2704053819179535,
      "logps/chosen": -1288.148193359375,
      "logps/rejected": -1242.779296875,
      "loss": 0.6891352534294128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.004841099493205547,
      "rewards/margins": 0.010067347437143326,
      "rewards/rejected": -0.014908447861671448,
      "step": 22
    },
    {
      "epoch": 0.12637362637362637,
      "grad_norm": 0.8795596957206726,
      "learning_rate": 2.45398773006135e-06,
      "logits/chosen": -0.5372273325920105,
      "logits/rejected": -0.4621075391769409,
      "logps/chosen": -1187.1552734375,
      "logps/rejected": -1210.7431640625,
      "loss": 0.6919819116592407,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02427225187420845,
      "rewards/margins": 0.0051599410362541676,
      "rewards/rejected": -0.02943219244480133,
      "step": 23
    },
    {
      "epoch": 0.13186813186813187,
      "grad_norm": 0.4124164581298828,
      "learning_rate": 2.4386503067484667e-06,
      "logits/chosen": -0.6080984473228455,
      "logits/rejected": -0.6881457567214966,
      "logps/chosen": -1473.2374267578125,
      "logps/rejected": -1214.7386474609375,
      "loss": 0.6928576231002808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.050905995070934296,
      "rewards/margins": -0.004748010542243719,
      "rewards/rejected": -0.04615798965096474,
      "step": 24
    },
    {
      "epoch": 0.13736263736263737,
      "grad_norm": 0.2888234555721283,
      "learning_rate": 2.423312883435583e-06,
      "logits/chosen": -0.31380707025527954,
      "logits/rejected": -0.2592722773551941,
      "logps/chosen": -1513.1060791015625,
      "logps/rejected": -1577.8150634765625,
      "loss": 0.6926202178001404,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06563874334096909,
      "rewards/margins": 0.008732328191399574,
      "rewards/rejected": -0.07437106966972351,
      "step": 25
    },
    {
      "epoch": 0.14285714285714285,
      "grad_norm": 0.5071586966514587,
      "learning_rate": 2.4079754601226995e-06,
      "logits/chosen": -0.6376692056655884,
      "logits/rejected": -0.6288321614265442,
      "logps/chosen": -1425.6273193359375,
      "logps/rejected": -1174.107421875,
      "loss": 0.6917777061462402,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.039514925330877304,
      "rewards/margins": -0.0018771695904433727,
      "rewards/rejected": -0.03763775900006294,
      "step": 26
    },
    {
      "epoch": 0.14835164835164835,
      "grad_norm": 0.7409914135932922,
      "learning_rate": 2.392638036809816e-06,
      "logits/chosen": -0.37571296095848083,
      "logits/rejected": -0.3492288589477539,
      "logps/chosen": -1434.89697265625,
      "logps/rejected": -1126.47314453125,
      "loss": 0.6918377876281738,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.019896959885954857,
      "rewards/margins": 0.004404466599225998,
      "rewards/rejected": -0.024301424622535706,
      "step": 27
    },
    {
      "epoch": 0.15384615384615385,
      "grad_norm": 1.4375437498092651,
      "learning_rate": 2.3773006134969327e-06,
      "logits/chosen": -0.4284808337688446,
      "logits/rejected": -0.4831678867340088,
      "logps/chosen": -1346.5638427734375,
      "logps/rejected": -1136.7266845703125,
      "loss": 0.6918051838874817,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.022301796823740005,
      "rewards/margins": -0.003937081433832645,
      "rewards/rejected": -0.018364714458584785,
      "step": 28
    },
    {
      "epoch": 0.15934065934065933,
      "grad_norm": 0.48184096813201904,
      "learning_rate": 2.3619631901840493e-06,
      "logits/chosen": -0.6718910336494446,
      "logits/rejected": -0.7271767854690552,
      "logps/chosen": -1507.7259521484375,
      "logps/rejected": -1534.1622314453125,
      "loss": 0.6906634569168091,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.05005672574043274,
      "rewards/margins": 0.0061715878546237946,
      "rewards/rejected": 0.043885137885808945,
      "step": 29
    },
    {
      "epoch": 0.16483516483516483,
      "grad_norm": 0.6680644750595093,
      "learning_rate": 2.346625766871166e-06,
      "logits/chosen": -0.5794727206230164,
      "logits/rejected": -0.3858964741230011,
      "logps/chosen": -1158.666748046875,
      "logps/rejected": -1081.162109375,
      "loss": 0.6908907890319824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012693958356976509,
      "rewards/margins": 0.008182154037058353,
      "rewards/rejected": 0.004511804785579443,
      "step": 30
    },
    {
      "epoch": 0.17032967032967034,
      "grad_norm": 0.9391990900039673,
      "learning_rate": 2.331288343558282e-06,
      "logits/chosen": -0.5199452638626099,
      "logits/rejected": -0.6326333284378052,
      "logps/chosen": -1681.7022705078125,
      "logps/rejected": -1511.8980712890625,
      "loss": 0.6941480040550232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.027041777968406677,
      "rewards/margins": -0.001771755749359727,
      "rewards/rejected": 0.02881353348493576,
      "step": 31
    },
    {
      "epoch": 0.17582417582417584,
      "grad_norm": 0.46425074338912964,
      "learning_rate": 2.3159509202453988e-06,
      "logits/chosen": -0.11656185239553452,
      "logits/rejected": -0.18580833077430725,
      "logps/chosen": -1278.0001220703125,
      "logps/rejected": -1106.93408203125,
      "loss": 0.692402720451355,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.022857239469885826,
      "rewards/margins": 0.0052628712728619576,
      "rewards/rejected": -0.02812010981142521,
      "step": 32
    },
    {
      "epoch": 0.1813186813186813,
      "grad_norm": 0.6273348331451416,
      "learning_rate": 2.3006134969325154e-06,
      "logits/chosen": -0.5983235836029053,
      "logits/rejected": -0.4410645663738251,
      "logps/chosen": -1521.9276123046875,
      "logps/rejected": -1621.8740234375,
      "loss": 0.6908812522888184,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.011174187064170837,
      "rewards/margins": 0.010386137291789055,
      "rewards/rejected": -0.02156032808125019,
      "step": 33
    },
    {
      "epoch": 0.18681318681318682,
      "grad_norm": 0.9936891794204712,
      "learning_rate": 2.2852760736196324e-06,
      "logits/chosen": -0.6642717719078064,
      "logits/rejected": -0.6868173480033875,
      "logps/chosen": -1817.156494140625,
      "logps/rejected": -1669.7921142578125,
      "loss": 0.6925203800201416,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.02615175023674965,
      "rewards/margins": -0.004008379764854908,
      "rewards/rejected": -0.022143369540572166,
      "step": 34
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 0.297181099653244,
      "learning_rate": 2.2699386503067486e-06,
      "logits/chosen": -0.8816071152687073,
      "logits/rejected": -0.7729086875915527,
      "logps/chosen": -1493.5185546875,
      "logps/rejected": -1499.0738525390625,
      "loss": 0.6920033693313599,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.04162220656871796,
      "rewards/margins": 0.004699449986219406,
      "rewards/rejected": -0.04632166028022766,
      "step": 35
    },
    {
      "epoch": 0.1978021978021978,
      "grad_norm": 0.6562932729721069,
      "learning_rate": 2.2546012269938652e-06,
      "logits/chosen": -0.4074722230434418,
      "logits/rejected": -0.5253076553344727,
      "logps/chosen": -1445.3853759765625,
      "logps/rejected": -1373.7484130859375,
      "loss": 0.6925094127655029,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.038453079760074615,
      "rewards/margins": -0.002792082028463483,
      "rewards/rejected": -0.0356609970331192,
      "step": 36
    },
    {
      "epoch": 0.2032967032967033,
      "grad_norm": 0.42993712425231934,
      "learning_rate": 2.239263803680982e-06,
      "logits/chosen": -0.10876737534999847,
      "logits/rejected": -0.1684437394142151,
      "logps/chosen": -1450.543701171875,
      "logps/rejected": -1280.059326171875,
      "loss": 0.6924898624420166,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.0590565986931324,
      "rewards/margins": -0.012079971842467785,
      "rewards/rejected": -0.04697662591934204,
      "step": 37
    },
    {
      "epoch": 0.2087912087912088,
      "grad_norm": 0.6291781067848206,
      "learning_rate": 2.2239263803680985e-06,
      "logits/chosen": -0.4873208999633789,
      "logits/rejected": -0.5962563753128052,
      "logps/chosen": -1363.3477783203125,
      "logps/rejected": -1316.7086181640625,
      "loss": 0.6922313570976257,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.03899176046252251,
      "rewards/margins": 0.0033729556016623974,
      "rewards/rejected": -0.04236472398042679,
      "step": 38
    },
    {
      "epoch": 0.21428571428571427,
      "grad_norm": 0.6414478421211243,
      "learning_rate": 2.208588957055215e-06,
      "logits/chosen": -0.3744695782661438,
      "logits/rejected": -0.45585522055625916,
      "logps/chosen": -1323.5474853515625,
      "logps/rejected": -1350.0068359375,
      "loss": 0.6929305791854858,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0353596955537796,
      "rewards/margins": 0.007274856325238943,
      "rewards/rejected": -0.04263455420732498,
      "step": 39
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 1.121484398841858,
      "learning_rate": 2.1932515337423317e-06,
      "logits/chosen": -0.3999641537666321,
      "logits/rejected": -0.5990539789199829,
      "logps/chosen": -1318.0543212890625,
      "logps/rejected": -1136.1405029296875,
      "loss": 0.6921414136886597,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0097041679546237,
      "rewards/margins": 0.002448990475386381,
      "rewards/rejected": -0.012153157964348793,
      "step": 40
    },
    {
      "epoch": 0.22527472527472528,
      "grad_norm": 1.376361608505249,
      "learning_rate": 2.177914110429448e-06,
      "logits/chosen": -0.538985550403595,
      "logits/rejected": -0.3736581802368164,
      "logps/chosen": -1605.4700927734375,
      "logps/rejected": -1410.381103515625,
      "loss": 0.6900919079780579,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.015452098101377487,
      "rewards/margins": 0.001963262911885977,
      "rewards/rejected": -0.01741536147892475,
      "step": 41
    },
    {
      "epoch": 0.23076923076923078,
      "grad_norm": 0.9494011402130127,
      "learning_rate": 2.1625766871165645e-06,
      "logits/chosen": -0.3661588430404663,
      "logits/rejected": -0.30248114466667175,
      "logps/chosen": -1482.0816650390625,
      "logps/rejected": -1587.1827392578125,
      "loss": 0.6923238635063171,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011823957785964012,
      "rewards/margins": 0.004251518286764622,
      "rewards/rejected": -0.01607547700405121,
      "step": 42
    },
    {
      "epoch": 0.23626373626373626,
      "grad_norm": 0.9896334409713745,
      "learning_rate": 2.147239263803681e-06,
      "logits/chosen": -0.5797193646430969,
      "logits/rejected": -0.6257858276367188,
      "logps/chosen": -1530.91552734375,
      "logps/rejected": -1547.6806640625,
      "loss": 0.6907969117164612,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021539613604545593,
      "rewards/margins": 0.01467025838792324,
      "rewards/rejected": -0.036209866404533386,
      "step": 43
    },
    {
      "epoch": 0.24175824175824176,
      "grad_norm": 0.9827613830566406,
      "learning_rate": 2.1319018404907978e-06,
      "logits/chosen": -0.7638057470321655,
      "logits/rejected": -0.8149251937866211,
      "logps/chosen": -1518.649169921875,
      "logps/rejected": -1399.217529296875,
      "loss": 0.6919956207275391,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.00867112074047327,
      "rewards/margins": 0.0037965585943311453,
      "rewards/rejected": -0.01246767956763506,
      "step": 44
    },
    {
      "epoch": 0.24725274725274726,
      "grad_norm": 1.2675107717514038,
      "learning_rate": 2.1165644171779144e-06,
      "logits/chosen": -0.2081916481256485,
      "logits/rejected": -0.17389225959777832,
      "logps/chosen": -1245.876708984375,
      "logps/rejected": -1316.34033203125,
      "loss": 0.6916300654411316,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0010032318532466888,
      "rewards/margins": 0.013676752336323261,
      "rewards/rejected": -0.012673520483076572,
      "step": 45
    },
    {
      "epoch": 0.25274725274725274,
      "grad_norm": 0.6310904622077942,
      "learning_rate": 2.101226993865031e-06,
      "logits/chosen": -0.6055498123168945,
      "logits/rejected": -0.5735322833061218,
      "logps/chosen": -1330.6837158203125,
      "logps/rejected": -1323.1474609375,
      "loss": 0.6929526925086975,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.03848934546113014,
      "rewards/margins": 0.0024320981465280056,
      "rewards/rejected": 0.0360572449862957,
      "step": 46
    },
    {
      "epoch": 0.25824175824175827,
      "grad_norm": 0.5979546308517456,
      "learning_rate": 2.085889570552147e-06,
      "logits/chosen": -0.19976076483726501,
      "logits/rejected": -0.3477155566215515,
      "logps/chosen": -1781.2822265625,
      "logps/rejected": -1497.507080078125,
      "loss": 0.691817045211792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.057623643428087234,
      "rewards/margins": 0.01321370154619217,
      "rewards/rejected": 0.044409941881895065,
      "step": 47
    },
    {
      "epoch": 0.26373626373626374,
      "grad_norm": 1.287549376487732,
      "learning_rate": 2.070552147239264e-06,
      "logits/chosen": -0.09788006544113159,
      "logits/rejected": -0.1335160732269287,
      "logps/chosen": -1499.1783447265625,
      "logps/rejected": -1211.41064453125,
      "loss": 0.6919518113136292,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.03057347610592842,
      "rewards/margins": 0.01761637255549431,
      "rewards/rejected": 0.012957105413079262,
      "step": 48
    },
    {
      "epoch": 0.2692307692307692,
      "grad_norm": 0.6202033758163452,
      "learning_rate": 2.0552147239263804e-06,
      "logits/chosen": -0.31665268540382385,
      "logits/rejected": -0.3232308626174927,
      "logps/chosen": -1485.8748779296875,
      "logps/rejected": -1326.1854248046875,
      "loss": 0.6928448677062988,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.07038865238428116,
      "rewards/margins": 0.009158019907772541,
      "rewards/rejected": 0.061230629682540894,
      "step": 49
    },
    {
      "epoch": 0.27472527472527475,
      "grad_norm": 0.5841394066810608,
      "learning_rate": 2.039877300613497e-06,
      "logits/chosen": -0.11269676685333252,
      "logits/rejected": -0.12115436792373657,
      "logps/chosen": -1302.87255859375,
      "logps/rejected": -1420.4437255859375,
      "loss": 0.6917054057121277,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.04229995980858803,
      "rewards/margins": 0.0015316582284867764,
      "rewards/rejected": 0.04076829552650452,
      "step": 50
    },
    {
      "epoch": 0.2802197802197802,
      "grad_norm": 0.7443718910217285,
      "learning_rate": 2.0245398773006137e-06,
      "logits/chosen": -0.49943816661834717,
      "logits/rejected": -0.5221395492553711,
      "logps/chosen": -1615.616943359375,
      "logps/rejected": -1565.77783203125,
      "loss": 0.6914383172988892,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04047242924571037,
      "rewards/margins": 0.011208095587790012,
      "rewards/rejected": 0.029264334589242935,
      "step": 51
    },
    {
      "epoch": 0.2857142857142857,
      "grad_norm": 1.0812468528747559,
      "learning_rate": 2.0092024539877303e-06,
      "logits/chosen": -0.5125001072883606,
      "logits/rejected": -0.5849840641021729,
      "logps/chosen": -1389.961181640625,
      "logps/rejected": -1265.2818603515625,
      "loss": 0.6922411322593689,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.008559723384678364,
      "rewards/margins": 0.001604671822860837,
      "rewards/rejected": 0.006955050863325596,
      "step": 52
    },
    {
      "epoch": 0.29120879120879123,
      "grad_norm": 1.38298761844635,
      "learning_rate": 1.9938650306748465e-06,
      "logits/chosen": -0.8314374685287476,
      "logits/rejected": -0.8688368797302246,
      "logps/chosen": -1503.9605712890625,
      "logps/rejected": -1391.91357421875,
      "loss": 0.6904184818267822,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0023605534806847572,
      "rewards/margins": 0.010243010707199574,
      "rewards/rejected": -0.01260356418788433,
      "step": 53
    },
    {
      "epoch": 0.2967032967032967,
      "grad_norm": 0.7907947301864624,
      "learning_rate": 1.9785276073619635e-06,
      "logits/chosen": -0.4663585126399994,
      "logits/rejected": -0.5531838536262512,
      "logps/chosen": -1565.290771484375,
      "logps/rejected": -1368.54833984375,
      "loss": 0.6908788681030273,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04032035544514656,
      "rewards/margins": 0.011873043142259121,
      "rewards/rejected": 0.028447313234210014,
      "step": 54
    },
    {
      "epoch": 0.3021978021978022,
      "grad_norm": 1.203404426574707,
      "learning_rate": 1.96319018404908e-06,
      "logits/chosen": -0.38841575384140015,
      "logits/rejected": -0.43689727783203125,
      "logps/chosen": -1580.765625,
      "logps/rejected": -1455.60498046875,
      "loss": 0.6918849945068359,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011032314039766788,
      "rewards/margins": 0.009747497737407684,
      "rewards/rejected": 0.0012848172336816788,
      "step": 55
    },
    {
      "epoch": 0.3076923076923077,
      "grad_norm": 0.7074547410011292,
      "learning_rate": 1.9478527607361967e-06,
      "logits/chosen": -0.3431719243526459,
      "logits/rejected": -0.2189161628484726,
      "logps/chosen": -1363.068603515625,
      "logps/rejected": -1273.5313720703125,
      "loss": 0.6918089389801025,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.029067430645227432,
      "rewards/margins": 0.013189269229769707,
      "rewards/rejected": 0.015878159552812576,
      "step": 56
    },
    {
      "epoch": 0.3131868131868132,
      "grad_norm": 0.8552976250648499,
      "learning_rate": 1.932515337423313e-06,
      "logits/chosen": -0.3242160379886627,
      "logits/rejected": -0.36963948607444763,
      "logps/chosen": -1408.054443359375,
      "logps/rejected": -1515.0474853515625,
      "loss": 0.691418468952179,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02214316464960575,
      "rewards/margins": 0.008005449548363686,
      "rewards/rejected": 0.01413771416991949,
      "step": 57
    },
    {
      "epoch": 0.31868131868131866,
      "grad_norm": 0.9317906498908997,
      "learning_rate": 1.9171779141104296e-06,
      "logits/chosen": -0.651016354560852,
      "logits/rejected": -0.646873414516449,
      "logps/chosen": -1354.4619140625,
      "logps/rejected": -1355.021728515625,
      "loss": 0.6914571523666382,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009268360212445259,
      "rewards/margins": 0.00419324915856123,
      "rewards/rejected": 0.005075111519545317,
      "step": 58
    },
    {
      "epoch": 0.3241758241758242,
      "grad_norm": 0.6546081304550171,
      "learning_rate": 1.9018404907975464e-06,
      "logits/chosen": -0.4001083970069885,
      "logits/rejected": -0.34092509746551514,
      "logps/chosen": -1422.7691650390625,
      "logps/rejected": -1336.7745361328125,
      "loss": 0.6927464008331299,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04447000473737717,
      "rewards/margins": 0.011220641434192657,
      "rewards/rejected": 0.03324935957789421,
      "step": 59
    },
    {
      "epoch": 0.32967032967032966,
      "grad_norm": 1.2057020664215088,
      "learning_rate": 1.8865030674846626e-06,
      "logits/chosen": -0.40792492032051086,
      "logits/rejected": -0.41026824712753296,
      "logps/chosen": -1616.24462890625,
      "logps/rejected": -1496.211669921875,
      "loss": 0.6902381181716919,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019096756353974342,
      "rewards/margins": 0.012553432956337929,
      "rewards/rejected": 0.00654332060366869,
      "step": 60
    },
    {
      "epoch": 0.33516483516483514,
      "grad_norm": 0.8189549446105957,
      "learning_rate": 1.8711656441717794e-06,
      "logits/chosen": -0.40066254138946533,
      "logits/rejected": -0.5438181757926941,
      "logps/chosen": -1495.165771484375,
      "logps/rejected": -1468.3994140625,
      "loss": 0.6911718845367432,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026379263028502464,
      "rewards/margins": 0.010899864137172699,
      "rewards/rejected": 0.01547939982265234,
      "step": 61
    },
    {
      "epoch": 0.34065934065934067,
      "grad_norm": 0.7142252922058105,
      "learning_rate": 1.855828220858896e-06,
      "logits/chosen": -0.4213395416736603,
      "logits/rejected": -0.4220888316631317,
      "logps/chosen": -1588.5791015625,
      "logps/rejected": -1450.305908203125,
      "loss": 0.6918849945068359,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.044086188077926636,
      "rewards/margins": 0.021000908687710762,
      "rewards/rejected": 0.023085277527570724,
      "step": 62
    },
    {
      "epoch": 0.34615384615384615,
      "grad_norm": 0.931616485118866,
      "learning_rate": 1.8404907975460124e-06,
      "logits/chosen": -0.4536420404911041,
      "logits/rejected": -0.553905189037323,
      "logps/chosen": -1479.9383544921875,
      "logps/rejected": -1247.5972900390625,
      "loss": 0.6907938122749329,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0278279110789299,
      "rewards/margins": 0.017392883077263832,
      "rewards/rejected": 0.010435027070343494,
      "step": 63
    },
    {
      "epoch": 0.3516483516483517,
      "grad_norm": 1.1059670448303223,
      "learning_rate": 1.825153374233129e-06,
      "logits/chosen": -0.1391632854938507,
      "logits/rejected": -0.29146745800971985,
      "logps/chosen": -1544.81103515625,
      "logps/rejected": -1467.8116455078125,
      "loss": 0.6943617463111877,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004199332557618618,
      "rewards/margins": -0.0030631115660071373,
      "rewards/rejected": 0.007262444589287043,
      "step": 64
    },
    {
      "epoch": 0.35714285714285715,
      "grad_norm": 2.532100200653076,
      "learning_rate": 1.8098159509202457e-06,
      "logits/chosen": -0.4176465570926666,
      "logits/rejected": -0.47569915652275085,
      "logps/chosen": -1727.229736328125,
      "logps/rejected": -1618.333984375,
      "loss": 0.6903217434883118,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.008169951848685741,
      "rewards/margins": 0.0118106072768569,
      "rewards/rejected": -0.01998055912554264,
      "step": 65
    },
    {
      "epoch": 0.3626373626373626,
      "grad_norm": 0.8319610953330994,
      "learning_rate": 1.794478527607362e-06,
      "logits/chosen": -0.5535542964935303,
      "logits/rejected": -0.45863211154937744,
      "logps/chosen": -1296.4996337890625,
      "logps/rejected": -1305.611083984375,
      "loss": 0.6914780139923096,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.003835830604657531,
      "rewards/margins": 0.00748834665864706,
      "rewards/rejected": -0.0036525153554975986,
      "step": 66
    },
    {
      "epoch": 0.36813186813186816,
      "grad_norm": 1.074711799621582,
      "learning_rate": 1.7791411042944787e-06,
      "logits/chosen": -0.6282920241355896,
      "logits/rejected": -0.6052903532981873,
      "logps/chosen": -1367.955078125,
      "logps/rejected": -1418.4644775390625,
      "loss": 0.691602349281311,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008508214727044106,
      "rewards/margins": 0.0025658695958554745,
      "rewards/rejected": -0.011074084788560867,
      "step": 67
    },
    {
      "epoch": 0.37362637362637363,
      "grad_norm": 1.048543930053711,
      "learning_rate": 1.7638036809815953e-06,
      "logits/chosen": -0.41451916098594666,
      "logits/rejected": -0.44973787665367126,
      "logps/chosen": -1594.5008544921875,
      "logps/rejected": -1379.6549072265625,
      "loss": 0.6915735602378845,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.013657140545547009,
      "rewards/margins": 0.010285510681569576,
      "rewards/rejected": 0.003371629398316145,
      "step": 68
    },
    {
      "epoch": 0.3791208791208791,
      "grad_norm": 1.4657976627349854,
      "learning_rate": 1.7484662576687117e-06,
      "logits/chosen": -0.42228737473487854,
      "logits/rejected": -0.5371153354644775,
      "logps/chosen": -1440.2550048828125,
      "logps/rejected": -1317.087890625,
      "loss": 0.6919211745262146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004323349334299564,
      "rewards/margins": 0.007630740292370319,
      "rewards/rejected": -0.0033073904924094677,
      "step": 69
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 0.7367867827415466,
      "learning_rate": 1.7331288343558283e-06,
      "logits/chosen": -0.741744875907898,
      "logits/rejected": -0.6823490858078003,
      "logps/chosen": -1317.0145263671875,
      "logps/rejected": -1323.838623046875,
      "loss": 0.6901740431785583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011306552216410637,
      "rewards/margins": 0.013367671519517899,
      "rewards/rejected": -0.0020611193031072617,
      "step": 70
    },
    {
      "epoch": 0.3901098901098901,
      "grad_norm": 1.0581424236297607,
      "learning_rate": 1.717791411042945e-06,
      "logits/chosen": -0.36263424158096313,
      "logits/rejected": -0.37243887782096863,
      "logps/chosen": -1136.444580078125,
      "logps/rejected": -1088.967529296875,
      "loss": 0.6923285722732544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009710215963423252,
      "rewards/margins": 0.0008770937565714121,
      "rewards/rejected": 0.008833122439682484,
      "step": 71
    },
    {
      "epoch": 0.3956043956043956,
      "grad_norm": 0.9798324704170227,
      "learning_rate": 1.7024539877300614e-06,
      "logits/chosen": -0.3991851508617401,
      "logits/rejected": -0.3783729374408722,
      "logps/chosen": -1416.0003662109375,
      "logps/rejected": -1323.023681640625,
      "loss": 0.6924745440483093,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.013836746104061604,
      "rewards/margins": 0.005006732419133186,
      "rewards/rejected": 0.008830012753605843,
      "step": 72
    },
    {
      "epoch": 0.4010989010989011,
      "grad_norm": 0.8338879346847534,
      "learning_rate": 1.687116564417178e-06,
      "logits/chosen": -0.19355978071689606,
      "logits/rejected": -0.2498997300863266,
      "logps/chosen": -1318.318359375,
      "logps/rejected": -1202.7623291015625,
      "loss": 0.6920049786567688,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.010977883823215961,
      "rewards/margins": 0.0031725564040243626,
      "rewards/rejected": 0.0078053283505141735,
      "step": 73
    },
    {
      "epoch": 0.4065934065934066,
      "grad_norm": 0.8426941633224487,
      "learning_rate": 1.6717791411042946e-06,
      "logits/chosen": -0.46479836106300354,
      "logits/rejected": -0.42796429991722107,
      "logps/chosen": -951.9476928710938,
      "logps/rejected": -1005.59521484375,
      "loss": 0.6904844045639038,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.011304330080747604,
      "rewards/margins": 0.0059456657618284225,
      "rewards/rejected": 0.005358664318919182,
      "step": 74
    },
    {
      "epoch": 0.41208791208791207,
      "grad_norm": 1.5550216436386108,
      "learning_rate": 1.656441717791411e-06,
      "logits/chosen": -0.6027710437774658,
      "logits/rejected": -0.632501482963562,
      "logps/chosen": -1343.49755859375,
      "logps/rejected": -1283.693115234375,
      "loss": 0.6909563541412354,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006782836746424437,
      "rewards/margins": 0.009988861158490181,
      "rewards/rejected": -0.003206023946404457,
      "step": 75
    },
    {
      "epoch": 0.4175824175824176,
      "grad_norm": 0.9994601607322693,
      "learning_rate": 1.6411042944785276e-06,
      "logits/chosen": -0.5772875547409058,
      "logits/rejected": -0.5767691731452942,
      "logps/chosen": -1284.95263671875,
      "logps/rejected": -1317.3885498046875,
      "loss": 0.6914191842079163,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002038019709289074,
      "rewards/margins": 0.01685798540711403,
      "rewards/rejected": -0.01481996476650238,
      "step": 76
    },
    {
      "epoch": 0.4230769230769231,
      "grad_norm": 1.4171066284179688,
      "learning_rate": 1.6257668711656445e-06,
      "logits/chosen": -0.4665534794330597,
      "logits/rejected": -0.432694673538208,
      "logps/chosen": -1695.040283203125,
      "logps/rejected": -1551.968505859375,
      "loss": 0.6929216384887695,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.02615145593881607,
      "rewards/margins": -0.0027590510435402393,
      "rewards/rejected": -0.02339240349829197,
      "step": 77
    },
    {
      "epoch": 0.42857142857142855,
      "grad_norm": 1.1727432012557983,
      "learning_rate": 1.6104294478527606e-06,
      "logits/chosen": -0.5112898349761963,
      "logits/rejected": -0.5043646097183228,
      "logps/chosen": -1548.087646484375,
      "logps/rejected": -1400.1192626953125,
      "loss": 0.6917024254798889,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.05165726691484451,
      "rewards/margins": -0.012301253154873848,
      "rewards/rejected": -0.039356015622615814,
      "step": 78
    },
    {
      "epoch": 0.4340659340659341,
      "grad_norm": 0.4238358736038208,
      "learning_rate": 1.5950920245398775e-06,
      "logits/chosen": -0.30220383405685425,
      "logits/rejected": -0.300650417804718,
      "logps/chosen": -1494.92333984375,
      "logps/rejected": -1436.374267578125,
      "loss": 0.6916438341140747,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.05291643738746643,
      "rewards/margins": -0.008324610069394112,
      "rewards/rejected": -0.04459183290600777,
      "step": 79
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 0.9021973013877869,
      "learning_rate": 1.579754601226994e-06,
      "logits/chosen": -0.10122132301330566,
      "logits/rejected": -0.2252417355775833,
      "logps/chosen": -1179.5601806640625,
      "logps/rejected": -1048.575439453125,
      "loss": 0.6910136342048645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021670779213309288,
      "rewards/margins": 0.0014739391626790166,
      "rewards/rejected": -0.023144718259572983,
      "step": 80
    },
    {
      "epoch": 0.44505494505494503,
      "grad_norm": 0.9659970998764038,
      "learning_rate": 1.5644171779141107e-06,
      "logits/chosen": -0.5470824837684631,
      "logits/rejected": -0.48158833384513855,
      "logps/chosen": -1314.2874755859375,
      "logps/rejected": -1259.9593505859375,
      "loss": 0.6916049718856812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01713727042078972,
      "rewards/margins": 0.005170955788344145,
      "rewards/rejected": -0.02230822481215,
      "step": 81
    },
    {
      "epoch": 0.45054945054945056,
      "grad_norm": 1.1842788457870483,
      "learning_rate": 1.5490797546012271e-06,
      "logits/chosen": -0.33465588092803955,
      "logits/rejected": -0.33280038833618164,
      "logps/chosen": -1350.5224609375,
      "logps/rejected": -1351.6611328125,
      "loss": 0.6924982666969299,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.011164035648107529,
      "rewards/margins": 0.004191071260720491,
      "rewards/rejected": -0.015355108305811882,
      "step": 82
    },
    {
      "epoch": 0.45604395604395603,
      "grad_norm": 1.148301124572754,
      "learning_rate": 1.5337423312883437e-06,
      "logits/chosen": -0.4244360625743866,
      "logits/rejected": -0.49099138379096985,
      "logps/chosen": -1331.2615966796875,
      "logps/rejected": -1181.3885498046875,
      "loss": 0.6928058862686157,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.007538508623838425,
      "rewards/margins": 0.0015701393131166697,
      "rewards/rejected": -0.00910864770412445,
      "step": 83
    },
    {
      "epoch": 0.46153846153846156,
      "grad_norm": 1.7485556602478027,
      "learning_rate": 1.5184049079754604e-06,
      "logits/chosen": -0.42688897252082825,
      "logits/rejected": -0.5053452253341675,
      "logps/chosen": -1439.1773681640625,
      "logps/rejected": -1262.5604248046875,
      "loss": 0.6921294331550598,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0023087309673428535,
      "rewards/margins": 0.0033039902336895466,
      "rewards/rejected": -0.0056127216666936874,
      "step": 84
    },
    {
      "epoch": 0.46703296703296704,
      "grad_norm": 1.6599191427230835,
      "learning_rate": 1.5030674846625768e-06,
      "logits/chosen": -0.43120938539505005,
      "logits/rejected": -0.6010267734527588,
      "logps/chosen": -1558.959228515625,
      "logps/rejected": -1472.342529296875,
      "loss": 0.6921494007110596,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.022558651864528656,
      "rewards/margins": 0.005641758441925049,
      "rewards/rejected": 0.016916893422603607,
      "step": 85
    },
    {
      "epoch": 0.4725274725274725,
      "grad_norm": 1.0252467393875122,
      "learning_rate": 1.4877300613496934e-06,
      "logits/chosen": -0.37089625000953674,
      "logits/rejected": -0.35030075907707214,
      "logps/chosen": -1299.443115234375,
      "logps/rejected": -1176.6553955078125,
      "loss": 0.6907151341438293,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.024281539022922516,
      "rewards/margins": 0.009798645041882992,
      "rewards/rejected": 0.014482893981039524,
      "step": 86
    },
    {
      "epoch": 0.47802197802197804,
      "grad_norm": 1.026886224746704,
      "learning_rate": 1.47239263803681e-06,
      "logits/chosen": -0.601810872554779,
      "logits/rejected": -0.5252466201782227,
      "logps/chosen": -1495.2203369140625,
      "logps/rejected": -1310.950927734375,
      "loss": 0.6917718648910522,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0073541309684515,
      "rewards/margins": 0.007022920064628124,
      "rewards/rejected": 0.00033121160231530666,
      "step": 87
    },
    {
      "epoch": 0.4835164835164835,
      "grad_norm": 0.6109346151351929,
      "learning_rate": 1.4570552147239264e-06,
      "logits/chosen": -0.3959222435951233,
      "logits/rejected": -0.2995205819606781,
      "logps/chosen": -1450.761474609375,
      "logps/rejected": -1450.51953125,
      "loss": 0.6905621886253357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0416596420109272,
      "rewards/margins": 0.013335322961211205,
      "rewards/rejected": 0.028324317187070847,
      "step": 88
    },
    {
      "epoch": 0.489010989010989,
      "grad_norm": 0.8334896564483643,
      "learning_rate": 1.441717791411043e-06,
      "logits/chosen": -0.3118741810321808,
      "logits/rejected": -0.4128867983818054,
      "logps/chosen": -1281.40478515625,
      "logps/rejected": -1296.41357421875,
      "loss": 0.6907057762145996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014409005641937256,
      "rewards/margins": 0.014574131928384304,
      "rewards/rejected": -0.0001651242928346619,
      "step": 89
    },
    {
      "epoch": 0.4945054945054945,
      "grad_norm": 0.8609052300453186,
      "learning_rate": 1.4263803680981596e-06,
      "logits/chosen": -0.2428097128868103,
      "logits/rejected": -0.23511645197868347,
      "logps/chosen": -1411.8843994140625,
      "logps/rejected": -1272.73876953125,
      "loss": 0.6919518709182739,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.026493338868021965,
      "rewards/margins": 0.014896931126713753,
      "rewards/rejected": 0.011596407741308212,
      "step": 90
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.7259723544120789,
      "learning_rate": 1.411042944785276e-06,
      "logits/chosen": -0.11918177455663681,
      "logits/rejected": -0.07845693826675415,
      "logps/chosen": -1513.873779296875,
      "logps/rejected": -1462.4080810546875,
      "loss": 0.6918542981147766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0520310252904892,
      "rewards/margins": 0.011454623192548752,
      "rewards/rejected": 0.040576402097940445,
      "step": 91
    },
    {
      "epoch": 0.5054945054945055,
      "grad_norm": 1.1002850532531738,
      "learning_rate": 1.3957055214723927e-06,
      "logits/chosen": -0.2559316158294678,
      "logits/rejected": -0.200018048286438,
      "logps/chosen": -1493.306396484375,
      "logps/rejected": -1425.4647216796875,
      "loss": 0.6930555105209351,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02612033672630787,
      "rewards/margins": -0.002100848825648427,
      "rewards/rejected": 0.028221188113093376,
      "step": 92
    },
    {
      "epoch": 0.510989010989011,
      "grad_norm": 0.7148520946502686,
      "learning_rate": 1.3803680981595095e-06,
      "logits/chosen": -0.367002010345459,
      "logits/rejected": -0.4066450893878937,
      "logps/chosen": -1357.5050048828125,
      "logps/rejected": -1381.8837890625,
      "loss": 0.6922594308853149,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.04670805484056473,
      "rewards/margins": 0.009145058691501617,
      "rewards/rejected": 0.03756299987435341,
      "step": 93
    },
    {
      "epoch": 0.5164835164835165,
      "grad_norm": 1.1037565469741821,
      "learning_rate": 1.3650306748466257e-06,
      "logits/chosen": -0.5499804615974426,
      "logits/rejected": -0.45308274030685425,
      "logps/chosen": -1548.565185546875,
      "logps/rejected": -1470.4267578125,
      "loss": 0.6900733709335327,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.038844555616378784,
      "rewards/margins": 0.01671256124973297,
      "rewards/rejected": 0.022131996229290962,
      "step": 94
    },
    {
      "epoch": 0.521978021978022,
      "grad_norm": 0.8843666911125183,
      "learning_rate": 1.3496932515337425e-06,
      "logits/chosen": 0.04877326637506485,
      "logits/rejected": -0.005727946758270264,
      "logps/chosen": -1560.8104248046875,
      "logps/rejected": -1314.3270263671875,
      "loss": 0.6904133558273315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.030398786067962646,
      "rewards/margins": 0.02400778792798519,
      "rewards/rejected": 0.006391000468283892,
      "step": 95
    },
    {
      "epoch": 0.5274725274725275,
      "grad_norm": 1.0202571153640747,
      "learning_rate": 1.3343558282208591e-06,
      "logits/chosen": -0.12934473156929016,
      "logits/rejected": -0.175532728433609,
      "logps/chosen": -1101.5224609375,
      "logps/rejected": -1186.7508544921875,
      "loss": 0.6912416219711304,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01369690801948309,
      "rewards/margins": 0.005995034705847502,
      "rewards/rejected": 0.007701873779296875,
      "step": 96
    },
    {
      "epoch": 0.532967032967033,
      "grad_norm": 1.3046600818634033,
      "learning_rate": 1.3190184049079755e-06,
      "logits/chosen": -0.3259139955043793,
      "logits/rejected": -0.36783546209335327,
      "logps/chosen": -1740.535888671875,
      "logps/rejected": -1574.8447265625,
      "loss": 0.6932768821716309,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.02200869657099247,
      "rewards/margins": 0.005453357473015785,
      "rewards/rejected": 0.016555337235331535,
      "step": 97
    },
    {
      "epoch": 0.5384615384615384,
      "grad_norm": 1.0123155117034912,
      "learning_rate": 1.3036809815950922e-06,
      "logits/chosen": -0.13210992515087128,
      "logits/rejected": -0.04369845241308212,
      "logps/chosen": -1536.9061279296875,
      "logps/rejected": -1519.1163330078125,
      "loss": 0.6910783648490906,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01258872076869011,
      "rewards/margins": 0.010054588317871094,
      "rewards/rejected": 0.002534131519496441,
      "step": 98
    },
    {
      "epoch": 0.5439560439560439,
      "grad_norm": 1.3854718208312988,
      "learning_rate": 1.2883435582822088e-06,
      "logits/chosen": -0.18825362622737885,
      "logits/rejected": -0.24444834887981415,
      "logps/chosen": -1398.3173828125,
      "logps/rejected": -1364.36865234375,
      "loss": 0.6922256946563721,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01920771598815918,
      "rewards/margins": 0.014828749001026154,
      "rewards/rejected": 0.004378966521471739,
      "step": 99
    },
    {
      "epoch": 0.5494505494505495,
      "grad_norm": 1.2779775857925415,
      "learning_rate": 1.2730061349693252e-06,
      "logits/chosen": -0.4430152475833893,
      "logits/rejected": -0.35647666454315186,
      "logps/chosen": -1487.2095947265625,
      "logps/rejected": -1470.6143798828125,
      "loss": 0.6919921040534973,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.009396498091518879,
      "rewards/margins": 0.005829436704516411,
      "rewards/rejected": -0.01522593479603529,
      "step": 100
    },
    {
      "epoch": 0.554945054945055,
      "grad_norm": 0.6213805675506592,
      "learning_rate": 1.2576687116564418e-06,
      "logits/chosen": -0.22846508026123047,
      "logits/rejected": -0.21416452527046204,
      "logps/chosen": -1408.79833984375,
      "logps/rejected": -1421.5936279296875,
      "loss": 0.6927986145019531,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.04689623415470123,
      "rewards/margins": -0.009503163397312164,
      "rewards/rejected": -0.03739307448267937,
      "step": 101
    },
    {
      "epoch": 0.5604395604395604,
      "grad_norm": 0.9802994132041931,
      "learning_rate": 1.2423312883435584e-06,
      "logits/chosen": -0.505018949508667,
      "logits/rejected": -0.5704793334007263,
      "logps/chosen": -1257.908447265625,
      "logps/rejected": -1249.787353515625,
      "loss": 0.6931822896003723,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.025434236973524094,
      "rewards/margins": -0.002385740401223302,
      "rewards/rejected": -0.02304849587380886,
      "step": 102
    },
    {
      "epoch": 0.5659340659340659,
      "grad_norm": 0.8993610739707947,
      "learning_rate": 1.226993865030675e-06,
      "logits/chosen": -0.41527795791625977,
      "logits/rejected": -0.29129812121391296,
      "logps/chosen": -1268.831787109375,
      "logps/rejected": -1143.24609375,
      "loss": 0.69354248046875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.008909551426768303,
      "rewards/margins": 0.0018367673037573695,
      "rewards/rejected": -0.010746316984295845,
      "step": 103
    },
    {
      "epoch": 0.5714285714285714,
      "grad_norm": 1.2592899799346924,
      "learning_rate": 1.2116564417177914e-06,
      "logits/chosen": -0.34576109051704407,
      "logits/rejected": -0.3690977990627289,
      "logps/chosen": -1669.214599609375,
      "logps/rejected": -1526.4136962890625,
      "loss": 0.6926941275596619,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0013259842526167631,
      "rewards/margins": 0.004025318659842014,
      "rewards/rejected": -0.005351303145289421,
      "step": 104
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 1.046607494354248,
      "learning_rate": 1.196319018404908e-06,
      "logits/chosen": -0.15831628441810608,
      "logits/rejected": -0.37302541732788086,
      "logps/chosen": -1464.148193359375,
      "logps/rejected": -1303.965576171875,
      "loss": 0.6931281685829163,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02865537628531456,
      "rewards/margins": -0.001078385510481894,
      "rewards/rejected": -0.027576986700296402,
      "step": 105
    },
    {
      "epoch": 0.5824175824175825,
      "grad_norm": 0.426277756690979,
      "learning_rate": 1.1809815950920247e-06,
      "logits/chosen": -0.5624100565910339,
      "logits/rejected": -0.5374657511711121,
      "logps/chosen": -1273.535888671875,
      "logps/rejected": -1335.310791015625,
      "loss": 0.6924761533737183,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0322088897228241,
      "rewards/margins": -0.0029357431922107935,
      "rewards/rejected": -0.029273148626089096,
      "step": 106
    },
    {
      "epoch": 0.5879120879120879,
      "grad_norm": 0.6133700013160706,
      "learning_rate": 1.165644171779141e-06,
      "logits/chosen": -0.20700356364250183,
      "logits/rejected": -0.22031204402446747,
      "logps/chosen": -1394.43017578125,
      "logps/rejected": -1308.563720703125,
      "loss": 0.6926645636558533,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04462941363453865,
      "rewards/margins": -0.008419161662459373,
      "rewards/rejected": -0.03621025010943413,
      "step": 107
    },
    {
      "epoch": 0.5934065934065934,
      "grad_norm": 0.6847041845321655,
      "learning_rate": 1.1503067484662577e-06,
      "logits/chosen": -0.14995786547660828,
      "logits/rejected": -0.14485234022140503,
      "logps/chosen": -1369.47998046875,
      "logps/rejected": -1361.496337890625,
      "loss": 0.6923136115074158,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.03646060824394226,
      "rewards/margins": 0.0012728970032185316,
      "rewards/rejected": -0.03773351013660431,
      "step": 108
    },
    {
      "epoch": 0.5989010989010989,
      "grad_norm": 0.6556602120399475,
      "learning_rate": 1.1349693251533743e-06,
      "logits/chosen": -0.17063872516155243,
      "logits/rejected": -0.25048452615737915,
      "logps/chosen": -1319.7943115234375,
      "logps/rejected": -1256.775634765625,
      "loss": 0.6931478977203369,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.025922460481524467,
      "rewards/margins": -0.00031948136165738106,
      "rewards/rejected": -0.025602979585528374,
      "step": 109
    },
    {
      "epoch": 0.6043956043956044,
      "grad_norm": 0.8980839848518372,
      "learning_rate": 1.119631901840491e-06,
      "logits/chosen": -0.3595108091831207,
      "logits/rejected": -0.34528863430023193,
      "logps/chosen": -1376.7236328125,
      "logps/rejected": -1430.8155517578125,
      "loss": 0.693233847618103,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022598477080464363,
      "rewards/margins": 0.000155882618855685,
      "rewards/rejected": -0.022754356265068054,
      "step": 110
    },
    {
      "epoch": 0.6098901098901099,
      "grad_norm": 1.0571579933166504,
      "learning_rate": 1.1042944785276075e-06,
      "logits/chosen": -0.37562742829322815,
      "logits/rejected": -0.4181535243988037,
      "logps/chosen": -1414.00732421875,
      "logps/rejected": -1326.544677734375,
      "loss": 0.6914387345314026,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.012199820019304752,
      "rewards/margins": 0.0014734650030732155,
      "rewards/rejected": -0.013673287816345692,
      "step": 111
    },
    {
      "epoch": 0.6153846153846154,
      "grad_norm": 1.7735028266906738,
      "learning_rate": 1.088957055214724e-06,
      "logits/chosen": -0.4293566346168518,
      "logits/rejected": -0.4803142547607422,
      "logps/chosen": -1466.178955078125,
      "logps/rejected": -1471.9337158203125,
      "loss": 0.6919817924499512,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010825528763234615,
      "rewards/margins": 0.010156643576920033,
      "rewards/rejected": -0.020982172340154648,
      "step": 112
    },
    {
      "epoch": 0.6208791208791209,
      "grad_norm": 1.0000537633895874,
      "learning_rate": 1.0736196319018406e-06,
      "logits/chosen": -0.5714986324310303,
      "logits/rejected": -0.5201742053031921,
      "logps/chosen": -1415.6246337890625,
      "logps/rejected": -1385.709716796875,
      "loss": 0.6931148767471313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03592289984226227,
      "rewards/margins": 0.010911607183516026,
      "rewards/rejected": 0.02501128986477852,
      "step": 113
    },
    {
      "epoch": 0.6263736263736264,
      "grad_norm": 0.8961329460144043,
      "learning_rate": 1.0582822085889572e-06,
      "logits/chosen": -0.45078492164611816,
      "logits/rejected": -0.4087766408920288,
      "logps/chosen": -1351.2510986328125,
      "logps/rejected": -1375.3194580078125,
      "loss": 0.6920664310455322,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.029718702659010887,
      "rewards/margins": 0.006335887126624584,
      "rewards/rejected": 0.02338281460106373,
      "step": 114
    },
    {
      "epoch": 0.6318681318681318,
      "grad_norm": 0.9695339202880859,
      "learning_rate": 1.0429447852760736e-06,
      "logits/chosen": -0.5634657144546509,
      "logits/rejected": -0.6761491894721985,
      "logps/chosen": -1333.12109375,
      "logps/rejected": -1317.586669921875,
      "loss": 0.6914012432098389,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.01340350229293108,
      "rewards/margins": 0.008562865667045116,
      "rewards/rejected": 0.0048406366258859634,
      "step": 115
    },
    {
      "epoch": 0.6373626373626373,
      "grad_norm": 0.8852068185806274,
      "learning_rate": 1.0276073619631902e-06,
      "logits/chosen": -0.2876928150653839,
      "logits/rejected": -0.397905558347702,
      "logps/chosen": -1615.9324951171875,
      "logps/rejected": -1533.5203857421875,
      "loss": 0.6928586959838867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00988774187862873,
      "rewards/margins": 0.0017955403309315443,
      "rewards/rejected": -0.011683283373713493,
      "step": 116
    },
    {
      "epoch": 0.6428571428571429,
      "grad_norm": 0.9956200122833252,
      "learning_rate": 1.0122699386503068e-06,
      "logits/chosen": -0.7941329479217529,
      "logits/rejected": -0.8391545414924622,
      "logps/chosen": -1485.06884765625,
      "logps/rejected": -1504.8533935546875,
      "loss": 0.6929861307144165,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.005250845104455948,
      "rewards/margins": -0.0009261607192456722,
      "rewards/rejected": -0.004324684850871563,
      "step": 117
    },
    {
      "epoch": 0.6483516483516484,
      "grad_norm": 0.8900634050369263,
      "learning_rate": 9.969325153374232e-07,
      "logits/chosen": -0.4403144121170044,
      "logits/rejected": -0.3117816746234894,
      "logps/chosen": -1124.2105712890625,
      "logps/rejected": -1162.626220703125,
      "loss": 0.6915737390518188,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.018135830760002136,
      "rewards/margins": 0.004251901991665363,
      "rewards/rejected": -0.022387731820344925,
      "step": 118
    },
    {
      "epoch": 0.6538461538461539,
      "grad_norm": 1.519210934638977,
      "learning_rate": 9.8159509202454e-07,
      "logits/chosen": -0.3049584925174713,
      "logits/rejected": -0.348927766084671,
      "logps/chosen": -1318.882568359375,
      "logps/rejected": -1179.972900390625,
      "loss": 0.6907892227172852,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.009297370910644531,
      "rewards/margins": 0.0032487385906279087,
      "rewards/rejected": -0.012546109966933727,
      "step": 119
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 1.4839621782302856,
      "learning_rate": 9.662576687116565e-07,
      "logits/chosen": -0.7371286153793335,
      "logits/rejected": -0.813791036605835,
      "logps/chosen": -1551.0048828125,
      "logps/rejected": -1457.481201171875,
      "loss": 0.692111074924469,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006462344899773598,
      "rewards/margins": 0.0030792569741606712,
      "rewards/rejected": -0.009541602805256844,
      "step": 120
    },
    {
      "epoch": 0.6648351648351648,
      "grad_norm": 1.420235514640808,
      "learning_rate": 9.509202453987732e-07,
      "logits/chosen": -0.3957596719264984,
      "logits/rejected": -0.44786208868026733,
      "logps/chosen": -1695.80517578125,
      "logps/rejected": -1655.417236328125,
      "loss": 0.6925563812255859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018932923674583435,
      "rewards/margins": 0.0009399798000231385,
      "rewards/rejected": -0.01987290196120739,
      "step": 121
    },
    {
      "epoch": 0.6703296703296703,
      "grad_norm": 1.4338594675064087,
      "learning_rate": 9.355828220858897e-07,
      "logits/chosen": -0.6365727186203003,
      "logits/rejected": -0.6079616546630859,
      "logps/chosen": -1449.3353271484375,
      "logps/rejected": -1306.079345703125,
      "loss": 0.6926460862159729,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01147129014134407,
      "rewards/margins": 0.005524273030459881,
      "rewards/rejected": -0.016995562240481377,
      "step": 122
    },
    {
      "epoch": 0.6758241758241759,
      "grad_norm": 1.0156636238098145,
      "learning_rate": 9.202453987730062e-07,
      "logits/chosen": -0.2907084822654724,
      "logits/rejected": -0.3623642325401306,
      "logps/chosen": -1366.35009765625,
      "logps/rejected": -1161.46875,
      "loss": 0.6904911398887634,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.017240682616829872,
      "rewards/margins": 0.014017186127603054,
      "rewards/rejected": 0.0032234953250736,
      "step": 123
    },
    {
      "epoch": 0.6813186813186813,
      "grad_norm": 1.0583997964859009,
      "learning_rate": 9.049079754601228e-07,
      "logits/chosen": -0.7642356157302856,
      "logits/rejected": -0.7943760752677917,
      "logps/chosen": -1484.6651611328125,
      "logps/rejected": -1499.005126953125,
      "loss": 0.6919360160827637,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.029063954949378967,
      "rewards/margins": 0.013380720280110836,
      "rewards/rejected": 0.015683233737945557,
      "step": 124
    },
    {
      "epoch": 0.6868131868131868,
      "grad_norm": 1.5796548128128052,
      "learning_rate": 8.895705521472393e-07,
      "logits/chosen": -0.727140486240387,
      "logits/rejected": -0.7532109022140503,
      "logps/chosen": -1440.1063232421875,
      "logps/rejected": -1624.1826171875,
      "loss": 0.6913582682609558,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -2.9935792554169893e-05,
      "rewards/margins": 0.007877510972321033,
      "rewards/rejected": -0.007907447405159473,
      "step": 125
    },
    {
      "epoch": 0.6923076923076923,
      "grad_norm": 0.86751389503479,
      "learning_rate": 8.742331288343559e-07,
      "logits/chosen": -0.389473021030426,
      "logits/rejected": -0.41523274779319763,
      "logps/chosen": -1585.4542236328125,
      "logps/rejected": -1302.420166015625,
      "loss": 0.6916378736495972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022927483543753624,
      "rewards/margins": -0.0011927932500839233,
      "rewards/rejected": -0.02173468843102455,
      "step": 126
    },
    {
      "epoch": 0.6978021978021978,
      "grad_norm": 0.7973591089248657,
      "learning_rate": 8.588957055214725e-07,
      "logits/chosen": -0.3697919547557831,
      "logits/rejected": -0.32609784603118896,
      "logps/chosen": -1386.21240234375,
      "logps/rejected": -1433.26171875,
      "loss": 0.6924079656600952,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.024928748607635498,
      "rewards/margins": 0.0017317293677479029,
      "rewards/rejected": -0.026660479605197906,
      "step": 127
    },
    {
      "epoch": 0.7032967032967034,
      "grad_norm": 0.8267586827278137,
      "learning_rate": 8.43558282208589e-07,
      "logits/chosen": -0.39392054080963135,
      "logits/rejected": -0.4094931483268738,
      "logps/chosen": -1449.90380859375,
      "logps/rejected": -1335.430908203125,
      "loss": 0.6908578872680664,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.037380997091531754,
      "rewards/margins": 0.0011315422598272562,
      "rewards/rejected": -0.038512542843818665,
      "step": 128
    },
    {
      "epoch": 0.7087912087912088,
      "grad_norm": 1.491594910621643,
      "learning_rate": 8.282208588957055e-07,
      "logits/chosen": -0.3594258725643158,
      "logits/rejected": -0.37132909893989563,
      "logps/chosen": -1539.3756103515625,
      "logps/rejected": -1565.0859375,
      "loss": 0.6924813985824585,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.011002148501574993,
      "rewards/margins": 0.0007907864637672901,
      "rewards/rejected": -0.011792936362326145,
      "step": 129
    },
    {
      "epoch": 0.7142857142857143,
      "grad_norm": 0.961751401424408,
      "learning_rate": 8.128834355828222e-07,
      "logits/chosen": -0.5297249555587769,
      "logits/rejected": -0.45530810952186584,
      "logps/chosen": -1412.378173828125,
      "logps/rejected": -1533.008544921875,
      "loss": 0.6914486885070801,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0015569772804155946,
      "rewards/margins": 0.0011243524495512247,
      "rewards/rejected": 0.0004326249472796917,
      "step": 130
    },
    {
      "epoch": 0.7197802197802198,
      "grad_norm": 1.0202986001968384,
      "learning_rate": 7.975460122699387e-07,
      "logits/chosen": -0.2956511974334717,
      "logits/rejected": -0.3015546500682831,
      "logps/chosen": -1375.919921875,
      "logps/rejected": -1224.4910888671875,
      "loss": 0.6927995681762695,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.004763889592140913,
      "rewards/margins": -0.001240363228134811,
      "rewards/rejected": -0.003523525781929493,
      "step": 131
    },
    {
      "epoch": 0.7252747252747253,
      "grad_norm": 1.1129013299942017,
      "learning_rate": 7.822085889570554e-07,
      "logits/chosen": -0.33333620429039,
      "logits/rejected": -0.3021361827850342,
      "logps/chosen": -1697.37109375,
      "logps/rejected": -1630.856689453125,
      "loss": 0.6946622729301453,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0037074373103678226,
      "rewards/margins": -0.003561133984476328,
      "rewards/rejected": -0.00014630379155278206,
      "step": 132
    },
    {
      "epoch": 0.7307692307692307,
      "grad_norm": 0.7518520355224609,
      "learning_rate": 7.668711656441719e-07,
      "logits/chosen": -0.8502654433250427,
      "logits/rejected": -0.8670777678489685,
      "logps/chosen": -1536.0880126953125,
      "logps/rejected": -1519.32666015625,
      "loss": 0.6922348737716675,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.001666030497290194,
      "rewards/margins": 0.007148093543946743,
      "rewards/rejected": -0.005482063163071871,
      "step": 133
    },
    {
      "epoch": 0.7362637362637363,
      "grad_norm": 1.1008754968643188,
      "learning_rate": 7.515337423312884e-07,
      "logits/chosen": -0.4814603924751282,
      "logits/rejected": -0.43353089690208435,
      "logps/chosen": -1490.4459228515625,
      "logps/rejected": -1474.66162109375,
      "loss": 0.6931823492050171,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.004679413046687841,
      "rewards/margins": 0.005811995826661587,
      "rewards/rejected": -0.010491408407688141,
      "step": 134
    },
    {
      "epoch": 0.7417582417582418,
      "grad_norm": 1.0830036401748657,
      "learning_rate": 7.36196319018405e-07,
      "logits/chosen": -0.5737816691398621,
      "logits/rejected": -0.5043767690658569,
      "logps/chosen": -1319.7476806640625,
      "logps/rejected": -1288.7066650390625,
      "loss": 0.6926530599594116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0063733747228980064,
      "rewards/margins": -0.0013508339179679751,
      "rewards/rejected": 0.007724207825958729,
      "step": 135
    },
    {
      "epoch": 0.7472527472527473,
      "grad_norm": 1.152644157409668,
      "learning_rate": 7.208588957055215e-07,
      "logits/chosen": -0.4295804798603058,
      "logits/rejected": -0.39793089032173157,
      "logps/chosen": -1226.8475341796875,
      "logps/rejected": -1343.85107421875,
      "loss": 0.6914076805114746,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03230779990553856,
      "rewards/margins": 0.011389164254069328,
      "rewards/rejected": 0.02091863378882408,
      "step": 136
    },
    {
      "epoch": 0.7527472527472527,
      "grad_norm": 0.779612123966217,
      "learning_rate": 7.05521472392638e-07,
      "logits/chosen": -0.689027726650238,
      "logits/rejected": -0.6783583164215088,
      "logps/chosen": -1387.3411865234375,
      "logps/rejected": -1337.02685546875,
      "loss": 0.6930306553840637,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.029504049569368362,
      "rewards/margins": 0.007792205549776554,
      "rewards/rejected": 0.021711843088269234,
      "step": 137
    },
    {
      "epoch": 0.7582417582417582,
      "grad_norm": 0.6696953773498535,
      "learning_rate": 6.901840490797547e-07,
      "logits/chosen": -0.4422488212585449,
      "logits/rejected": -0.40494176745414734,
      "logps/chosen": -1538.20556640625,
      "logps/rejected": -1443.9151611328125,
      "loss": 0.6923375725746155,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.03989330306649208,
      "rewards/margins": 0.011228306218981743,
      "rewards/rejected": 0.02866499498486519,
      "step": 138
    },
    {
      "epoch": 0.7637362637362637,
      "grad_norm": 1.1540745496749878,
      "learning_rate": 6.748466257668713e-07,
      "logits/chosen": -0.40560609102249146,
      "logits/rejected": -0.41341060400009155,
      "logps/chosen": -1528.6448974609375,
      "logps/rejected": -1515.0938720703125,
      "loss": 0.6929686069488525,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.017328977584838867,
      "rewards/margins": 0.005272225942462683,
      "rewards/rejected": 0.012056750245392323,
      "step": 139
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 0.9930852651596069,
      "learning_rate": 6.595092024539878e-07,
      "logits/chosen": -0.07321417331695557,
      "logits/rejected": -0.08877704292535782,
      "logps/chosen": -1580.5616455078125,
      "logps/rejected": -1326.6715087890625,
      "loss": 0.6931515336036682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02595691569149494,
      "rewards/margins": 0.0032394053414463997,
      "rewards/rejected": 0.022717511281371117,
      "step": 140
    },
    {
      "epoch": 0.7747252747252747,
      "grad_norm": 1.2813407182693481,
      "learning_rate": 6.441717791411044e-07,
      "logits/chosen": -0.3404533267021179,
      "logits/rejected": -0.39101195335388184,
      "logps/chosen": -1059.5224609375,
      "logps/rejected": -1201.876953125,
      "loss": 0.6932750940322876,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.011713255196809769,
      "rewards/margins": 0.004767118953168392,
      "rewards/rejected": 0.0069461362436413765,
      "step": 141
    },
    {
      "epoch": 0.7802197802197802,
      "grad_norm": 0.8233911991119385,
      "learning_rate": 6.288343558282209e-07,
      "logits/chosen": -0.4058606028556824,
      "logits/rejected": -0.2887468934059143,
      "logps/chosen": -993.4666137695312,
      "logps/rejected": -1229.0377197265625,
      "loss": 0.6933331489562988,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.021297745406627655,
      "rewards/margins": 0.007736450061202049,
      "rewards/rejected": 0.01356129627674818,
      "step": 142
    },
    {
      "epoch": 0.7857142857142857,
      "grad_norm": 0.9581381678581238,
      "learning_rate": 6.134969325153375e-07,
      "logits/chosen": -0.08297372609376907,
      "logits/rejected": -0.16077230870723724,
      "logps/chosen": -1555.29931640625,
      "logps/rejected": -1433.138916015625,
      "loss": 0.6934653520584106,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03898211941123009,
      "rewards/margins": 0.010908895172178745,
      "rewards/rejected": 0.028073228895664215,
      "step": 143
    },
    {
      "epoch": 0.7912087912087912,
      "grad_norm": 1.3117345571517944,
      "learning_rate": 5.98159509202454e-07,
      "logits/chosen": -0.2987571656703949,
      "logits/rejected": -0.33382365107536316,
      "logps/chosen": -1226.4559326171875,
      "logps/rejected": -1198.4481201171875,
      "loss": 0.6925820112228394,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0050286054611206055,
      "rewards/margins": 0.005197577178478241,
      "rewards/rejected": -0.0001689721830189228,
      "step": 144
    },
    {
      "epoch": 0.7967032967032966,
      "grad_norm": 1.4236594438552856,
      "learning_rate": 5.828220858895705e-07,
      "logits/chosen": -0.49541574716567993,
      "logits/rejected": -0.4821420907974243,
      "logps/chosen": -1369.834716796875,
      "logps/rejected": -1285.265625,
      "loss": 0.6933659315109253,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.005504836328327656,
      "rewards/margins": 0.00019674794748425484,
      "rewards/rejected": -0.005701584275811911,
      "step": 145
    },
    {
      "epoch": 0.8021978021978022,
      "grad_norm": 1.0803322792053223,
      "learning_rate": 5.674846625766872e-07,
      "logits/chosen": -0.41038578748703003,
      "logits/rejected": -0.22637057304382324,
      "logps/chosen": -1731.6552734375,
      "logps/rejected": -1467.64599609375,
      "loss": 0.6921501755714417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021399000659585,
      "rewards/margins": 0.0027879904955625534,
      "rewards/rejected": -0.024186991155147552,
      "step": 146
    },
    {
      "epoch": 0.8076923076923077,
      "grad_norm": 1.168318748474121,
      "learning_rate": 5.521472392638038e-07,
      "logits/chosen": -0.034145765006542206,
      "logits/rejected": -0.19365057349205017,
      "logps/chosen": -1311.8702392578125,
      "logps/rejected": -1272.918701171875,
      "loss": 0.6925601959228516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009253215976059437,
      "rewards/margins": 0.007820549421012402,
      "rewards/rejected": 0.0014326669042930007,
      "step": 147
    },
    {
      "epoch": 0.8131868131868132,
      "grad_norm": 1.2571437358856201,
      "learning_rate": 5.368098159509203e-07,
      "logits/chosen": -0.6215896010398865,
      "logits/rejected": -0.6399307250976562,
      "logps/chosen": -1441.660888671875,
      "logps/rejected": -1511.095947265625,
      "loss": 0.6926032304763794,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008588919416069984,
      "rewards/margins": 0.0011720561888068914,
      "rewards/rejected": 0.007416863460093737,
      "step": 148
    },
    {
      "epoch": 0.8186813186813187,
      "grad_norm": 1.3910409212112427,
      "learning_rate": 5.214723926380368e-07,
      "logits/chosen": -0.7271393537521362,
      "logits/rejected": -0.761420726776123,
      "logps/chosen": -1419.0078125,
      "logps/rejected": -1475.87353515625,
      "loss": 0.6915566325187683,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.022086886689066887,
      "rewards/margins": 0.0090789794921875,
      "rewards/rejected": 0.013007907196879387,
      "step": 149
    },
    {
      "epoch": 0.8241758241758241,
      "grad_norm": 1.4809271097183228,
      "learning_rate": 5.061349693251534e-07,
      "logits/chosen": -0.7618609666824341,
      "logits/rejected": -0.7325096130371094,
      "logps/chosen": -1367.3497314453125,
      "logps/rejected": -1415.16943359375,
      "loss": 0.6933605074882507,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.014094429090619087,
      "rewards/margins": 0.0031739422120153904,
      "rewards/rejected": 0.010920487344264984,
      "step": 150
    },
    {
      "epoch": 0.8296703296703297,
      "grad_norm": 1.8276501893997192,
      "learning_rate": 4.9079754601227e-07,
      "logits/chosen": -0.8397077322006226,
      "logits/rejected": -0.7813823223114014,
      "logps/chosen": -1705.48486328125,
      "logps/rejected": -1569.8170166015625,
      "loss": 0.6919160485267639,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012699166312813759,
      "rewards/margins": 0.012700291350483894,
      "rewards/rejected": -1.1255033314228058e-06,
      "step": 151
    },
    {
      "epoch": 0.8351648351648352,
      "grad_norm": 1.1467257738113403,
      "learning_rate": 4.754601226993866e-07,
      "logits/chosen": -0.6820580959320068,
      "logits/rejected": -0.8008350729942322,
      "logps/chosen": -1449.619140625,
      "logps/rejected": -1359.572021484375,
      "loss": 0.6915912628173828,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.024284247308969498,
      "rewards/margins": 0.009465246461331844,
      "rewards/rejected": 0.014819002710282803,
      "step": 152
    },
    {
      "epoch": 0.8406593406593407,
      "grad_norm": 1.1940540075302124,
      "learning_rate": 4.601226993865031e-07,
      "logits/chosen": -0.4987485110759735,
      "logits/rejected": -0.4857948124408722,
      "logps/chosen": -1300.3497314453125,
      "logps/rejected": -1322.68017578125,
      "loss": 0.6913399696350098,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018459968268871307,
      "rewards/margins": 0.00594030786305666,
      "rewards/rejected": 0.012519659474492073,
      "step": 153
    },
    {
      "epoch": 0.8461538461538461,
      "grad_norm": 1.2268966436386108,
      "learning_rate": 4.447852760736197e-07,
      "logits/chosen": -0.6148173213005066,
      "logits/rejected": -0.5686666965484619,
      "logps/chosen": -1196.4783935546875,
      "logps/rejected": -1201.1766357421875,
      "loss": 0.6927800178527832,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009037494659423828,
      "rewards/margins": 0.0022115707397460938,
      "rewards/rejected": -0.011249064467847347,
      "step": 154
    },
    {
      "epoch": 0.8516483516483516,
      "grad_norm": 1.4325149059295654,
      "learning_rate": 4.2944785276073624e-07,
      "logits/chosen": -0.09442466497421265,
      "logits/rejected": 0.06447166204452515,
      "logps/chosen": -1344.0181884765625,
      "logps/rejected": -1390.199462890625,
      "loss": 0.6922627091407776,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014286688528954983,
      "rewards/margins": -0.0005080983391962945,
      "rewards/rejected": -0.013778590597212315,
      "step": 155
    },
    {
      "epoch": 0.8571428571428571,
      "grad_norm": 1.6210157871246338,
      "learning_rate": 4.1411042944785275e-07,
      "logits/chosen": -0.5195028781890869,
      "logits/rejected": -0.581871509552002,
      "logps/chosen": -1449.6171875,
      "logps/rejected": -1445.11962890625,
      "loss": 0.6920541524887085,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.007332286797463894,
      "rewards/margins": 0.004986848682165146,
      "rewards/rejected": -0.012319136410951614,
      "step": 156
    },
    {
      "epoch": 0.8626373626373627,
      "grad_norm": 1.2568283081054688,
      "learning_rate": 3.9877300613496937e-07,
      "logits/chosen": -0.39413487911224365,
      "logits/rejected": -0.3989291191101074,
      "logps/chosen": -1560.168212890625,
      "logps/rejected": -1546.84326171875,
      "loss": 0.6931201219558716,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03144290670752525,
      "rewards/margins": -0.005113277584314346,
      "rewards/rejected": -0.026329632848501205,
      "step": 157
    },
    {
      "epoch": 0.8681318681318682,
      "grad_norm": 0.5363301634788513,
      "learning_rate": 3.8343558282208593e-07,
      "logits/chosen": -0.32399898767471313,
      "logits/rejected": -0.2612525522708893,
      "logps/chosen": -1289.2220458984375,
      "logps/rejected": -1368.276123046875,
      "loss": 0.6930078268051147,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020566385239362717,
      "rewards/margins": 0.007330484688282013,
      "rewards/rejected": -0.02789686992764473,
      "step": 158
    },
    {
      "epoch": 0.8736263736263736,
      "grad_norm": 1.1271432638168335,
      "learning_rate": 3.680981595092025e-07,
      "logits/chosen": -0.8580625057220459,
      "logits/rejected": -0.8349483013153076,
      "logps/chosen": -1564.2672119140625,
      "logps/rejected": -1513.3536376953125,
      "loss": 0.6922256946563721,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021702442318201065,
      "rewards/margins": 0.0017409119755029678,
      "rewards/rejected": -0.023443354293704033,
      "step": 159
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 1.249580979347229,
      "learning_rate": 3.52760736196319e-07,
      "logits/chosen": -0.24156060814857483,
      "logits/rejected": -0.3282901644706726,
      "logps/chosen": -1468.79931640625,
      "logps/rejected": -1360.5396728515625,
      "loss": 0.6932394504547119,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016846224665641785,
      "rewards/margins": -0.0005967399920336902,
      "rewards/rejected": -0.016249485313892365,
      "step": 160
    },
    {
      "epoch": 0.8846153846153846,
      "grad_norm": 1.7817763090133667,
      "learning_rate": 3.3742331288343563e-07,
      "logits/chosen": -0.4720512926578522,
      "logits/rejected": -0.47025078535079956,
      "logps/chosen": -1514.5360107421875,
      "logps/rejected": -1491.627197265625,
      "loss": 0.6926375031471252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020506229251623154,
      "rewards/margins": 0.008882999420166016,
      "rewards/rejected": -0.02938922867178917,
      "step": 161
    },
    {
      "epoch": 0.8901098901098901,
      "grad_norm": 0.959372878074646,
      "learning_rate": 3.220858895705522e-07,
      "logits/chosen": -0.35722965002059937,
      "logits/rejected": -0.41712379455566406,
      "logps/chosen": -1411.179931640625,
      "logps/rejected": -1289.341796875,
      "loss": 0.6943350434303284,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.024521712213754654,
      "rewards/margins": -0.0049790311604738235,
      "rewards/rejected": -0.01954268291592598,
      "step": 162
    },
    {
      "epoch": 0.8956043956043956,
      "grad_norm": 1.2075046300888062,
      "learning_rate": 3.0674846625766876e-07,
      "logits/chosen": -0.8041161298751831,
      "logits/rejected": -0.8109002113342285,
      "logps/chosen": -1505.929931640625,
      "logps/rejected": -1477.200927734375,
      "loss": 0.6926769614219666,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.028787925839424133,
      "rewards/margins": -0.00552023109048605,
      "rewards/rejected": -0.023267695680260658,
      "step": 163
    },
    {
      "epoch": 0.9010989010989011,
      "grad_norm": 1.387386441230774,
      "learning_rate": 2.9141104294478527e-07,
      "logits/chosen": -0.44806283712387085,
      "logits/rejected": -0.61948162317276,
      "logps/chosen": -1527.060302734375,
      "logps/rejected": -1494.800537109375,
      "loss": 0.692619264125824,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011695308610796928,
      "rewards/margins": -0.0015909149078652263,
      "rewards/rejected": -0.01010439358651638,
      "step": 164
    },
    {
      "epoch": 0.9065934065934066,
      "grad_norm": 1.484518051147461,
      "learning_rate": 2.760736196319019e-07,
      "logits/chosen": -0.622553288936615,
      "logits/rejected": -0.7360720634460449,
      "logps/chosen": -1395.18798828125,
      "logps/rejected": -1419.4432373046875,
      "loss": 0.6941399574279785,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008678913116455078,
      "rewards/margins": 0.001724362256936729,
      "rewards/rejected": -0.010403275489807129,
      "step": 165
    },
    {
      "epoch": 0.9120879120879121,
      "grad_norm": 0.7226335406303406,
      "learning_rate": 2.607361963190184e-07,
      "logits/chosen": -0.43059980869293213,
      "logits/rejected": -0.5786530375480652,
      "logps/chosen": -1415.7808837890625,
      "logps/rejected": -1407.481201171875,
      "loss": 0.6924831867218018,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02000749483704567,
      "rewards/margins": -0.002660846570506692,
      "rewards/rejected": -0.01734665036201477,
      "step": 166
    },
    {
      "epoch": 0.9175824175824175,
      "grad_norm": 1.1836820840835571,
      "learning_rate": 2.45398773006135e-07,
      "logits/chosen": -0.3440017104148865,
      "logits/rejected": -0.3202272355556488,
      "logps/chosen": -1425.4095458984375,
      "logps/rejected": -1382.1885986328125,
      "loss": 0.6946048140525818,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.013712071813642979,
      "rewards/margins": -0.006773996166884899,
      "rewards/rejected": -0.0069380756467580795,
      "step": 167
    },
    {
      "epoch": 0.9230769230769231,
      "grad_norm": 2.162140369415283,
      "learning_rate": 2.3006134969325155e-07,
      "logits/chosen": -0.6371763944625854,
      "logits/rejected": -0.7157647609710693,
      "logps/chosen": -1485.0994873046875,
      "logps/rejected": -1396.60693359375,
      "loss": 0.6927782297134399,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.014371508732438087,
      "rewards/margins": 0.0014920367393642664,
      "rewards/rejected": -0.01586354523897171,
      "step": 168
    },
    {
      "epoch": 0.9285714285714286,
      "grad_norm": 1.5967764854431152,
      "learning_rate": 2.1472392638036812e-07,
      "logits/chosen": -0.5323659777641296,
      "logits/rejected": -0.5269442200660706,
      "logps/chosen": -1347.4677734375,
      "logps/rejected": -1475.41064453125,
      "loss": 0.6943572163581848,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0019814111292362213,
      "rewards/margins": -0.00141085684299469,
      "rewards/rejected": -0.0005705545772798359,
      "step": 169
    },
    {
      "epoch": 0.9340659340659341,
      "grad_norm": 1.2801460027694702,
      "learning_rate": 1.9938650306748468e-07,
      "logits/chosen": -0.7320770621299744,
      "logits/rejected": -0.7078484892845154,
      "logps/chosen": -1049.31298828125,
      "logps/rejected": -1216.61083984375,
      "loss": 0.6933168172836304,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009347095154225826,
      "rewards/margins": -0.0006960107129998505,
      "rewards/rejected": -0.008651085197925568,
      "step": 170
    },
    {
      "epoch": 0.9395604395604396,
      "grad_norm": 1.4801154136657715,
      "learning_rate": 1.8404907975460125e-07,
      "logits/chosen": -0.8624351620674133,
      "logits/rejected": -0.6278290748596191,
      "logps/chosen": -1554.202392578125,
      "logps/rejected": -1768.3101806640625,
      "loss": 0.6939010620117188,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.011780891567468643,
      "rewards/margins": -0.0007463262882083654,
      "rewards/rejected": -0.011034565046429634,
      "step": 171
    },
    {
      "epoch": 0.945054945054945,
      "grad_norm": 1.2581945657730103,
      "learning_rate": 1.6871165644171781e-07,
      "logits/chosen": -0.8042242527008057,
      "logits/rejected": -0.8509213328361511,
      "logps/chosen": -1742.9061279296875,
      "logps/rejected": -1663.30078125,
      "loss": 0.6920545101165771,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.009515208192169666,
      "rewards/margins": 0.006152589339762926,
      "rewards/rejected": -0.01566779799759388,
      "step": 172
    },
    {
      "epoch": 0.9505494505494505,
      "grad_norm": 1.6917072534561157,
      "learning_rate": 1.5337423312883438e-07,
      "logits/chosen": -0.677159309387207,
      "logits/rejected": -0.708314836025238,
      "logps/chosen": -1458.487548828125,
      "logps/rejected": -1540.4312744140625,
      "loss": 0.6939187049865723,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008841114118695259,
      "rewards/margins": -0.0041299439035356045,
      "rewards/rejected": -0.004711170680820942,
      "step": 173
    },
    {
      "epoch": 0.9560439560439561,
      "grad_norm": 1.562403917312622,
      "learning_rate": 1.3803680981595094e-07,
      "logits/chosen": -0.8315936923027039,
      "logits/rejected": -0.8156816959381104,
      "logps/chosen": -1405.054931640625,
      "logps/rejected": -1378.0836181640625,
      "loss": 0.6930129528045654,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008151168935000896,
      "rewards/margins": 0.0004372410476207733,
      "rewards/rejected": -0.008588409051299095,
      "step": 174
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 1.8198566436767578,
      "learning_rate": 1.226993865030675e-07,
      "logits/chosen": -0.35468000173568726,
      "logits/rejected": -0.46720781922340393,
      "logps/chosen": -1518.917236328125,
      "logps/rejected": -1435.27783203125,
      "loss": 0.6903528571128845,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005114269442856312,
      "rewards/margins": 0.011224641464650631,
      "rewards/rejected": -0.006110372021794319,
      "step": 175
    },
    {
      "epoch": 0.967032967032967,
      "grad_norm": 2.1096837520599365,
      "learning_rate": 1.0736196319018406e-07,
      "logits/chosen": -0.6721988320350647,
      "logits/rejected": -0.6404861211776733,
      "logps/chosen": -1544.8468017578125,
      "logps/rejected": -1587.005615234375,
      "loss": 0.6921041011810303,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.010166358202695847,
      "rewards/margins": 0.003511047223582864,
      "rewards/rejected": -0.013677406124770641,
      "step": 176
    },
    {
      "epoch": 0.9725274725274725,
      "grad_norm": 1.5950981378555298,
      "learning_rate": 9.202453987730062e-08,
      "logits/chosen": -0.867381751537323,
      "logits/rejected": -0.753036379814148,
      "logps/chosen": -1559.811767578125,
      "logps/rejected": -1612.46826171875,
      "loss": 0.6940016150474548,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014099950902163982,
      "rewards/margins": -0.0024051289074122906,
      "rewards/rejected": -0.011694822460412979,
      "step": 177
    },
    {
      "epoch": 0.978021978021978,
      "grad_norm": 1.7663509845733643,
      "learning_rate": 7.668711656441719e-08,
      "logits/chosen": -0.49028313159942627,
      "logits/rejected": -0.6348991990089417,
      "logps/chosen": -1656.97216796875,
      "logps/rejected": -1574.3087158203125,
      "loss": 0.6930438876152039,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0007067109690979123,
      "rewards/margins": 0.002308547031134367,
      "rewards/rejected": -0.0030152583494782448,
      "step": 178
    },
    {
      "epoch": 0.9835164835164835,
      "grad_norm": 2.180283308029175,
      "learning_rate": 6.134969325153375e-08,
      "logits/chosen": -0.3077111840248108,
      "logits/rejected": -0.3495293855667114,
      "logps/chosen": -1459.8404541015625,
      "logps/rejected": -1447.5089111328125,
      "loss": 0.6924480199813843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008424444124102592,
      "rewards/margins": 0.0035952467005699873,
      "rewards/rejected": -0.012019690126180649,
      "step": 179
    },
    {
      "epoch": 0.989010989010989,
      "grad_norm": 1.005254864692688,
      "learning_rate": 4.601226993865031e-08,
      "logits/chosen": -0.6914213299751282,
      "logits/rejected": -0.6896047592163086,
      "logps/chosen": -1281.80419921875,
      "logps/rejected": -1390.142578125,
      "loss": 0.692180871963501,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00413126964122057,
      "rewards/margins": 0.0011504649883136153,
      "rewards/rejected": -0.0052817342802882195,
      "step": 180
    },
    {
      "epoch": 0.9945054945054945,
      "grad_norm": 1.4081807136535645,
      "learning_rate": 3.067484662576688e-08,
      "logits/chosen": -0.8277162313461304,
      "logits/rejected": -0.8068577647209167,
      "logps/chosen": -1470.6419677734375,
      "logps/rejected": -1466.707763671875,
      "loss": 0.6934786438941956,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0031227758154273033,
      "rewards/margins": -0.004466504789888859,
      "rewards/rejected": 0.0013437292072921991,
      "step": 181
    },
    {
      "epoch": 1.0,
      "grad_norm": 1.1653679609298706,
      "learning_rate": 1.533742331288344e-08,
      "logits/chosen": -0.3629406690597534,
      "logits/rejected": -0.33546456694602966,
      "logps/chosen": -1682.532470703125,
      "logps/rejected": -1659.32470703125,
      "loss": 0.6933432221412659,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00286815594881773,
      "rewards/margins": 0.000943202874623239,
      "rewards/rejected": 0.0019249536562711,
      "step": 182
    }
  ],
  "logging_steps": 1,
  "max_steps": 182,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}