{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9494205566987977,
  "eval_steps": 500,
  "global_step": 9000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010830715910321671,
      "grad_norm": 16.5,
      "learning_rate": 8.36940836940837e-08,
      "logits/chosen": 0.17667897045612335,
      "logits/rejected": 0.18992166221141815,
      "logps/chosen": -228.5318603515625,
      "logps/rejected": -219.2080535888672,
      "loss": 0.6954,
      "rewards/accuracies": 0.25833332538604736,
      "rewards/chosen": -0.0026082005351781845,
      "rewards/margins": -0.004379781428724527,
      "rewards/rejected": 0.0017715801950544119,
      "step": 5
    },
    {
      "epoch": 0.0021661431820643342,
      "grad_norm": 16.375,
      "learning_rate": 1.8831168831168833e-07,
      "logits/chosen": 0.20830440521240234,
      "logits/rejected": 0.22600331902503967,
      "logps/chosen": -217.9087371826172,
      "logps/rejected": -210.5413055419922,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0019045192748308182,
      "rewards/margins": -0.000423138786572963,
      "rewards/rejected": 0.0023276584688574076,
      "step": 10
    },
    {
      "epoch": 0.0032492147730965016,
      "grad_norm": 15.875,
      "learning_rate": 2.9292929292929296e-07,
      "logits/chosen": 0.14393453299999237,
      "logits/rejected": 0.19447365403175354,
      "logps/chosen": -219.7711944580078,
      "logps/rejected": -218.0955047607422,
      "loss": 0.6918,
      "rewards/accuracies": 0.5583333969116211,
      "rewards/chosen": 0.003835944924503565,
      "rewards/margins": 0.003081849543377757,
      "rewards/rejected": 0.0007540957303717732,
      "step": 15
    },
    {
      "epoch": 0.0043322863641286685,
      "grad_norm": 17.125,
      "learning_rate": 3.9754689754689755e-07,
      "logits/chosen": 0.2310166358947754,
      "logits/rejected": 0.23797667026519775,
      "logps/chosen": -222.1182098388672,
      "logps/rejected": -235.3577423095703,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0020106316078454256,
      "rewards/margins": -0.0006976284203119576,
      "rewards/rejected": -0.0013130026636645198,
      "step": 20
    },
    {
      "epoch": 0.005415357955160836,
      "grad_norm": 18.125,
      "learning_rate": 5.021645021645022e-07,
      "logits/chosen": 0.17390426993370056,
      "logits/rejected": 0.19121184945106506,
      "logps/chosen": -218.84103393554688,
      "logps/rejected": -214.4557647705078,
      "loss": 0.6936,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": -0.0018068982753902674,
      "rewards/margins": -0.0006516234716400504,
      "rewards/rejected": -0.001155274803750217,
      "step": 25
    },
    {
      "epoch": 0.006498429546193003,
      "grad_norm": 15.0,
      "learning_rate": 6.067821067821068e-07,
      "logits/chosen": 0.2221207618713379,
      "logits/rejected": 0.2510652542114258,
      "logps/chosen": -226.53683471679688,
      "logps/rejected": -233.11550903320312,
      "loss": 0.6932,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": -0.0023401163052767515,
      "rewards/margins": 0.00026242900639772415,
      "rewards/rejected": -0.0026025453116744757,
      "step": 30
    },
    {
      "epoch": 0.0075815011372251705,
      "grad_norm": 16.5,
      "learning_rate": 7.113997113997114e-07,
      "logits/chosen": 0.1823590248823166,
      "logits/rejected": 0.215450257062912,
      "logps/chosen": -210.1569366455078,
      "logps/rejected": -224.08871459960938,
      "loss": 0.6914,
      "rewards/accuracies": 0.5416667461395264,
      "rewards/chosen": 0.0019681265112012625,
      "rewards/margins": 0.0038307090289890766,
      "rewards/rejected": -0.001862583332695067,
      "step": 35
    },
    {
      "epoch": 0.008664572728257337,
      "grad_norm": 14.75,
      "learning_rate": 8.16017316017316e-07,
      "logits/chosen": 0.18444928526878357,
      "logits/rejected": 0.21638306975364685,
      "logps/chosen": -219.398681640625,
      "logps/rejected": -216.0818634033203,
      "loss": 0.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004425227176398039,
      "rewards/margins": 0.004670655354857445,
      "rewards/rejected": -0.000245428120251745,
      "step": 40
    },
    {
      "epoch": 0.009747644319289504,
      "grad_norm": 15.5625,
      "learning_rate": 9.206349206349206e-07,
      "logits/chosen": 0.1671595573425293,
      "logits/rejected": 0.1765199601650238,
      "logps/chosen": -226.755859375,
      "logps/rejected": -218.59622192382812,
      "loss": 0.6931,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": -0.0004696275573223829,
      "rewards/margins": 0.000543489120900631,
      "rewards/rejected": -0.0010131166782230139,
      "step": 45
    },
    {
      "epoch": 0.010830715910321672,
      "grad_norm": 17.0,
      "learning_rate": 1.0252525252525253e-06,
      "logits/chosen": 0.21027633547782898,
      "logits/rejected": 0.2441544085741043,
      "logps/chosen": -217.8820343017578,
      "logps/rejected": -220.40103149414062,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033965972252190113,
      "rewards/margins": -0.0007302761077880859,
      "rewards/rejected": 0.00412687286734581,
      "step": 50
    },
    {
      "epoch": 0.011913787501353839,
      "grad_norm": 14.5,
      "learning_rate": 1.1298701298701298e-06,
      "logits/chosen": 0.1656002700328827,
      "logits/rejected": 0.2080727368593216,
      "logps/chosen": -215.3459014892578,
      "logps/rejected": -210.1002960205078,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0032512664329260588,
      "rewards/margins": 0.0022753654047846794,
      "rewards/rejected": 0.0009759014355950058,
      "step": 55
    },
    {
      "epoch": 0.012996859092386006,
      "grad_norm": 17.125,
      "learning_rate": 1.2344877344877345e-06,
      "logits/chosen": 0.2250998318195343,
      "logits/rejected": 0.2201727330684662,
      "logps/chosen": -234.3174591064453,
      "logps/rejected": -215.80093383789062,
      "loss": 0.6917,
      "rewards/accuracies": 0.4750000536441803,
      "rewards/chosen": 0.0010917853796854615,
      "rewards/margins": 0.003119278233498335,
      "rewards/rejected": -0.0020274922717362642,
      "step": 60
    },
    {
      "epoch": 0.014079930683418174,
      "grad_norm": 16.375,
      "learning_rate": 1.3391053391053391e-06,
      "logits/chosen": 0.23286700248718262,
      "logits/rejected": 0.18912740051746368,
      "logps/chosen": -210.560791015625,
      "logps/rejected": -209.21542358398438,
      "loss": 0.6942,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": -0.0019684159196913242,
      "rewards/margins": -0.0018725426634773612,
      "rewards/rejected": -9.587309614289552e-05,
      "step": 65
    },
    {
      "epoch": 0.015163002274450341,
      "grad_norm": 17.0,
      "learning_rate": 1.4437229437229438e-06,
      "logits/chosen": 0.13524389266967773,
      "logits/rejected": 0.1275445520877838,
      "logps/chosen": -214.03506469726562,
      "logps/rejected": -224.35000610351562,
      "loss": 0.6908,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.003972552716732025,
      "rewards/margins": 0.004983250983059406,
      "rewards/rejected": -0.001010697684250772,
      "step": 70
    },
    {
      "epoch": 0.01624607386548251,
      "grad_norm": 17.375,
      "learning_rate": 1.5483405483405483e-06,
      "logits/chosen": 0.23101718723773956,
      "logits/rejected": 0.1899498999118805,
      "logps/chosen": -223.79202270507812,
      "logps/rejected": -208.25222778320312,
      "loss": 0.6928,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -9.316140494775027e-05,
      "rewards/margins": 0.0008930111071094871,
      "rewards/rejected": -0.0009861725848168135,
      "step": 75
    },
    {
      "epoch": 0.017329145456514674,
      "grad_norm": 15.5625,
      "learning_rate": 1.652958152958153e-06,
      "logits/chosen": 0.11149375140666962,
      "logits/rejected": 0.15525493025779724,
      "logps/chosen": -223.00326538085938,
      "logps/rejected": -211.6472930908203,
      "loss": 0.6918,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": 0.00044558863737620413,
      "rewards/margins": 0.0030272328294813633,
      "rewards/rejected": -0.0025816443376243114,
      "step": 80
    },
    {
      "epoch": 0.018412217047546843,
      "grad_norm": 15.8125,
      "learning_rate": 1.7575757575757577e-06,
      "logits/chosen": 0.2238444983959198,
      "logits/rejected": 0.19738969206809998,
      "logps/chosen": -227.70101928710938,
      "logps/rejected": -216.350341796875,
      "loss": 0.69,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": 0.006675106938928366,
      "rewards/margins": 0.006547251250594854,
      "rewards/rejected": 0.0001278559648199007,
      "step": 85
    },
    {
      "epoch": 0.01949528863857901,
      "grad_norm": 14.5,
      "learning_rate": 1.8621933621933622e-06,
      "logits/chosen": 0.18416202068328857,
      "logits/rejected": 0.12733903527259827,
      "logps/chosen": -211.986328125,
      "logps/rejected": -202.76695251464844,
      "loss": 0.6908,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": 0.0032894518226385117,
      "rewards/margins": 0.005134430713951588,
      "rewards/rejected": -0.0018449786584824324,
      "step": 90
    },
    {
      "epoch": 0.020578360229611178,
      "grad_norm": 17.0,
      "learning_rate": 1.966810966810967e-06,
      "logits/chosen": 0.2190691977739334,
      "logits/rejected": 0.2271117866039276,
      "logps/chosen": -237.76748657226562,
      "logps/rejected": -225.3360137939453,
      "loss": 0.6933,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": 0.00934534426778555,
      "rewards/margins": 0.00020391792349983007,
      "rewards/rejected": 0.009141424670815468,
      "step": 95
    },
    {
      "epoch": 0.021661431820643343,
      "grad_norm": 17.0,
      "learning_rate": 2.0714285714285713e-06,
      "logits/chosen": 0.18905824422836304,
      "logits/rejected": 0.1897907555103302,
      "logps/chosen": -214.9163818359375,
      "logps/rejected": -229.14163208007812,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00526689225807786,
      "rewards/margins": 0.002697401214390993,
      "rewards/rejected": 0.0025694910436868668,
      "step": 100
    },
    {
      "epoch": 0.022744503411675512,
      "grad_norm": 14.75,
      "learning_rate": 2.1760461760461762e-06,
      "logits/chosen": 0.15061673521995544,
      "logits/rejected": 0.1797480434179306,
      "logps/chosen": -213.80166625976562,
      "logps/rejected": -211.4798126220703,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011642328463494778,
      "rewards/margins": 0.000982316443696618,
      "rewards/rejected": 0.010660013183951378,
      "step": 105
    },
    {
      "epoch": 0.023827575002707678,
      "grad_norm": 15.6875,
      "learning_rate": 2.2806637806637807e-06,
      "logits/chosen": 0.20328040421009064,
      "logits/rejected": 0.27254828810691833,
      "logps/chosen": -225.86880493164062,
      "logps/rejected": -220.21728515625,
      "loss": 0.6893,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.015297770500183105,
      "rewards/margins": 0.008100544102489948,
      "rewards/rejected": 0.007197224535048008,
      "step": 110
    },
    {
      "epoch": 0.024910646593739847,
      "grad_norm": 15.1875,
      "learning_rate": 2.3852813852813856e-06,
      "logits/chosen": 0.16896960139274597,
      "logits/rejected": 0.2158946543931961,
      "logps/chosen": -226.8479461669922,
      "logps/rejected": -230.7816925048828,
      "loss": 0.6882,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.02283587120473385,
      "rewards/margins": 0.010508944280445576,
      "rewards/rejected": 0.012326927855610847,
      "step": 115
    },
    {
      "epoch": 0.025993718184772013,
      "grad_norm": 16.125,
      "learning_rate": 2.4898989898989896e-06,
      "logits/chosen": 0.2090664654970169,
      "logits/rejected": 0.2101777344942093,
      "logps/chosen": -212.3173065185547,
      "logps/rejected": -213.52926635742188,
      "loss": 0.6875,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.027097251266241074,
      "rewards/margins": 0.011709763668477535,
      "rewards/rejected": 0.015387487597763538,
      "step": 120
    },
    {
      "epoch": 0.02707678977580418,
      "grad_norm": 15.9375,
      "learning_rate": 2.5945165945165945e-06,
      "logits/chosen": 0.09042676538228989,
      "logits/rejected": 0.06563390046358109,
      "logps/chosen": -221.8674774169922,
      "logps/rejected": -211.6996307373047,
      "loss": 0.6906,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.024939533323049545,
      "rewards/margins": 0.005633659660816193,
      "rewards/rejected": 0.019305873662233353,
      "step": 125
    },
    {
      "epoch": 0.028159861366836347,
      "grad_norm": 15.0,
      "learning_rate": 2.699134199134199e-06,
      "logits/chosen": 0.19284012913703918,
      "logits/rejected": 0.16713497042655945,
      "logps/chosen": -212.7108917236328,
      "logps/rejected": -221.2731170654297,
      "loss": 0.689,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.028165480121970177,
      "rewards/margins": 0.008775850757956505,
      "rewards/rejected": 0.01938963122665882,
      "step": 130
    },
    {
      "epoch": 0.029242932957868516,
      "grad_norm": 23.5,
      "learning_rate": 2.803751803751804e-06,
      "logits/chosen": 0.14735063910484314,
      "logits/rejected": 0.16729684174060822,
      "logps/chosen": -233.75106811523438,
      "logps/rejected": -227.17117309570312,
      "loss": 0.6879,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": 0.03102860413491726,
      "rewards/margins": 0.011173945851624012,
      "rewards/rejected": 0.019854659214615822,
      "step": 135
    },
    {
      "epoch": 0.030326004548900682,
      "grad_norm": 17.5,
      "learning_rate": 2.9083694083694084e-06,
      "logits/chosen": 0.1849445402622223,
      "logits/rejected": 0.2174483984708786,
      "logps/chosen": -229.0571746826172,
      "logps/rejected": -226.49288940429688,
      "loss": 0.6884,
      "rewards/accuracies": 0.5583333969116211,
      "rewards/chosen": 0.026196394115686417,
      "rewards/margins": 0.0100344717502594,
      "rewards/rejected": 0.01616191864013672,
      "step": 140
    },
    {
      "epoch": 0.03140907613993285,
      "grad_norm": 17.5,
      "learning_rate": 3.0129870129870133e-06,
      "logits/chosen": 0.24837879836559296,
      "logits/rejected": 0.2510320246219635,
      "logps/chosen": -232.657470703125,
      "logps/rejected": -217.3362274169922,
      "loss": 0.6901,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.038914259523153305,
      "rewards/margins": 0.0068990872241556644,
      "rewards/rejected": 0.03201517462730408,
      "step": 145
    },
    {
      "epoch": 0.03249214773096502,
      "grad_norm": 14.125,
      "learning_rate": 3.1176046176046178e-06,
      "logits/chosen": 0.1638328582048416,
      "logits/rejected": 0.1919899731874466,
      "logps/chosen": -203.75811767578125,
      "logps/rejected": -209.1589813232422,
      "loss": 0.6918,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.020969491451978683,
      "rewards/margins": 0.003389708697795868,
      "rewards/rejected": 0.017579780891537666,
      "step": 150
    },
    {
      "epoch": 0.03357521932199718,
      "grad_norm": 15.625,
      "learning_rate": 3.2222222222222222e-06,
      "logits/chosen": 0.16412168741226196,
      "logits/rejected": 0.20279774069786072,
      "logps/chosen": -224.6465301513672,
      "logps/rejected": -219.6198272705078,
      "loss": 0.6888,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.018671419471502304,
      "rewards/margins": 0.0097122797742486,
      "rewards/rejected": 0.008959141559898853,
      "step": 155
    },
    {
      "epoch": 0.03465829091302935,
      "grad_norm": 15.75,
      "learning_rate": 3.3268398268398267e-06,
      "logits/chosen": 0.20621733367443085,
      "logits/rejected": 0.2078239470720291,
      "logps/chosen": -218.03140258789062,
      "logps/rejected": -216.9964141845703,
      "loss": 0.6889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.031520817428827286,
      "rewards/margins": 0.009486904367804527,
      "rewards/rejected": 0.022033914923667908,
      "step": 160
    },
    {
      "epoch": 0.03574136250406152,
      "grad_norm": 16.125,
      "learning_rate": 3.431457431457431e-06,
      "logits/chosen": 0.2025558203458786,
      "logits/rejected": 0.20375001430511475,
      "logps/chosen": -212.5625457763672,
      "logps/rejected": -214.9900665283203,
      "loss": 0.6887,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": 0.03941547870635986,
      "rewards/margins": 0.010077424347400665,
      "rewards/rejected": 0.02933805249631405,
      "step": 165
    },
    {
      "epoch": 0.036824434095093686,
      "grad_norm": 17.25,
      "learning_rate": 3.536075036075036e-06,
      "logits/chosen": 0.12229833751916885,
      "logits/rejected": 0.17494967579841614,
      "logps/chosen": -238.6661376953125,
      "logps/rejected": -219.2349090576172,
      "loss": 0.6902,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.027362888678908348,
      "rewards/margins": 0.007107920944690704,
      "rewards/rejected": 0.020254969596862793,
      "step": 170
    },
    {
      "epoch": 0.03790750568612585,
      "grad_norm": 15.1875,
      "learning_rate": 3.6406926406926406e-06,
      "logits/chosen": 0.19002211093902588,
      "logits/rejected": 0.1998985856771469,
      "logps/chosen": -200.9371795654297,
      "logps/rejected": -199.4198455810547,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018550869077444077,
      "rewards/margins": 0.02260984107851982,
      "rewards/rejected": -0.004058970604091883,
      "step": 175
    },
    {
      "epoch": 0.03899057727715802,
      "grad_norm": 15.3125,
      "learning_rate": 3.7453102453102455e-06,
      "logits/chosen": 0.2148468792438507,
      "logits/rejected": 0.22806307673454285,
      "logps/chosen": -211.16720581054688,
      "logps/rejected": -217.891357421875,
      "loss": 0.6833,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": 0.0350518561899662,
      "rewards/margins": 0.02125047706067562,
      "rewards/rejected": 0.01380138099193573,
      "step": 180
    },
    {
      "epoch": 0.04007364886819019,
      "grad_norm": 14.5,
      "learning_rate": 3.84992784992785e-06,
      "logits/chosen": 0.15949121117591858,
      "logits/rejected": 0.1509333699941635,
      "logps/chosen": -221.38143920898438,
      "logps/rejected": -197.42440795898438,
      "loss": 0.6779,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": 0.05458617955446243,
      "rewards/margins": 0.03268914669752121,
      "rewards/rejected": 0.021897029131650925,
      "step": 185
    },
    {
      "epoch": 0.041156720459222355,
      "grad_norm": 14.625,
      "learning_rate": 3.954545454545454e-06,
      "logits/chosen": 0.15800204873085022,
      "logits/rejected": 0.17494437098503113,
      "logps/chosen": -227.35000610351562,
      "logps/rejected": -217.22531127929688,
      "loss": 0.6818,
      "rewards/accuracies": 0.5500000715255737,
      "rewards/chosen": 0.052388746291399,
      "rewards/margins": 0.024859856814146042,
      "rewards/rejected": 0.02752888761460781,
      "step": 190
    },
    {
      "epoch": 0.04223979205025452,
      "grad_norm": 14.125,
      "learning_rate": 4.059163059163059e-06,
      "logits/chosen": 0.13270123302936554,
      "logits/rejected": 0.17745420336723328,
      "logps/chosen": -235.30062866210938,
      "logps/rejected": -230.95068359375,
      "loss": 0.6879,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": 0.07011522352695465,
      "rewards/margins": 0.012587398290634155,
      "rewards/rejected": 0.057527828961610794,
      "step": 195
    },
    {
      "epoch": 0.04332286364128669,
      "grad_norm": 15.0,
      "learning_rate": 4.163780663780664e-06,
      "logits/chosen": 0.17457877099514008,
      "logits/rejected": 0.20219676196575165,
      "logps/chosen": -213.1217803955078,
      "logps/rejected": -207.86550903320312,
      "loss": 0.6799,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.06976068019866943,
      "rewards/margins": 0.029045337811112404,
      "rewards/rejected": 0.04071534797549248,
      "step": 200
    },
    {
      "epoch": 0.04440593523231886,
      "grad_norm": 15.0625,
      "learning_rate": 4.268398268398269e-06,
      "logits/chosen": 0.1448940485715866,
      "logits/rejected": 0.12982790172100067,
      "logps/chosen": -221.27120971679688,
      "logps/rejected": -219.8614044189453,
      "loss": 0.6822,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.07221545279026031,
      "rewards/margins": 0.02508801594376564,
      "rewards/rejected": 0.047127436846494675,
      "step": 205
    },
    {
      "epoch": 0.045489006823351025,
      "grad_norm": 14.6875,
      "learning_rate": 4.373015873015873e-06,
      "logits/chosen": 0.2132929265499115,
      "logits/rejected": 0.20203714072704315,
      "logps/chosen": -216.64883422851562,
      "logps/rejected": -210.4493865966797,
      "loss": 0.6778,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.0664815828204155,
      "rewards/margins": 0.03378552943468094,
      "rewards/rejected": 0.03269605338573456,
      "step": 210
    },
    {
      "epoch": 0.04657207841438319,
      "grad_norm": 18.0,
      "learning_rate": 4.477633477633478e-06,
      "logits/chosen": 0.17215529084205627,
      "logits/rejected": 0.12756475806236267,
      "logps/chosen": -219.190185546875,
      "logps/rejected": -219.6051788330078,
      "loss": 0.6763,
      "rewards/accuracies": 0.5499999523162842,
      "rewards/chosen": 0.08808019757270813,
      "rewards/margins": 0.039142124354839325,
      "rewards/rejected": 0.04893806576728821,
      "step": 215
    },
    {
      "epoch": 0.047655150005415356,
      "grad_norm": 15.6875,
      "learning_rate": 4.582251082251082e-06,
      "logits/chosen": 0.10778156667947769,
      "logits/rejected": 0.1031012311577797,
      "logps/chosen": -203.22616577148438,
      "logps/rejected": -214.05514526367188,
      "loss": 0.6925,
      "rewards/accuracies": 0.491666704416275,
      "rewards/chosen": 0.05749046057462692,
      "rewards/margins": 0.004513267427682877,
      "rewards/rejected": 0.052977193146944046,
      "step": 220
    },
    {
      "epoch": 0.04873822159644753,
      "grad_norm": 14.6875,
      "learning_rate": 4.6868686868686874e-06,
      "logits/chosen": 0.17147362232208252,
      "logits/rejected": 0.10385062545537949,
      "logps/chosen": -214.05587768554688,
      "logps/rejected": -204.98165893554688,
      "loss": 0.6778,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": 0.06210683658719063,
      "rewards/margins": 0.03475421667098999,
      "rewards/rejected": 0.027352621778845787,
      "step": 225
    },
    {
      "epoch": 0.049821293187479694,
      "grad_norm": 15.75,
      "learning_rate": 4.791486291486291e-06,
      "logits/chosen": 0.19758161902427673,
      "logits/rejected": 0.2416074573993683,
      "logps/chosen": -215.2012176513672,
      "logps/rejected": -216.90518188476562,
      "loss": 0.6791,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.06089228391647339,
      "rewards/margins": 0.03359086066484451,
      "rewards/rejected": 0.027301426976919174,
      "step": 230
    },
    {
      "epoch": 0.05090436477851186,
      "grad_norm": 15.5625,
      "learning_rate": 4.896103896103896e-06,
      "logits/chosen": 0.1594085693359375,
      "logits/rejected": 0.218278169631958,
      "logps/chosen": -208.37255859375,
      "logps/rejected": -209.7632598876953,
      "loss": 0.6784,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04494715481996536,
      "rewards/margins": 0.03450086712837219,
      "rewards/rejected": 0.01044628955423832,
      "step": 235
    },
    {
      "epoch": 0.051987436369544025,
      "grad_norm": 16.0,
      "learning_rate": 5.000721500721501e-06,
      "logits/chosen": 0.1588178128004074,
      "logits/rejected": 0.13714508712291718,
      "logps/chosen": -222.5147705078125,
      "logps/rejected": -217.7800750732422,
      "loss": 0.6792,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.05659140273928642,
      "rewards/margins": 0.032453883439302444,
      "rewards/rejected": 0.02413751557469368,
      "step": 240
    },
    {
      "epoch": 0.05307050796057619,
      "grad_norm": 17.625,
      "learning_rate": 5.105339105339105e-06,
      "logits/chosen": 0.07788325846195221,
      "logits/rejected": 0.1057925671339035,
      "logps/chosen": -225.3970184326172,
      "logps/rejected": -223.532958984375,
      "loss": 0.6716,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.05703939124941826,
      "rewards/margins": 0.049827590584754944,
      "rewards/rejected": 0.007211802992969751,
      "step": 245
    },
    {
      "epoch": 0.05415357955160836,
      "grad_norm": 16.625,
      "learning_rate": 5.20995670995671e-06,
      "logits/chosen": 0.1398146152496338,
      "logits/rejected": 0.17941217124462128,
      "logps/chosen": -218.00048828125,
      "logps/rejected": -222.38833618164062,
      "loss": 0.6816,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.041317857801914215,
      "rewards/margins": 0.029922693967819214,
      "rewards/rejected": 0.011395159177482128,
      "step": 250
    },
    {
      "epoch": 0.05523665114264053,
      "grad_norm": 15.6875,
      "learning_rate": 5.314574314574314e-06,
      "logits/chosen": 0.09205114841461182,
      "logits/rejected": 0.21182039380073547,
      "logps/chosen": -231.6905059814453,
      "logps/rejected": -218.140625,
      "loss": 0.6693,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": 0.06155116483569145,
      "rewards/margins": 0.05434355139732361,
      "rewards/rejected": 0.007207618560642004,
      "step": 255
    },
    {
      "epoch": 0.056319722733672695,
      "grad_norm": 14.1875,
      "learning_rate": 5.41919191919192e-06,
      "logits/chosen": 0.1382916271686554,
      "logits/rejected": 0.14163051545619965,
      "logps/chosen": -224.4682159423828,
      "logps/rejected": -227.1024627685547,
      "loss": 0.6756,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.05809459090232849,
      "rewards/margins": 0.045375265181064606,
      "rewards/rejected": 0.012719330377876759,
      "step": 260
    },
    {
      "epoch": 0.05740279432470486,
      "grad_norm": 17.75,
      "learning_rate": 5.523809523809523e-06,
      "logits/chosen": 0.1413591206073761,
      "logits/rejected": 0.15697917342185974,
      "logps/chosen": -240.87149047851562,
      "logps/rejected": -237.2823028564453,
      "loss": 0.6772,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.06327317655086517,
      "rewards/margins": 0.03917023167014122,
      "rewards/rejected": 0.024102944880723953,
      "step": 265
    },
    {
      "epoch": 0.05848586591573703,
      "grad_norm": 15.625,
      "learning_rate": 5.6284271284271286e-06,
      "logits/chosen": 0.11499279737472534,
      "logits/rejected": 0.18311162292957306,
      "logps/chosen": -227.8660888671875,
      "logps/rejected": -222.97164916992188,
      "loss": 0.6715,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.04170214384794235,
      "rewards/margins": 0.05182771012187004,
      "rewards/rejected": -0.010125571861863136,
      "step": 270
    },
    {
      "epoch": 0.0595689375067692,
      "grad_norm": 16.125,
      "learning_rate": 5.733044733044733e-06,
      "logits/chosen": 0.10107754170894623,
      "logits/rejected": 0.12470720708370209,
      "logps/chosen": -216.609619140625,
      "logps/rejected": -215.77609252929688,
      "loss": 0.675,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": 0.03126338869333267,
      "rewards/margins": 0.046991366893053055,
      "rewards/rejected": -0.015727978199720383,
      "step": 275
    },
    {
      "epoch": 0.060652009097801364,
      "grad_norm": 18.375,
      "learning_rate": 5.837662337662338e-06,
      "logits/chosen": 0.1357327401638031,
      "logits/rejected": 0.1775515377521515,
      "logps/chosen": -227.7059326171875,
      "logps/rejected": -223.28793334960938,
      "loss": 0.6629,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.07952183485031128,
      "rewards/margins": 0.0734293982386589,
      "rewards/rejected": 0.0060924445278942585,
      "step": 280
    },
    {
      "epoch": 0.06173508068883353,
      "grad_norm": 16.25,
      "learning_rate": 5.942279942279942e-06,
      "logits/chosen": 0.038322318345308304,
      "logits/rejected": 0.08118084818124771,
      "logps/chosen": -213.03182983398438,
      "logps/rejected": -220.00198364257812,
      "loss": 0.6719,
      "rewards/accuracies": 0.5750001072883606,
      "rewards/chosen": 0.03023967146873474,
      "rewards/margins": 0.055994223803281784,
      "rewards/rejected": -0.025754550471901894,
      "step": 285
    },
    {
      "epoch": 0.0628181522798657,
      "grad_norm": 17.875,
      "learning_rate": 6.046897546897547e-06,
      "logits/chosen": 0.0733521431684494,
      "logits/rejected": 0.09133279323577881,
      "logps/chosen": -223.6934051513672,
      "logps/rejected": -222.9249725341797,
      "loss": 0.6693,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": 0.014139917679131031,
      "rewards/margins": 0.06154952570796013,
      "rewards/rejected": -0.047409605234861374,
      "step": 290
    },
    {
      "epoch": 0.06390122387089786,
      "grad_norm": 15.5,
      "learning_rate": 6.151515151515152e-06,
      "logits/chosen": 0.05263315886259079,
      "logits/rejected": 0.0594375841319561,
      "logps/chosen": -220.9426727294922,
      "logps/rejected": -214.3191680908203,
      "loss": 0.6757,
      "rewards/accuracies": 0.5000000596046448,
      "rewards/chosen": 2.415962444501929e-05,
      "rewards/margins": 0.05154747888445854,
      "rewards/rejected": -0.05152331665158272,
      "step": 295
    },
    {
      "epoch": 0.06498429546193003,
      "grad_norm": 14.875,
      "learning_rate": 6.256132756132756e-06,
      "logits/chosen": 0.06373531371355057,
      "logits/rejected": 0.03701222315430641,
      "logps/chosen": -233.1228485107422,
      "logps/rejected": -218.87979125976562,
      "loss": 0.6752,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0801302045583725,
      "rewards/margins": 0.053546614944934845,
      "rewards/rejected": 0.026583600789308548,
      "step": 300
    },
    {
      "epoch": 0.0660673670529622,
      "grad_norm": 17.75,
      "learning_rate": 6.360750360750361e-06,
      "logits/chosen": 0.02424457110464573,
      "logits/rejected": 0.03697611019015312,
      "logps/chosen": -224.5616455078125,
      "logps/rejected": -214.629150390625,
      "loss": 0.6676,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.041827913373708725,
      "rewards/margins": 0.0685119554400444,
      "rewards/rejected": -0.02668403461575508,
      "step": 305
    },
    {
      "epoch": 0.06715043864399436,
      "grad_norm": 22.375,
      "learning_rate": 6.465367965367965e-06,
      "logits/chosen": -0.03262064605951309,
      "logits/rejected": -0.022605426609516144,
      "logps/chosen": -245.93026733398438,
      "logps/rejected": -240.89523315429688,
      "loss": 0.6644,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.08075283467769623,
      "rewards/margins": 0.08061286062002182,
      "rewards/rejected": 0.00013995691551826894,
      "step": 310
    },
    {
      "epoch": 0.06823351023502654,
      "grad_norm": 18.375,
      "learning_rate": 6.5699855699855705e-06,
      "logits/chosen": 0.027265310287475586,
      "logits/rejected": 0.05428730696439743,
      "logps/chosen": -224.80923461914062,
      "logps/rejected": -236.8264617919922,
      "loss": 0.66,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": 0.1664566993713379,
      "rewards/margins": 0.08755120635032654,
      "rewards/rejected": 0.07890550047159195,
      "step": 315
    },
    {
      "epoch": 0.0693165818260587,
      "grad_norm": 17.25,
      "learning_rate": 6.674603174603174e-06,
      "logits/chosen": -0.04035397246479988,
      "logits/rejected": -0.05586632341146469,
      "logps/chosen": -210.6708984375,
      "logps/rejected": -209.8553924560547,
      "loss": 0.6974,
      "rewards/accuracies": 0.4833333492279053,
      "rewards/chosen": 0.09260793775320053,
      "rewards/margins": 0.0016569237923249602,
      "rewards/rejected": 0.09095102548599243,
      "step": 320
    },
    {
      "epoch": 0.07039965341709087,
      "grad_norm": 14.1875,
      "learning_rate": 6.7792207792207795e-06,
      "logits/chosen": 0.024581201374530792,
      "logits/rejected": 0.047686561942100525,
      "logps/chosen": -213.9678497314453,
      "logps/rejected": -220.0950927734375,
      "loss": 0.6652,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": 0.15962712466716766,
      "rewards/margins": 0.0764254555106163,
      "rewards/rejected": 0.08320166915655136,
      "step": 325
    },
    {
      "epoch": 0.07148272500812304,
      "grad_norm": 17.0,
      "learning_rate": 6.883838383838384e-06,
      "logits/chosen": -0.014322754926979542,
      "logits/rejected": -0.014882424846291542,
      "logps/chosen": -221.64755249023438,
      "logps/rejected": -219.2441864013672,
      "loss": 0.6559,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": 0.1371501386165619,
      "rewards/margins": 0.09361808001995087,
      "rewards/rejected": 0.043532054871320724,
      "step": 330
    },
    {
      "epoch": 0.0725657965991552,
      "grad_norm": 16.625,
      "learning_rate": 6.9884559884559884e-06,
      "logits/chosen": -0.01833265833556652,
      "logits/rejected": -0.02524412050843239,
      "logps/chosen": -220.4396209716797,
      "logps/rejected": -229.5175323486328,
      "loss": 0.6562,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": 0.10498034954071045,
      "rewards/margins": 0.09323505312204361,
      "rewards/rejected": 0.011745313182473183,
      "step": 335
    },
    {
      "epoch": 0.07364886819018737,
      "grad_norm": 15.875,
      "learning_rate": 7.093073593073593e-06,
      "logits/chosen": 0.027937984094023705,
      "logits/rejected": 0.036226432770490646,
      "logps/chosen": -205.34121704101562,
      "logps/rejected": -212.7598114013672,
      "loss": 0.6689,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.10677335411310196,
      "rewards/margins": 0.06815284490585327,
      "rewards/rejected": 0.03862052038311958,
      "step": 340
    },
    {
      "epoch": 0.07473193978121954,
      "grad_norm": 17.5,
      "learning_rate": 7.197691197691197e-06,
      "logits/chosen": -0.04075038060545921,
      "logits/rejected": -0.03592031076550484,
      "logps/chosen": -228.18234252929688,
      "logps/rejected": -232.10067749023438,
      "loss": 0.6666,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.049600694328546524,
      "rewards/margins": 0.07749857753515244,
      "rewards/rejected": -0.12709927558898926,
      "step": 345
    },
    {
      "epoch": 0.0758150113722517,
      "grad_norm": 17.75,
      "learning_rate": 7.302308802308803e-06,
      "logits/chosen": -0.07422243058681488,
      "logits/rejected": -0.030797475948929787,
      "logps/chosen": -220.2054901123047,
      "logps/rejected": -217.99929809570312,
      "loss": 0.6659,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": -0.050814319401979446,
      "rewards/margins": 0.0966099351644516,
      "rewards/rejected": -0.14742425084114075,
      "step": 350
    },
    {
      "epoch": 0.07689808296328388,
      "grad_norm": 18.0,
      "learning_rate": 7.406926406926407e-06,
      "logits/chosen": -0.03892730176448822,
      "logits/rejected": -0.013924472033977509,
      "logps/chosen": -218.81619262695312,
      "logps/rejected": -218.31472778320312,
      "loss": 0.6641,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08665670454502106,
      "rewards/margins": 0.08160597085952759,
      "rewards/rejected": -0.16826267540454865,
      "step": 355
    },
    {
      "epoch": 0.07798115455431603,
      "grad_norm": 25.25,
      "learning_rate": 7.5115440115440125e-06,
      "logits/chosen": -0.0042107971385121346,
      "logits/rejected": -0.013185444287955761,
      "logps/chosen": -216.36599731445312,
      "logps/rejected": -217.8490753173828,
      "loss": 0.6618,
      "rewards/accuracies": 0.5250000357627869,
      "rewards/chosen": 0.004081724677234888,
      "rewards/margins": 0.08835210651159286,
      "rewards/rejected": -0.08427038043737411,
      "step": 360
    },
    {
      "epoch": 0.07906422614534821,
      "grad_norm": 18.75,
      "learning_rate": 7.616161616161617e-06,
      "logits/chosen": -0.007300414144992828,
      "logits/rejected": -0.02867843583226204,
      "logps/chosen": -231.9053955078125,
      "logps/rejected": -222.1162872314453,
      "loss": 0.6582,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": -0.05084763094782829,
      "rewards/margins": 0.097461998462677,
      "rewards/rejected": -0.1483096182346344,
      "step": 365
    },
    {
      "epoch": 0.08014729773638038,
      "grad_norm": 22.0,
      "learning_rate": 7.72077922077922e-06,
      "logits/chosen": -0.045838575810194016,
      "logits/rejected": -0.0707140639424324,
      "logps/chosen": -247.84750366210938,
      "logps/rejected": -246.65872192382812,
      "loss": 0.6641,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.09232313930988312,
      "rewards/margins": 0.09281657636165619,
      "rewards/rejected": -0.1851397007703781,
      "step": 370
    },
    {
      "epoch": 0.08123036932741254,
      "grad_norm": 18.125,
      "learning_rate": 7.825396825396824e-06,
      "logits/chosen": -0.12464293092489243,
      "logits/rejected": -0.12060137838125229,
      "logps/chosen": -226.3322296142578,
      "logps/rejected": -232.9085693359375,
      "loss": 0.6568,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.17425629496574402,
      "rewards/margins": 0.10088887065649033,
      "rewards/rejected": -0.27514511346817017,
      "step": 375
    },
    {
      "epoch": 0.08231344091844471,
      "grad_norm": 18.75,
      "learning_rate": 7.93001443001443e-06,
      "logits/chosen": -0.09103721380233765,
      "logits/rejected": -0.07036791741847992,
      "logps/chosen": -233.3095703125,
      "logps/rejected": -234.51498413085938,
      "loss": 0.638,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28643327951431274,
      "rewards/margins": 0.14495348930358887,
      "rewards/rejected": -0.4313867688179016,
      "step": 380
    },
    {
      "epoch": 0.08339651250947688,
      "grad_norm": 16.875,
      "learning_rate": 8.034632034632035e-06,
      "logits/chosen": -0.11814365535974503,
      "logits/rejected": -0.11152653396129608,
      "logps/chosen": -231.97702026367188,
      "logps/rejected": -218.13406372070312,
      "loss": 0.6672,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -0.34995827078819275,
      "rewards/margins": 0.0800056904554367,
      "rewards/rejected": -0.42996397614479065,
      "step": 385
    },
    {
      "epoch": 0.08447958410050904,
      "grad_norm": 20.125,
      "learning_rate": 8.13924963924964e-06,
      "logits/chosen": -0.08928295969963074,
      "logits/rejected": -0.041645657271146774,
      "logps/chosen": -229.9120635986328,
      "logps/rejected": -227.5409698486328,
      "loss": 0.6633,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.44191884994506836,
      "rewards/margins": 0.08006083965301514,
      "rewards/rejected": -0.5219796895980835,
      "step": 390
    },
    {
      "epoch": 0.08556265569154121,
      "grad_norm": 13.4375,
      "learning_rate": 8.243867243867244e-06,
      "logits/chosen": -0.10651804506778717,
      "logits/rejected": -0.05364533141255379,
      "logps/chosen": -228.7627716064453,
      "logps/rejected": -235.9076690673828,
      "loss": 0.6391,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.23561103641986847,
      "rewards/margins": 0.14723941683769226,
      "rewards/rejected": -0.3828504681587219,
      "step": 395
    },
    {
      "epoch": 0.08664572728257337,
      "grad_norm": 17.0,
      "learning_rate": 8.348484848484849e-06,
      "logits/chosen": -0.09878838807344437,
      "logits/rejected": -0.09179657697677612,
      "logps/chosen": -231.31005859375,
      "logps/rejected": -237.8728485107422,
      "loss": 0.666,
      "rewards/accuracies": 0.5833333134651184,
      "rewards/chosen": -0.41086870431900024,
      "rewards/margins": 0.11027824878692627,
      "rewards/rejected": -0.5211469531059265,
      "step": 400
    },
    {
      "epoch": 0.08772879887360555,
      "grad_norm": 16.25,
      "learning_rate": 8.453102453102453e-06,
      "logits/chosen": -0.09299886226654053,
      "logits/rejected": -0.07696051895618439,
      "logps/chosen": -204.10049438476562,
      "logps/rejected": -205.93869018554688,
      "loss": 0.6414,
      "rewards/accuracies": 0.60833340883255,
      "rewards/chosen": -0.33197206258773804,
      "rewards/margins": 0.15815743803977966,
      "rewards/rejected": -0.4901294708251953,
      "step": 405
    },
    {
      "epoch": 0.08881187046463772,
      "grad_norm": 17.875,
      "learning_rate": 8.557720057720056e-06,
      "logits/chosen": -0.0773329809308052,
      "logits/rejected": -0.12662465870380402,
      "logps/chosen": -222.2372283935547,
      "logps/rejected": -218.5686798095703,
      "loss": 0.663,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -0.45116180181503296,
      "rewards/margins": 0.11333755403757095,
      "rewards/rejected": -0.5644994378089905,
      "step": 410
    },
    {
      "epoch": 0.08989494205566988,
      "grad_norm": 18.125,
      "learning_rate": 8.662337662337662e-06,
      "logits/chosen": -0.0009993922431021929,
      "logits/rejected": -0.021893642842769623,
      "logps/chosen": -236.42977905273438,
      "logps/rejected": -253.7687225341797,
      "loss": 0.6505,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.2991836667060852,
      "rewards/margins": 0.14704163372516632,
      "rewards/rejected": -0.44622525572776794,
      "step": 415
    },
    {
      "epoch": 0.09097801364670205,
      "grad_norm": 14.8125,
      "learning_rate": 8.766955266955267e-06,
      "logits/chosen": -0.05385687202215195,
      "logits/rejected": -0.09862412512302399,
      "logps/chosen": -227.34921264648438,
      "logps/rejected": -221.52084350585938,
      "loss": 0.6456,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2808450162410736,
      "rewards/margins": 0.1373121440410614,
      "rewards/rejected": -0.418157160282135,
      "step": 420
    },
    {
      "epoch": 0.09206108523773421,
      "grad_norm": 15.8125,
      "learning_rate": 8.871572871572872e-06,
      "logits/chosen": 0.011044780723750591,
      "logits/rejected": 0.015712622553110123,
      "logps/chosen": -230.70608520507812,
      "logps/rejected": -225.9601287841797,
      "loss": 0.6497,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.28303125500679016,
      "rewards/margins": 0.12953974306583405,
      "rewards/rejected": -0.4125710129737854,
      "step": 425
    },
    {
      "epoch": 0.09314415682876638,
      "grad_norm": 18.625,
      "learning_rate": 8.976190476190476e-06,
      "logits/chosen": -0.06477946788072586,
      "logits/rejected": -0.06184889003634453,
      "logps/chosen": -243.337646484375,
      "logps/rejected": -246.96011352539062,
      "loss": 0.6596,
      "rewards/accuracies": 0.60833340883255,
      "rewards/chosen": -0.3403577208518982,
      "rewards/margins": 0.13501493632793427,
      "rewards/rejected": -0.47537264227867126,
      "step": 430
    },
    {
      "epoch": 0.09422722841979855,
      "grad_norm": 12.75,
      "learning_rate": 9.080808080808081e-06,
      "logits/chosen": -0.06717297434806824,
      "logits/rejected": -0.06188998743891716,
      "logps/chosen": -223.9677276611328,
      "logps/rejected": -222.96072387695312,
      "loss": 0.6527,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -0.3672919273376465,
      "rewards/margins": 0.13971467316150665,
      "rewards/rejected": -0.5070066452026367,
      "step": 435
    },
    {
      "epoch": 0.09531030001083071,
      "grad_norm": 14.5625,
      "learning_rate": 9.185425685425687e-06,
      "logits/chosen": -0.03993793576955795,
      "logits/rejected": -0.025322193279862404,
      "logps/chosen": -212.3140869140625,
      "logps/rejected": -215.5955047607422,
      "loss": 0.6603,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -0.40081533789634705,
      "rewards/margins": 0.10362126678228378,
      "rewards/rejected": -0.5044366121292114,
      "step": 440
    },
    {
      "epoch": 0.09639337160186288,
      "grad_norm": 14.4375,
      "learning_rate": 9.29004329004329e-06,
      "logits/chosen": -0.08699233084917068,
      "logits/rejected": -0.09693624824285507,
      "logps/chosen": -225.108642578125,
      "logps/rejected": -220.5970916748047,
      "loss": 0.6444,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": -0.387997031211853,
      "rewards/margins": 0.1516789048910141,
      "rewards/rejected": -0.5396758913993835,
      "step": 445
    },
    {
      "epoch": 0.09747644319289506,
      "grad_norm": 18.125,
      "learning_rate": 9.394660894660894e-06,
      "logits/chosen": -0.019105562940239906,
      "logits/rejected": -0.014042097143828869,
      "logps/chosen": -255.1407928466797,
      "logps/rejected": -253.59469604492188,
      "loss": 0.6521,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.6593637466430664,
      "rewards/margins": 0.14445437490940094,
      "rewards/rejected": -0.8038181066513062,
      "step": 450
    },
    {
      "epoch": 0.09855951478392722,
      "grad_norm": 20.625,
      "learning_rate": 9.4992784992785e-06,
      "logits/chosen": -0.03950546681880951,
      "logits/rejected": -0.05264546349644661,
      "logps/chosen": -228.1761932373047,
      "logps/rejected": -242.2548065185547,
      "loss": 0.6454,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -0.5241656303405762,
      "rewards/margins": 0.17984014749526978,
      "rewards/rejected": -0.704005777835846,
      "step": 455
    },
    {
      "epoch": 0.09964258637495939,
      "grad_norm": 19.5,
      "learning_rate": 9.603896103896105e-06,
      "logits/chosen": -0.009676331654191017,
      "logits/rejected": -0.01860310509800911,
      "logps/chosen": -243.0873565673828,
      "logps/rejected": -237.82028198242188,
      "loss": 0.6656,
      "rewards/accuracies": 0.5916667580604553,
      "rewards/chosen": -0.5300077199935913,
      "rewards/margins": 0.11482787132263184,
      "rewards/rejected": -0.6448356509208679,
      "step": 460
    },
    {
      "epoch": 0.10072565796599155,
      "grad_norm": 16.125,
      "learning_rate": 9.708513708513708e-06,
      "logits/chosen": -0.08747365325689316,
      "logits/rejected": -0.1010398119688034,
      "logps/chosen": -236.60763549804688,
      "logps/rejected": -228.28707885742188,
      "loss": 0.6647,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7339878082275391,
      "rewards/margins": 0.1313539296388626,
      "rewards/rejected": -0.8653417825698853,
      "step": 465
    },
    {
      "epoch": 0.10180872955702372,
      "grad_norm": 19.875,
      "learning_rate": 9.813131313131313e-06,
      "logits/chosen": 0.01666446402668953,
      "logits/rejected": 0.004104399587959051,
      "logps/chosen": -234.9974822998047,
      "logps/rejected": -247.69287109375,
      "loss": 0.6677,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -0.8212010264396667,
      "rewards/margins": 0.15505845844745636,
      "rewards/rejected": -0.9762595295906067,
      "step": 470
    },
    {
      "epoch": 0.10289180114805589,
      "grad_norm": 16.875,
      "learning_rate": 9.917748917748919e-06,
      "logits/chosen": 0.02370421402156353,
      "logits/rejected": -0.003465892281383276,
      "logps/chosen": -228.34353637695312,
      "logps/rejected": -241.65304565429688,
      "loss": 0.6426,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.6112276315689087,
      "rewards/margins": 0.21334891021251678,
      "rewards/rejected": -0.8245763778686523,
      "step": 475
    },
    {
      "epoch": 0.10397487273908805,
      "grad_norm": 20.75,
      "learning_rate": 1.0022366522366522e-05,
      "logits/chosen": -0.007748133037239313,
      "logits/rejected": -0.017456555739045143,
      "logps/chosen": -257.01812744140625,
      "logps/rejected": -240.5902099609375,
      "loss": 0.7052,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7258480191230774,
      "rewards/margins": 0.04560457170009613,
      "rewards/rejected": -0.7714525461196899,
      "step": 480
    },
    {
      "epoch": 0.10505794433012022,
      "grad_norm": 17.625,
      "learning_rate": 1.0126984126984126e-05,
      "logits/chosen": -0.050066132098436356,
      "logits/rejected": -0.01838819310069084,
      "logps/chosen": -225.6297607421875,
      "logps/rejected": -227.48971557617188,
      "loss": 0.6554,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": -0.6196441054344177,
      "rewards/margins": 0.1641055792570114,
      "rewards/rejected": -0.7837496995925903,
      "step": 485
    },
    {
      "epoch": 0.10614101592115238,
      "grad_norm": 16.75,
      "learning_rate": 1.0231601731601731e-05,
      "logits/chosen": 0.034629739820957184,
      "logits/rejected": 0.05039944499731064,
      "logps/chosen": -232.01718139648438,
      "logps/rejected": -230.2293243408203,
      "loss": 0.6189,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -0.42303770780563354,
      "rewards/margins": 0.24148838222026825,
      "rewards/rejected": -0.664526104927063,
      "step": 490
    },
    {
      "epoch": 0.10722408751218455,
      "grad_norm": 19.875,
      "learning_rate": 1.0336219336219337e-05,
      "logits/chosen": -0.04614820331335068,
      "logits/rejected": -0.013111609034240246,
      "logps/chosen": -223.7080078125,
      "logps/rejected": -231.42041015625,
      "loss": 0.6344,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.46331697702407837,
      "rewards/margins": 0.21723313629627228,
      "rewards/rejected": -0.6805501580238342,
      "step": 495
    },
    {
      "epoch": 0.10830715910321673,
      "grad_norm": 17.0,
      "learning_rate": 1.044083694083694e-05,
      "logits/chosen": -0.0733383446931839,
      "logits/rejected": -0.06720686703920364,
      "logps/chosen": -230.7458038330078,
      "logps/rejected": -230.1775360107422,
      "loss": 0.643,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -0.7206427454948425,
      "rewards/margins": 0.168254092335701,
      "rewards/rejected": -0.8888967633247375,
      "step": 500
    },
    {
      "epoch": 0.10830715910321673,
      "eval_logits/chosen": 0.05496968701481819,
      "eval_logits/rejected": 0.1021183654665947,
      "eval_logps/chosen": -225.04754638671875,
      "eval_logps/rejected": -231.59039306640625,
      "eval_loss": 0.6541995406150818,
      "eval_rewards/accuracies": 0.550000011920929,
      "eval_rewards/chosen": -0.5423142910003662,
      "eval_rewards/margins": 0.15434904396533966,
      "eval_rewards/rejected": -0.6966633796691895,
      "eval_runtime": 14.3512,
      "eval_samples_per_second": 13.936,
      "eval_steps_per_second": 13.936,
      "step": 500
    },
    {
      "epoch": 0.10939023069424889,
      "grad_norm": 17.75,
      "learning_rate": 1.0545454545454546e-05,
      "logits/chosen": 0.07579326629638672,
      "logits/rejected": 0.03287886455655098,
      "logps/chosen": -260.544189453125,
      "logps/rejected": -247.03909301757812,
      "loss": 0.6208,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.5900067687034607,
      "rewards/margins": 0.2331467568874359,
      "rewards/rejected": -0.8231536149978638,
      "step": 505
    },
    {
      "epoch": 0.11047330228528106,
      "grad_norm": 18.75,
      "learning_rate": 1.0650072150072151e-05,
      "logits/chosen": -0.011065884493291378,
      "logits/rejected": -0.01818707212805748,
      "logps/chosen": -243.81973266601562,
      "logps/rejected": -236.8044891357422,
      "loss": 0.6337,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.606857180595398,
      "rewards/margins": 0.18417838215827942,
      "rewards/rejected": -0.791035532951355,
      "step": 510
    },
    {
      "epoch": 0.11155637387631323,
      "grad_norm": 22.625,
      "learning_rate": 1.0754689754689756e-05,
      "logits/chosen": -0.04111623018980026,
      "logits/rejected": -0.049740999937057495,
      "logps/chosen": -238.60964965820312,
      "logps/rejected": -232.7783203125,
      "loss": 0.6693,
      "rewards/accuracies": 0.4833333492279053,
      "rewards/chosen": -0.7352408766746521,
      "rewards/margins": 0.13630637526512146,
      "rewards/rejected": -0.8715473413467407,
      "step": 515
    },
    {
      "epoch": 0.11263944546734539,
      "grad_norm": 16.375,
      "learning_rate": 1.0859307359307358e-05,
      "logits/chosen": -0.05193039029836655,
      "logits/rejected": -0.01749078370630741,
      "logps/chosen": -218.5793914794922,
      "logps/rejected": -210.26889038085938,
      "loss": 0.6668,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.6944423913955688,
      "rewards/margins": 0.13761630654335022,
      "rewards/rejected": -0.8320587277412415,
      "step": 520
    },
    {
      "epoch": 0.11372251705837756,
      "grad_norm": 20.25,
      "learning_rate": 1.0963924963924964e-05,
      "logits/chosen": -0.0348048210144043,
      "logits/rejected": -0.010846045799553394,
      "logps/chosen": -241.81216430664062,
      "logps/rejected": -235.23867797851562,
      "loss": 0.6177,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -0.801439106464386,
      "rewards/margins": 0.25389865040779114,
      "rewards/rejected": -1.05533766746521,
      "step": 525
    },
    {
      "epoch": 0.11480558864940972,
      "grad_norm": 27.875,
      "learning_rate": 1.1068542568542569e-05,
      "logits/chosen": -0.07003988325595856,
      "logits/rejected": -0.008445831015706062,
      "logps/chosen": -254.4282684326172,
      "logps/rejected": -250.74160766601562,
      "loss": 0.6666,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -1.1523172855377197,
      "rewards/margins": 0.17704735696315765,
      "rewards/rejected": -1.3293647766113281,
      "step": 530
    },
    {
      "epoch": 0.1158886602404419,
      "grad_norm": 16.375,
      "learning_rate": 1.1173160173160173e-05,
      "logits/chosen": -0.07043690979480743,
      "logits/rejected": -0.053999342024326324,
      "logps/chosen": -234.5797882080078,
      "logps/rejected": -235.4439239501953,
      "loss": 0.6056,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.49969393014907837,
      "rewards/margins": 0.2952284812927246,
      "rewards/rejected": -0.794922411441803,
      "step": 535
    },
    {
      "epoch": 0.11697173183147407,
      "grad_norm": 16.75,
      "learning_rate": 1.1277777777777778e-05,
      "logits/chosen": -0.1617811620235443,
      "logits/rejected": -0.08604887127876282,
      "logps/chosen": -232.7028350830078,
      "logps/rejected": -237.9148406982422,
      "loss": 0.6268,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.15587683022022247,
      "rewards/margins": 0.2105422466993332,
      "rewards/rejected": -0.3664190173149109,
      "step": 540
    },
    {
      "epoch": 0.11805480342250622,
      "grad_norm": 18.875,
      "learning_rate": 1.1382395382395383e-05,
      "logits/chosen": -0.1122334748506546,
      "logits/rejected": -0.1178315058350563,
      "logps/chosen": -226.0067596435547,
      "logps/rejected": -228.5623779296875,
      "loss": 0.6398,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -0.17773057520389557,
      "rewards/margins": 0.1887197047472,
      "rewards/rejected": -0.3664502799510956,
      "step": 545
    },
    {
      "epoch": 0.1191378750135384,
      "grad_norm": 17.625,
      "learning_rate": 1.1487012987012988e-05,
      "logits/chosen": -0.2635635435581207,
      "logits/rejected": -0.18755410611629486,
      "logps/chosen": -222.5266876220703,
      "logps/rejected": -221.80154418945312,
      "loss": 0.6423,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": -0.47580036520957947,
      "rewards/margins": 0.1991702914237976,
      "rewards/rejected": -0.6749706864356995,
      "step": 550
    },
    {
      "epoch": 0.12022094660457056,
      "grad_norm": 15.375,
      "learning_rate": 1.159163059163059e-05,
      "logits/chosen": -0.19910454750061035,
      "logits/rejected": -0.20506444573402405,
      "logps/chosen": -225.6069793701172,
      "logps/rejected": -224.68978881835938,
      "loss": 0.6227,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.4802628457546234,
      "rewards/margins": 0.23474624752998352,
      "rewards/rejected": -0.7150090932846069,
      "step": 555
    },
    {
      "epoch": 0.12130401819560273,
      "grad_norm": 17.625,
      "learning_rate": 1.1696248196248196e-05,
      "logits/chosen": -0.16570356488227844,
      "logits/rejected": -0.1947268694639206,
      "logps/chosen": -230.8046875,
      "logps/rejected": -240.2813262939453,
      "loss": 0.6184,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.5233299136161804,
      "rewards/margins": 0.2735401690006256,
      "rewards/rejected": -0.7968700528144836,
      "step": 560
    },
    {
      "epoch": 0.1223870897866349,
      "grad_norm": 16.0,
      "learning_rate": 1.1800865800865801e-05,
      "logits/chosen": -0.2131640464067459,
      "logits/rejected": -0.23064446449279785,
      "logps/chosen": -228.71322631835938,
      "logps/rejected": -228.3175811767578,
      "loss": 0.6069,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4273219704627991,
      "rewards/margins": 0.3079976439476013,
      "rewards/rejected": -0.7353196144104004,
      "step": 565
    },
    {
      "epoch": 0.12347016137766706,
      "grad_norm": 21.25,
      "learning_rate": 1.1905483405483405e-05,
      "logits/chosen": -0.14151768386363983,
      "logits/rejected": -0.15140186250209808,
      "logps/chosen": -230.6851348876953,
      "logps/rejected": -231.881103515625,
      "loss": 0.6583,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.3910333812236786,
      "rewards/margins": 0.22296765446662903,
      "rewards/rejected": -0.6140009760856628,
      "step": 570
    },
    {
      "epoch": 0.1236867756958735,
      "eval_logits/chosen": -0.18697232007980347,
      "eval_logits/rejected": -0.14263956248760223,
      "eval_logps/chosen": -221.12689208984375,
      "eval_logps/rejected": -228.36573791503906,
      "eval_loss": 0.6643649935722351,
      "eval_rewards/accuracies": 0.574999988079071,
      "eval_rewards/chosen": -0.3462824523448944,
      "eval_rewards/margins": 0.1891491860151291,
      "eval_rewards/rejected": -0.5354316830635071,
      "eval_runtime": 14.3337,
      "eval_samples_per_second": 13.953,
      "eval_steps_per_second": 13.953,
      "step": 571
    },
    {
      "epoch": 0.12455323296869923,
      "grad_norm": 20.375,
      "learning_rate": 1.201010101010101e-05,
      "logits/chosen": -0.21821177005767822,
      "logits/rejected": -0.18691490590572357,
      "logps/chosen": -214.8188934326172,
      "logps/rejected": -233.9182586669922,
      "loss": 0.6875,
      "rewards/accuracies": 0.5166667103767395,
      "rewards/chosen": -0.5784415006637573,
      "rewards/margins": 0.11610467731952667,
      "rewards/rejected": -0.694546103477478,
      "step": 575
    },
    {
      "epoch": 0.1256363045597314,
      "grad_norm": 15.5,
      "learning_rate": 1.2114718614718615e-05,
      "logits/chosen": -0.27361172437667847,
      "logits/rejected": -0.24664859473705292,
      "logps/chosen": -224.4439239501953,
      "logps/rejected": -228.7960968017578,
      "loss": 0.637,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.2837611436843872,
      "rewards/margins": 0.25621241331100464,
      "rewards/rejected": -0.5399735569953918,
      "step": 580
    },
    {
      "epoch": 0.12671937615076356,
      "grad_norm": 17.125,
      "learning_rate": 1.221933621933622e-05,
      "logits/chosen": -0.32113558053970337,
      "logits/rejected": -0.2455451786518097,
      "logps/chosen": -226.5062255859375,
      "logps/rejected": -237.758544921875,
      "loss": 0.6357,
      "rewards/accuracies": 0.5916666388511658,
      "rewards/chosen": -0.3651789128780365,
      "rewards/margins": 0.22158005833625793,
      "rewards/rejected": -0.5867589712142944,
      "step": 585
    },
    {
      "epoch": 0.12780244774179572,
      "grad_norm": 19.875,
      "learning_rate": 1.2323953823953824e-05,
      "logits/chosen": -0.35714685916900635,
      "logits/rejected": -0.34889084100723267,
      "logps/chosen": -231.9417724609375,
      "logps/rejected": -228.2250213623047,
      "loss": 0.6452,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.4237850308418274,
      "rewards/margins": 0.1717890053987503,
      "rewards/rejected": -0.5955739617347717,
      "step": 590
    },
    {
      "epoch": 0.1288855193328279,
      "grad_norm": 19.125,
      "learning_rate": 1.2428571428571428e-05,
      "logits/chosen": -0.29041045904159546,
      "logits/rejected": -0.30822116136550903,
      "logps/chosen": -219.8535919189453,
      "logps/rejected": -223.1898956298828,
      "loss": 0.6243,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3775472640991211,
      "rewards/margins": 0.27673906087875366,
      "rewards/rejected": -0.6542863845825195,
      "step": 595
    },
    {
      "epoch": 0.12996859092386007,
      "grad_norm": 20.375,
      "learning_rate": 1.2533189033189033e-05,
      "logits/chosen": -0.34044113755226135,
      "logits/rejected": -0.3079504370689392,
      "logps/chosen": -233.765625,
      "logps/rejected": -232.15054321289062,
      "loss": 0.6795,
      "rewards/accuracies": 0.5083333253860474,
      "rewards/chosen": -0.5737680196762085,
      "rewards/margins": 0.11013711988925934,
      "rewards/rejected": -0.6839050650596619,
      "step": 600
    },
    {
      "epoch": 0.13105166251489223,
      "grad_norm": 17.125,
      "learning_rate": 1.2637806637806637e-05,
      "logits/chosen": -0.2641218304634094,
      "logits/rejected": -0.3342331051826477,
      "logps/chosen": -232.18545532226562,
      "logps/rejected": -215.61392211914062,
      "loss": 0.657,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7415876984596252,
      "rewards/margins": 0.2052229344844818,
      "rewards/rejected": -0.9468106031417847,
      "step": 605
    },
    {
      "epoch": 0.1321347341059244,
      "grad_norm": 21.25,
      "learning_rate": 1.2742424242424242e-05,
      "logits/chosen": -0.22296516597270966,
      "logits/rejected": -0.226253479719162,
      "logps/chosen": -227.89169311523438,
      "logps/rejected": -238.3671417236328,
      "loss": 0.6477,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -0.6642526388168335,
      "rewards/margins": 0.21578654646873474,
      "rewards/rejected": -0.8800390958786011,
      "step": 610
    },
    {
      "epoch": 0.13321780569695657,
      "grad_norm": 22.125,
      "learning_rate": 1.2847041847041848e-05,
      "logits/chosen": -0.2526755928993225,
      "logits/rejected": -0.22909300029277802,
      "logps/chosen": -244.8173828125,
      "logps/rejected": -245.470458984375,
      "loss": 0.6687,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -0.7103447318077087,
      "rewards/margins": 0.1600169688463211,
      "rewards/rejected": -0.8703616857528687,
      "step": 615
    },
    {
      "epoch": 0.13430087728798873,
      "grad_norm": 19.375,
      "learning_rate": 1.2951659451659453e-05,
      "logits/chosen": -0.19063261151313782,
      "logits/rejected": -0.1503545492887497,
      "logps/chosen": -244.8570098876953,
      "logps/rejected": -245.0282745361328,
      "loss": 0.6378,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -0.6390973329544067,
      "rewards/margins": 0.23612360656261444,
      "rewards/rejected": -0.8752208948135376,
      "step": 620
    },
    {
      "epoch": 0.13538394887902092,
      "grad_norm": 17.25,
      "learning_rate": 1.3056277056277056e-05,
      "logits/chosen": -0.2866547107696533,
      "logits/rejected": -0.2897084355354309,
      "logps/chosen": -221.10629272460938,
      "logps/rejected": -236.65798950195312,
      "loss": 0.6569,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -0.7125317454338074,
      "rewards/margins": 0.19851545989513397,
      "rewards/rejected": -0.9110471606254578,
      "step": 625
    },
    {
      "epoch": 0.13646702047005307,
      "grad_norm": 18.875,
      "learning_rate": 1.316089466089466e-05,
      "logits/chosen": -0.3254905343055725,
      "logits/rejected": -0.2721087634563446,
      "logps/chosen": -234.27999877929688,
      "logps/rejected": -247.65121459960938,
      "loss": 0.6619,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.7822526693344116,
      "rewards/margins": 0.18413785099983215,
      "rewards/rejected": -0.9663906097412109,
      "step": 630
    },
    {
      "epoch": 0.13755009206108523,
      "grad_norm": 15.5625,
      "learning_rate": 1.3265512265512265e-05,
      "logits/chosen": -0.28657814860343933,
      "logits/rejected": -0.2504207193851471,
      "logps/chosen": -241.0985565185547,
      "logps/rejected": -237.91226196289062,
      "loss": 0.6185,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7706317901611328,
      "rewards/margins": 0.2408422976732254,
      "rewards/rejected": -1.0114742517471313,
      "step": 635
    },
    {
      "epoch": 0.1386331636521174,
      "grad_norm": 18.375,
      "learning_rate": 1.3370129870129869e-05,
      "logits/chosen": -0.2968321442604065,
      "logits/rejected": -0.25686365365982056,
      "logps/chosen": -263.0161437988281,
      "logps/rejected": -268.9186096191406,
      "loss": 0.5786,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -0.9727151989936829,
      "rewards/margins": 0.4044572710990906,
      "rewards/rejected": -1.3771723508834839,
      "step": 640
    },
    {
      "epoch": 0.13971623524314958,
      "grad_norm": 22.0,
      "learning_rate": 1.3474747474747474e-05,
      "logits/chosen": -0.38663262128829956,
      "logits/rejected": -0.32514679431915283,
      "logps/chosen": -240.7966766357422,
      "logps/rejected": -256.1520080566406,
      "loss": 0.6329,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": -1.5926291942596436,
      "rewards/margins": 0.30646812915802,
      "rewards/rejected": -1.8990974426269531,
      "step": 645
    },
    {
      "epoch": 0.14079930683418174,
      "grad_norm": 18.875,
      "learning_rate": 1.357936507936508e-05,
      "logits/chosen": -0.46904048323631287,
      "logits/rejected": -0.4902328550815582,
      "logps/chosen": -246.84487915039062,
      "logps/rejected": -243.66659545898438,
      "loss": 0.6345,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4838995933532715,
      "rewards/margins": 0.2877000868320465,
      "rewards/rejected": -1.771599531173706,
      "step": 650
    },
    {
      "epoch": 0.1418823784252139,
      "grad_norm": 21.625,
      "learning_rate": 1.3683982683982685e-05,
      "logits/chosen": -0.35558757185935974,
      "logits/rejected": -0.32473695278167725,
      "logps/chosen": -244.750732421875,
      "logps/rejected": -254.18466186523438,
      "loss": 0.6526,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -1.168528437614441,
      "rewards/margins": 0.1853720098733902,
      "rewards/rejected": -1.3539003133773804,
      "step": 655
    },
    {
      "epoch": 0.14296545001624608,
      "grad_norm": 17.625,
      "learning_rate": 1.3788600288600289e-05,
      "logits/chosen": -0.5020805597305298,
      "logits/rejected": -0.4747430384159088,
      "logps/chosen": -263.0487060546875,
      "logps/rejected": -253.09970092773438,
      "loss": 0.6337,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.0716708898544312,
      "rewards/margins": 0.2554951608181,
      "rewards/rejected": -1.327165961265564,
      "step": 660
    },
    {
      "epoch": 0.14404852160727824,
      "grad_norm": 18.125,
      "learning_rate": 1.3893217893217894e-05,
      "logits/chosen": -0.3936226963996887,
      "logits/rejected": -0.36970776319503784,
      "logps/chosen": -222.0332489013672,
      "logps/rejected": -222.17977905273438,
      "loss": 0.6928,
      "rewards/accuracies": 0.533333420753479,
      "rewards/chosen": -0.6166380643844604,
      "rewards/margins": 0.16012415289878845,
      "rewards/rejected": -0.7767621874809265,
      "step": 665
    },
    {
      "epoch": 0.1451315931983104,
      "grad_norm": 17.25,
      "learning_rate": 1.3997835497835498e-05,
      "logits/chosen": -0.48185500502586365,
      "logits/rejected": -0.49040842056274414,
      "logps/chosen": -227.02395629882812,
      "logps/rejected": -227.2687225341797,
      "loss": 0.646,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": -0.49275097250938416,
      "rewards/margins": 0.1937883347272873,
      "rewards/rejected": -0.6865392923355103,
      "step": 670
    },
    {
      "epoch": 0.14621466478934259,
      "grad_norm": 18.125,
      "learning_rate": 1.4102453102453101e-05,
      "logits/chosen": -0.36353346705436707,
      "logits/rejected": -0.3784257173538208,
      "logps/chosen": -231.03640747070312,
      "logps/rejected": -225.4133758544922,
      "loss": 0.6416,
      "rewards/accuracies": 0.5916666388511658,
      "rewards/chosen": -0.2878168523311615,
      "rewards/margins": 0.20317164063453674,
      "rewards/rejected": -0.490988552570343,
      "step": 675
    },
    {
      "epoch": 0.14729773638037474,
      "grad_norm": 12.875,
      "learning_rate": 1.4207070707070707e-05,
      "logits/chosen": -0.36986222863197327,
      "logits/rejected": -0.31882062554359436,
      "logps/chosen": -227.06509399414062,
      "logps/rejected": -230.50180053710938,
      "loss": 0.6484,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -0.12796230614185333,
      "rewards/margins": 0.22108499705791473,
      "rewards/rejected": -0.3490472733974457,
      "step": 680
    },
    {
      "epoch": 0.1483808079714069,
      "grad_norm": 13.0,
      "learning_rate": 1.4311688311688312e-05,
      "logits/chosen": -0.308027058839798,
      "logits/rejected": -0.2708956003189087,
      "logps/chosen": -218.1095733642578,
      "logps/rejected": -210.42916870117188,
      "loss": 0.612,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -0.14365635812282562,
      "rewards/margins": 0.31176090240478516,
      "rewards/rejected": -0.4554172158241272,
      "step": 685
    },
    {
      "epoch": 0.1494638795624391,
      "grad_norm": 17.125,
      "learning_rate": 1.4416305916305917e-05,
      "logits/chosen": -0.2272690236568451,
      "logits/rejected": -0.2500956058502197,
      "logps/chosen": -218.70303344726562,
      "logps/rejected": -227.6377716064453,
      "loss": 0.6416,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -0.3050081133842468,
      "rewards/margins": 0.18805281817913055,
      "rewards/rejected": -0.4930609166622162,
      "step": 690
    },
    {
      "epoch": 0.15054695115347125,
      "grad_norm": 17.75,
      "learning_rate": 1.4499999844944451e-05,
      "logits/chosen": -0.274125337600708,
      "logits/rejected": -0.2569858133792877,
      "logps/chosen": -229.4094696044922,
      "logps/rejected": -236.3606414794922,
      "loss": 0.6559,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.37352675199508667,
      "rewards/margins": 0.20331712067127228,
      "rewards/rejected": -0.5768438577651978,
      "step": 695
    },
    {
      "epoch": 0.1516300227445034,
      "grad_norm": 19.375,
      "learning_rate": 1.4499994418001168e-05,
      "logits/chosen": -0.23347747325897217,
      "logits/rejected": -0.2249656468629837,
      "logps/chosen": -239.08370971679688,
      "logps/rejected": -238.60433959960938,
      "loss": 0.6508,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.4247639775276184,
      "rewards/margins": 0.19854626059532166,
      "rewards/rejected": -0.6233102679252625,
      "step": 700
    },
    {
      "epoch": 0.15271309433553557,
      "grad_norm": 19.25,
      "learning_rate": 1.4499981238289282e-05,
      "logits/chosen": -0.273943156003952,
      "logits/rejected": -0.23270830512046814,
      "logps/chosen": -221.48080444335938,
      "logps/rejected": -245.1928253173828,
      "loss": 0.6278,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4344627857208252,
      "rewards/margins": 0.2935953736305237,
      "rewards/rejected": -0.7280580997467041,
      "step": 705
    },
    {
      "epoch": 0.15379616592656775,
      "grad_norm": 17.625,
      "learning_rate": 1.4499960305827585e-05,
      "logits/chosen": -0.3461950123310089,
      "logits/rejected": -0.37449464201927185,
      "logps/chosen": -226.706298828125,
      "logps/rejected": -225.2688446044922,
      "loss": 0.6154,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -0.5238646268844604,
      "rewards/margins": 0.3101276755332947,
      "rewards/rejected": -0.8339923024177551,
      "step": 710
    },
    {
      "epoch": 0.1548792375175999,
      "grad_norm": 14.375,
      "learning_rate": 1.4499931620645923e-05,
      "logits/chosen": -0.4055313169956207,
      "logits/rejected": -0.3884700834751129,
      "logps/chosen": -223.248291015625,
      "logps/rejected": -218.5846405029297,
      "loss": 0.6219,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.5440430045127869,
      "rewards/margins": 0.30822134017944336,
      "rewards/rejected": -0.8522642850875854,
      "step": 715
    },
    {
      "epoch": 0.15596230910863207,
      "grad_norm": 17.75,
      "learning_rate": 1.4499895182785195e-05,
      "logits/chosen": -0.31495344638824463,
      "logits/rejected": -0.3310261368751526,
      "logps/chosen": -237.7204132080078,
      "logps/rejected": -235.2788848876953,
      "loss": 0.5967,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -0.4934661388397217,
      "rewards/margins": 0.35858526825904846,
      "rewards/rejected": -0.8520514369010925,
      "step": 720
    },
    {
      "epoch": 0.15704538069966426,
      "grad_norm": 13.6875,
      "learning_rate": 1.4499850992297353e-05,
      "logits/chosen": -0.3323173522949219,
      "logits/rejected": -0.27078571915626526,
      "logps/chosen": -236.0640411376953,
      "logps/rejected": -238.24362182617188,
      "loss": 0.6622,
      "rewards/accuracies": 0.5500000715255737,
      "rewards/chosen": -0.576398491859436,
      "rewards/margins": 0.1917855590581894,
      "rewards/rejected": -0.7681840658187866,
      "step": 725
    },
    {
      "epoch": 0.15812845229069641,
      "grad_norm": 17.75,
      "learning_rate": 1.4499799049245405e-05,
      "logits/chosen": -0.31259989738464355,
      "logits/rejected": -0.31948864459991455,
      "logps/chosen": -229.38784790039062,
      "logps/rejected": -231.2362518310547,
      "loss": 0.6211,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.42887917160987854,
      "rewards/margins": 0.27277007699012756,
      "rewards/rejected": -0.7016493082046509,
      "step": 730
    },
    {
      "epoch": 0.15921152388172857,
      "grad_norm": 18.875,
      "learning_rate": 1.449973935370341e-05,
      "logits/chosen": -0.36724361777305603,
      "logits/rejected": -0.31422799825668335,
      "logps/chosen": -249.6378936767578,
      "logps/rejected": -238.80111694335938,
      "loss": 0.6594,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -0.38307029008865356,
      "rewards/margins": 0.2451951801776886,
      "rewards/rejected": -0.628265380859375,
      "step": 735
    },
    {
      "epoch": 0.16029459547276076,
      "grad_norm": 16.25,
      "learning_rate": 1.4499671905756485e-05,
      "logits/chosen": -0.26126721501350403,
      "logits/rejected": -0.26691120862960815,
      "logps/chosen": -228.2554168701172,
      "logps/rejected": -233.48300170898438,
      "loss": 0.6401,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -0.3549858033657074,
      "rewards/margins": 0.2318643033504486,
      "rewards/rejected": -0.586850106716156,
      "step": 740
    },
    {
      "epoch": 0.16137766706379292,
      "grad_norm": 18.0,
      "learning_rate": 1.4499596705500792e-05,
      "logits/chosen": -0.34698179364204407,
      "logits/rejected": -0.3227859139442444,
      "logps/chosen": -246.04434204101562,
      "logps/rejected": -229.87130737304688,
      "loss": 0.6107,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -0.6188342571258545,
      "rewards/margins": 0.2819725573062897,
      "rewards/rejected": -0.9008067846298218,
      "step": 745
    },
    {
      "epoch": 0.16246073865482508,
      "grad_norm": 14.0625,
      "learning_rate": 1.4499513753043555e-05,
      "logits/chosen": -0.3285478949546814,
      "logits/rejected": -0.2592875063419342,
      "logps/chosen": -242.3033447265625,
      "logps/rejected": -244.37863159179688,
      "loss": 0.6204,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.687212347984314,
      "rewards/margins": 0.3234688639640808,
      "rewards/rejected": -1.0106810331344604,
      "step": 750
    },
    {
      "epoch": 0.16354381024585726,
      "grad_norm": 18.75,
      "learning_rate": 1.4499423048503046e-05,
      "logits/chosen": -0.2667035162448883,
      "logits/rejected": -0.2690730094909668,
      "logps/chosen": -229.2095947265625,
      "logps/rejected": -234.4394073486328,
      "loss": 0.6336,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -0.947296142578125,
      "rewards/margins": 0.2761627435684204,
      "rewards/rejected": -1.223459005355835,
      "step": 755
    },
    {
      "epoch": 0.16462688183688942,
      "grad_norm": 19.625,
      "learning_rate": 1.4499324592008594e-05,
      "logits/chosen": -0.3499573767185211,
      "logits/rejected": -0.2950364649295807,
      "logps/chosen": -255.4847869873047,
      "logps/rejected": -242.3585968017578,
      "loss": 0.5869,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -0.9345858693122864,
      "rewards/margins": 0.4139852523803711,
      "rewards/rejected": -1.3485710620880127,
      "step": 760
    },
    {
      "epoch": 0.16570995342792158,
      "grad_norm": 20.25,
      "learning_rate": 1.4499218383700576e-05,
      "logits/chosen": -0.34292224049568176,
      "logits/rejected": -0.3146454691886902,
      "logps/chosen": -241.71157836914062,
      "logps/rejected": -243.81192016601562,
      "loss": 0.6512,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0744554996490479,
      "rewards/margins": 0.2668365240097046,
      "rewards/rejected": -1.341291904449463,
      "step": 765
    },
    {
      "epoch": 0.16679302501895377,
      "grad_norm": 14.1875,
      "learning_rate": 1.4499104423730421e-05,
      "logits/chosen": -0.3574206531047821,
      "logits/rejected": -0.33590954542160034,
      "logps/chosen": -249.6627197265625,
      "logps/rejected": -262.67041015625,
      "loss": 0.6045,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -0.7686626315116882,
      "rewards/margins": 0.36995929479599,
      "rewards/rejected": -1.1386218070983887,
      "step": 770
    },
    {
      "epoch": 0.16787609660998593,
      "grad_norm": 23.875,
      "learning_rate": 1.4498982712260617e-05,
      "logits/chosen": -0.4615628719329834,
      "logits/rejected": -0.4099772870540619,
      "logps/chosen": -253.5238800048828,
      "logps/rejected": -249.9038543701172,
      "loss": 0.6768,
      "rewards/accuracies": 0.5500000715255737,
      "rewards/chosen": -1.1717009544372559,
      "rewards/margins": 0.232270747423172,
      "rewards/rejected": -1.403971791267395,
      "step": 775
    },
    {
      "epoch": 0.16895916820101808,
      "grad_norm": 17.75,
      "learning_rate": 1.44988532494647e-05,
      "logits/chosen": -0.49601849913597107,
      "logits/rejected": -0.5579857230186462,
      "logps/chosen": -236.09323120117188,
      "logps/rejected": -233.7776336669922,
      "loss": 0.6319,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -1.0762754678726196,
      "rewards/margins": 0.2984899580478668,
      "rewards/rejected": -1.3747655153274536,
      "step": 780
    },
    {
      "epoch": 0.17004223979205024,
      "grad_norm": 20.75,
      "learning_rate": 1.4498716035527255e-05,
      "logits/chosen": -0.3665028512477875,
      "logits/rejected": -0.3021479845046997,
      "logps/chosen": -245.8675994873047,
      "logps/rejected": -250.3119354248047,
      "loss": 0.6201,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8807302713394165,
      "rewards/margins": 0.4032112658023834,
      "rewards/rejected": -1.283941388130188,
      "step": 785
    },
    {
      "epoch": 0.17112531138308243,
      "grad_norm": 15.4375,
      "learning_rate": 1.4498571070643921e-05,
      "logits/chosen": -0.37830302119255066,
      "logits/rejected": -0.3760519027709961,
      "logps/chosen": -262.40911865234375,
      "logps/rejected": -256.75909423828125,
      "loss": 0.6002,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": -1.1192796230316162,
      "rewards/margins": 0.4616367816925049,
      "rewards/rejected": -1.5809166431427002,
      "step": 790
    },
    {
      "epoch": 0.1722083829741146,
      "grad_norm": 20.375,
      "learning_rate": 1.4498418355021392e-05,
      "logits/chosen": -0.3115169107913971,
      "logits/rejected": -0.3008628487586975,
      "logps/chosen": -256.7808837890625,
      "logps/rejected": -254.7860870361328,
      "loss": 0.6696,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -1.427552580833435,
      "rewards/margins": 0.25245940685272217,
      "rewards/rejected": -1.6800121068954468,
      "step": 795
    },
    {
      "epoch": 0.17329145456514675,
      "grad_norm": 12.625,
      "learning_rate": 1.4498257888877407e-05,
      "logits/chosen": -0.4118855893611908,
      "logits/rejected": -0.3780352771282196,
      "logps/chosen": -226.22177124023438,
      "logps/rejected": -229.22640991210938,
      "loss": 0.6786,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.2883083820343018,
      "rewards/margins": 0.1932557374238968,
      "rewards/rejected": -1.4815640449523926,
      "step": 800
    },
    {
      "epoch": 0.17437452615617893,
      "grad_norm": 17.625,
      "learning_rate": 1.4498089672440762e-05,
      "logits/chosen": -0.4935850501060486,
      "logits/rejected": -0.41620177030563354,
      "logps/chosen": -239.71328735351562,
      "logps/rejected": -246.760009765625,
      "loss": 0.5756,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.0668771266937256,
      "rewards/margins": 0.4998171329498291,
      "rewards/rejected": -1.5666941404342651,
      "step": 805
    },
    {
      "epoch": 0.1754575977472111,
      "grad_norm": 24.625,
      "learning_rate": 1.4497913705951293e-05,
      "logits/chosen": -0.5555132031440735,
      "logits/rejected": -0.5195592045783997,
      "logps/chosen": -237.3116912841797,
      "logps/rejected": -242.3108367919922,
      "loss": 0.6086,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.5386208295822144,
      "rewards/margins": 0.3710339367389679,
      "rewards/rejected": -1.9096548557281494,
      "step": 810
    },
    {
      "epoch": 0.17654066933824325,
      "grad_norm": 21.125,
      "learning_rate": 1.44977299896599e-05,
      "logits/chosen": -0.4909455180168152,
      "logits/rejected": -0.5195225477218628,
      "logps/chosen": -273.40216064453125,
      "logps/rejected": -258.47528076171875,
      "loss": 0.675,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -1.9044948816299438,
      "rewards/margins": 0.23271982371807098,
      "rewards/rejected": -2.1372151374816895,
      "step": 815
    },
    {
      "epoch": 0.17762374092927544,
      "grad_norm": 19.0,
      "learning_rate": 1.4497538523828519e-05,
      "logits/chosen": -0.6464676856994629,
      "logits/rejected": -0.6165034174919128,
      "logps/chosen": -269.4308166503906,
      "logps/rejected": -272.34747314453125,
      "loss": 0.6278,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -2.2519779205322266,
      "rewards/margins": 0.3564743399620056,
      "rewards/rejected": -2.608452558517456,
      "step": 820
    },
    {
      "epoch": 0.1787068125203076,
      "grad_norm": 23.5,
      "learning_rate": 1.4497339308730146e-05,
      "logits/chosen": -0.6236425638198853,
      "logits/rejected": -0.6261758208274841,
      "logps/chosen": -261.0154113769531,
      "logps/rejected": -265.5215759277344,
      "loss": 0.7071,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -2.338916301727295,
      "rewards/margins": 0.25896400213241577,
      "rewards/rejected": -2.5978801250457764,
      "step": 825
    },
    {
      "epoch": 0.17978988411133975,
      "grad_norm": 35.0,
      "learning_rate": 1.4497132344648819e-05,
      "logits/chosen": -0.5946786403656006,
      "logits/rejected": -0.5437528491020203,
      "logps/chosen": -281.34454345703125,
      "logps/rejected": -273.1812438964844,
      "loss": 0.7318,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.635209321975708,
      "rewards/margins": 0.22759974002838135,
      "rewards/rejected": -2.8628089427948,
      "step": 830
    },
    {
      "epoch": 0.18087295570237194,
      "grad_norm": 23.0,
      "learning_rate": 1.449691763187963e-05,
      "logits/chosen": -0.4379182755947113,
      "logits/rejected": -0.4615257680416107,
      "logps/chosen": -266.9535217285156,
      "logps/rejected": -257.15081787109375,
      "loss": 0.6552,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -1.6634538173675537,
      "rewards/margins": 0.25891169905662537,
      "rewards/rejected": -1.9223655462265015,
      "step": 835
    },
    {
      "epoch": 0.1819560272934041,
      "grad_norm": 18.625,
      "learning_rate": 1.449669517072871e-05,
      "logits/chosen": -0.3618869185447693,
      "logits/rejected": -0.3624362051486969,
      "logps/chosen": -257.85858154296875,
      "logps/rejected": -255.617431640625,
      "loss": 0.5973,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3950719833374023,
      "rewards/margins": 0.38865116238594055,
      "rewards/rejected": -1.7837231159210205,
      "step": 840
    },
    {
      "epoch": 0.18303909888443626,
      "grad_norm": 16.875,
      "learning_rate": 1.449646496151325e-05,
      "logits/chosen": -0.36781197786331177,
      "logits/rejected": -0.30683714151382446,
      "logps/chosen": -267.01812744140625,
      "logps/rejected": -269.34332275390625,
      "loss": 0.601,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -1.787444829940796,
      "rewards/margins": 0.3785509467124939,
      "rewards/rejected": -2.1659958362579346,
      "step": 845
    },
    {
      "epoch": 0.18412217047546842,
      "grad_norm": 24.5,
      "learning_rate": 1.449622700456148e-05,
      "logits/chosen": -0.31594809889793396,
      "logits/rejected": -0.31250837445259094,
      "logps/chosen": -267.1878356933594,
      "logps/rejected": -276.07000732421875,
      "loss": 0.6448,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.7256494760513306,
      "rewards/margins": 0.2959543764591217,
      "rewards/rejected": -2.02160382270813,
      "step": 850
    },
    {
      "epoch": 0.1852052420665006,
      "grad_norm": 23.5,
      "learning_rate": 1.4495981300212677e-05,
      "logits/chosen": -0.3309677243232727,
      "logits/rejected": -0.30699384212493896,
      "logps/chosen": -258.7310485839844,
      "logps/rejected": -241.2162628173828,
      "loss": 0.6135,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.2959928512573242,
      "rewards/margins": 0.2982693314552307,
      "rewards/rejected": -1.5942620038986206,
      "step": 855
    },
    {
      "epoch": 0.18628831365753276,
      "grad_norm": 20.625,
      "learning_rate": 1.4495727848817168e-05,
      "logits/chosen": -0.38755857944488525,
      "logits/rejected": -0.3828616738319397,
      "logps/chosen": -244.0364227294922,
      "logps/rejected": -246.99880981445312,
      "loss": 0.6161,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -0.879804253578186,
      "rewards/margins": 0.3173539638519287,
      "rewards/rejected": -1.1971582174301147,
      "step": 860
    },
    {
      "epoch": 0.18737138524856492,
      "grad_norm": 17.375,
      "learning_rate": 1.4495466650736322e-05,
      "logits/chosen": -0.3464072048664093,
      "logits/rejected": -0.2857853174209595,
      "logps/chosen": -224.05929565429688,
      "logps/rejected": -237.05862426757812,
      "loss": 0.6002,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -0.7999496459960938,
      "rewards/margins": 0.42959722876548767,
      "rewards/rejected": -1.2295467853546143,
      "step": 865
    },
    {
      "epoch": 0.1884544568395971,
      "grad_norm": 25.25,
      "learning_rate": 1.4495197706342553e-05,
      "logits/chosen": -0.32289618253707886,
      "logits/rejected": -0.32329314947128296,
      "logps/chosen": -246.7991180419922,
      "logps/rejected": -250.8223114013672,
      "loss": 0.6157,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -0.9370712041854858,
      "rewards/margins": 0.36822184920310974,
      "rewards/rejected": -1.305293321609497,
      "step": 870
    },
    {
      "epoch": 0.18953752843062927,
      "grad_norm": 18.25,
      "learning_rate": 1.4494921016019326e-05,
      "logits/chosen": -0.35323861241340637,
      "logits/rejected": -0.3620947003364563,
      "logps/chosen": -261.53546142578125,
      "logps/rejected": -270.94671630859375,
      "loss": 0.6408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8733810186386108,
      "rewards/margins": 0.2590027153491974,
      "rewards/rejected": -2.1323838233947754,
      "step": 875
    },
    {
      "epoch": 0.19062060002166142,
      "grad_norm": 14.5,
      "learning_rate": 1.4494636580161141e-05,
      "logits/chosen": -0.3451077938079834,
      "logits/rejected": -0.36713337898254395,
      "logps/chosen": -260.6431579589844,
      "logps/rejected": -259.15716552734375,
      "loss": 0.5965,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.3031201362609863,
      "rewards/margins": 0.4897305369377136,
      "rewards/rejected": -2.792850971221924,
      "step": 880
    },
    {
      "epoch": 0.1917036716126936,
      "grad_norm": 19.875,
      "learning_rate": 1.4494344399173548e-05,
      "logits/chosen": -0.4403704106807709,
      "logits/rejected": -0.42872968316078186,
      "logps/chosen": -270.8282775878906,
      "logps/rejected": -276.2127990722656,
      "loss": 0.6726,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": -2.594325065612793,
      "rewards/margins": 0.22175589203834534,
      "rewards/rejected": -2.8160812854766846,
      "step": 885
    },
    {
      "epoch": 0.19278674320372577,
      "grad_norm": 18.875,
      "learning_rate": 1.4494044473473138e-05,
      "logits/chosen": -0.4329702854156494,
      "logits/rejected": -0.38462215662002563,
      "logps/chosen": -268.43109130859375,
      "logps/rejected": -266.3245849609375,
      "loss": 0.6246,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -2.29178524017334,
      "rewards/margins": 0.2931796908378601,
      "rewards/rejected": -2.584965229034424,
      "step": 890
    },
    {
      "epoch": 0.19386981479475793,
      "grad_norm": 19.875,
      "learning_rate": 1.4493736803487546e-05,
      "logits/chosen": -0.5599915385246277,
      "logits/rejected": -0.6114809513092041,
      "logps/chosen": -256.8448791503906,
      "logps/rejected": -256.194580078125,
      "loss": 0.6647,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -2.215803861618042,
      "rewards/margins": 0.21020975708961487,
      "rewards/rejected": -2.426013469696045,
      "step": 895
    },
    {
      "epoch": 0.19495288638579011,
      "grad_norm": 13.625,
      "learning_rate": 1.4493421389655442e-05,
      "logits/chosen": -0.6140152215957642,
      "logits/rejected": -0.5496706962585449,
      "logps/chosen": -248.46646118164062,
      "logps/rejected": -260.5649719238281,
      "loss": 0.617,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -1.423843264579773,
      "rewards/margins": 0.33675917983055115,
      "rewards/rejected": -1.760602355003357,
      "step": 900
    },
    {
      "epoch": 0.19603595797682227,
      "grad_norm": 15.8125,
      "learning_rate": 1.449309823242655e-05,
      "logits/chosen": -0.5929363965988159,
      "logits/rejected": -0.596908688545227,
      "logps/chosen": -254.1724395751953,
      "logps/rejected": -254.25146484375,
      "loss": 0.6257,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1556651592254639,
      "rewards/margins": 0.32957789301872253,
      "rewards/rejected": -1.4852432012557983,
      "step": 905
    },
    {
      "epoch": 0.19711902956785443,
      "grad_norm": 19.75,
      "learning_rate": 1.449276733226162e-05,
      "logits/chosen": -0.581192672252655,
      "logits/rejected": -0.60381019115448,
      "logps/chosen": -248.6654815673828,
      "logps/rejected": -246.63296508789062,
      "loss": 0.6053,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.3521748781204224,
      "rewards/margins": 0.3377714157104492,
      "rewards/rejected": -1.6899464130401611,
      "step": 910
    },
    {
      "epoch": 0.1982021011588866,
      "grad_norm": 15.9375,
      "learning_rate": 1.4492428689632456e-05,
      "logits/chosen": -0.6221476793289185,
      "logits/rejected": -0.5684457421302795,
      "logps/chosen": -240.48654174804688,
      "logps/rejected": -239.5816650390625,
      "loss": 0.623,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -1.1328710317611694,
      "rewards/margins": 0.37279266119003296,
      "rewards/rejected": -1.5056636333465576,
      "step": 915
    },
    {
      "epoch": 0.19928517274991878,
      "grad_norm": 21.75,
      "learning_rate": 1.4492082305021886e-05,
      "logits/chosen": -0.5589616894721985,
      "logits/rejected": -0.5555655360221863,
      "logps/chosen": -265.9097595214844,
      "logps/rejected": -271.87506103515625,
      "loss": 0.5766,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9331512451171875,
      "rewards/margins": 0.40755611658096313,
      "rewards/rejected": -2.340707302093506,
      "step": 920
    },
    {
      "epoch": 0.20036824434095094,
      "grad_norm": 22.125,
      "learning_rate": 1.4491728178923788e-05,
      "logits/chosen": -0.5604474544525146,
      "logits/rejected": -0.5666874051094055,
      "logps/chosen": -261.94329833984375,
      "logps/rejected": -259.55816650390625,
      "loss": 0.661,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9687204360961914,
      "rewards/margins": 0.3331766724586487,
      "rewards/rejected": -2.3018970489501953,
      "step": 925
    },
    {
      "epoch": 0.2014513159319831,
      "grad_norm": 16.375,
      "learning_rate": 1.4491366311843076e-05,
      "logits/chosen": -0.5484265089035034,
      "logits/rejected": -0.5626634359359741,
      "logps/chosen": -252.776611328125,
      "logps/rejected": -257.74932861328125,
      "loss": 0.6125,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.7037357091903687,
      "rewards/margins": 0.42012786865234375,
      "rewards/rejected": -2.123863697052002,
      "step": 930
    },
    {
      "epoch": 0.20253438752301528,
      "grad_norm": 16.0,
      "learning_rate": 1.4490996704295698e-05,
      "logits/chosen": -0.5050543546676636,
      "logits/rejected": -0.49235111474990845,
      "logps/chosen": -240.30166625976562,
      "logps/rejected": -248.2396240234375,
      "loss": 0.6198,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.415097951889038,
      "rewards/margins": 0.2975335717201233,
      "rewards/rejected": -1.7126315832138062,
      "step": 935
    },
    {
      "epoch": 0.20361745911404744,
      "grad_norm": 15.25,
      "learning_rate": 1.4490619356808641e-05,
      "logits/chosen": -0.39765504002571106,
      "logits/rejected": -0.3562256693840027,
      "logps/chosen": -244.9200439453125,
      "logps/rejected": -259.78619384765625,
      "loss": 0.6719,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -1.3268649578094482,
      "rewards/margins": 0.24011754989624023,
      "rewards/rejected": -1.566982626914978,
      "step": 940
    },
    {
      "epoch": 0.2047005307050796,
      "grad_norm": 17.625,
      "learning_rate": 1.4490234269919922e-05,
      "logits/chosen": -0.3865676522254944,
      "logits/rejected": -0.4370853006839752,
      "logps/chosen": -226.709228515625,
      "logps/rejected": -230.2788848876953,
      "loss": 0.5907,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -0.7181413173675537,
      "rewards/margins": 0.3694535791873932,
      "rewards/rejected": -1.0875948667526245,
      "step": 945
    },
    {
      "epoch": 0.20578360229611178,
      "grad_norm": 15.8125,
      "learning_rate": 1.44898414441786e-05,
      "logits/chosen": -0.39653101563453674,
      "logits/rejected": -0.35863572359085083,
      "logps/chosen": -230.885986328125,
      "logps/rejected": -236.51455688476562,
      "loss": 0.6122,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -0.5346847176551819,
      "rewards/margins": 0.3764137029647827,
      "rewards/rejected": -0.9110983610153198,
      "step": 950
    },
    {
      "epoch": 0.20686667388714394,
      "grad_norm": 23.375,
      "learning_rate": 1.4489440880144766e-05,
      "logits/chosen": -0.3637053072452545,
      "logits/rejected": -0.3581816852092743,
      "logps/chosen": -253.2418670654297,
      "logps/rejected": -241.5313720703125,
      "loss": 0.593,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -0.7789807915687561,
      "rewards/margins": 0.42516031861305237,
      "rewards/rejected": -1.2041410207748413,
      "step": 955
    },
    {
      "epoch": 0.2079497454781761,
      "grad_norm": 18.0,
      "learning_rate": 1.4489032578389544e-05,
      "logits/chosen": -0.39388224482536316,
      "logits/rejected": -0.3822733461856842,
      "logps/chosen": -241.7135772705078,
      "logps/rejected": -245.359375,
      "loss": 0.6441,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -1.2317988872528076,
      "rewards/margins": 0.2839626669883728,
      "rewards/rejected": -1.5157614946365356,
      "step": 960
    },
    {
      "epoch": 0.2090328170692083,
      "grad_norm": 19.25,
      "learning_rate": 1.4488616539495085e-05,
      "logits/chosen": -0.2972071170806885,
      "logits/rejected": -0.28080636262893677,
      "logps/chosen": -248.03085327148438,
      "logps/rejected": -254.10791015625,
      "loss": 0.6254,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3986542224884033,
      "rewards/margins": 0.3080451190471649,
      "rewards/rejected": -1.7066996097564697,
      "step": 965
    },
    {
      "epoch": 0.21011588866024045,
      "grad_norm": 13.75,
      "learning_rate": 1.4488192764054582e-05,
      "logits/chosen": -0.29594630002975464,
      "logits/rejected": -0.2356942892074585,
      "logps/chosen": -244.5025634765625,
      "logps/rejected": -246.6710662841797,
      "loss": 0.6118,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1517527103424072,
      "rewards/margins": 0.39364543557167053,
      "rewards/rejected": -1.545398235321045,
      "step": 970
    },
    {
      "epoch": 0.2111989602512726,
      "grad_norm": 17.25,
      "learning_rate": 1.4487761252672253e-05,
      "logits/chosen": -0.28070899844169617,
      "logits/rejected": -0.2997274100780487,
      "logps/chosen": -266.51873779296875,
      "logps/rejected": -250.923828125,
      "loss": 0.5952,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.185519814491272,
      "rewards/margins": 0.43934112787246704,
      "rewards/rejected": -1.6248607635498047,
      "step": 975
    },
    {
      "epoch": 0.21228203184230476,
      "grad_norm": 17.125,
      "learning_rate": 1.448732200596334e-05,
      "logits/chosen": -0.35207849740982056,
      "logits/rejected": -0.3376738429069519,
      "logps/chosen": -235.95608520507812,
      "logps/rejected": -237.05209350585938,
      "loss": 0.6782,
      "rewards/accuracies": 0.5833333134651184,
      "rewards/chosen": -1.336116075515747,
      "rewards/margins": 0.15997274219989777,
      "rewards/rejected": -1.496088981628418,
      "step": 980
    },
    {
      "epoch": 0.21336510343333695,
      "grad_norm": 15.125,
      "learning_rate": 1.4486875024554127e-05,
      "logits/chosen": -0.27672213315963745,
      "logits/rejected": -0.2869156301021576,
      "logps/chosen": -233.16152954101562,
      "logps/rejected": -238.7923126220703,
      "loss": 0.6233,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1215320825576782,
      "rewards/margins": 0.28090354800224304,
      "rewards/rejected": -1.402435541152954,
      "step": 985
    },
    {
      "epoch": 0.2144481750243691,
      "grad_norm": 13.6875,
      "learning_rate": 1.4486420309081913e-05,
      "logits/chosen": -0.25480058789253235,
      "logits/rejected": -0.3392016291618347,
      "logps/chosen": -254.06155395507812,
      "logps/rejected": -245.26736450195312,
      "loss": 0.6419,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.1311604976654053,
      "rewards/margins": 0.3942778706550598,
      "rewards/rejected": -1.5254383087158203,
      "step": 990
    },
    {
      "epoch": 0.21553124661540127,
      "grad_norm": 18.375,
      "learning_rate": 1.4485957860195037e-05,
      "logits/chosen": -0.22970561683177948,
      "logits/rejected": -0.1720375120639801,
      "logps/chosen": -250.5047607421875,
      "logps/rejected": -255.9875030517578,
      "loss": 0.6321,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -1.2296007871627808,
      "rewards/margins": 0.291873037815094,
      "rewards/rejected": -1.52147376537323,
      "step": 995
    },
    {
      "epoch": 0.21661431820643345,
      "grad_norm": 15.625,
      "learning_rate": 1.4485487678552854e-05,
      "logits/chosen": -0.23437604308128357,
      "logits/rejected": -0.21203160285949707,
      "logps/chosen": -249.0915985107422,
      "logps/rejected": -254.8578338623047,
      "loss": 0.6694,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -1.1631896495819092,
      "rewards/margins": 0.20001360774040222,
      "rewards/rejected": -1.3632032871246338,
      "step": 1000
    },
    {
      "epoch": 0.21661431820643345,
      "eval_logits/chosen": -0.1723344475030899,
      "eval_logits/rejected": -0.11806318908929825,
      "eval_logps/chosen": -231.26580810546875,
      "eval_logps/rejected": -242.27203369140625,
      "eval_loss": 0.6020339727401733,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -0.8532256484031677,
      "eval_rewards/margins": 0.37751975655555725,
      "eval_rewards/rejected": -1.2307451963424683,
      "eval_runtime": 14.3691,
      "eval_samples_per_second": 13.919,
      "eval_steps_per_second": 13.919,
      "step": 1000
    },
    {
      "epoch": 0.2176973897974656,
      "grad_norm": 16.125,
      "learning_rate": 1.4485009764825748e-05,
      "logits/chosen": -0.1882922351360321,
      "logits/rejected": -0.17127127945423126,
      "logps/chosen": -238.6695098876953,
      "logps/rejected": -252.12777709960938,
      "loss": 0.5896,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.2045838832855225,
      "rewards/margins": 0.4487917423248291,
      "rewards/rejected": -1.6533756256103516,
      "step": 1005
    },
    {
      "epoch": 0.21878046138849777,
      "grad_norm": 14.5625,
      "learning_rate": 1.4484524119695126e-05,
      "logits/chosen": -0.25199565291404724,
      "logits/rejected": -0.2312936782836914,
      "logps/chosen": -229.2948455810547,
      "logps/rejected": -230.24258422851562,
      "loss": 0.5898,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -0.9131423830986023,
      "rewards/margins": 0.43871617317199707,
      "rewards/rejected": -1.3518586158752441,
      "step": 1010
    },
    {
      "epoch": 0.21986353297952996,
      "grad_norm": 13.1875,
      "learning_rate": 1.4484030743853424e-05,
      "logits/chosen": -0.2044476568698883,
      "logits/rejected": -0.15977132320404053,
      "logps/chosen": -238.2078399658203,
      "logps/rejected": -242.61801147460938,
      "loss": 0.6413,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -0.9573381543159485,
      "rewards/margins": 0.31907230615615845,
      "rewards/rejected": -1.2764105796813965,
      "step": 1015
    },
    {
      "epoch": 0.22094660457056212,
      "grad_norm": 15.0625,
      "learning_rate": 1.4483529638004092e-05,
      "logits/chosen": -0.1612534075975418,
      "logits/rejected": -0.12939000129699707,
      "logps/chosen": -244.5234832763672,
      "logps/rejected": -253.8475799560547,
      "loss": 0.5969,
      "rewards/accuracies": 0.6166667342185974,
      "rewards/chosen": -0.8875646591186523,
      "rewards/margins": 0.41719937324523926,
      "rewards/rejected": -1.3047640323638916,
      "step": 1020
    },
    {
      "epoch": 0.22202967616159427,
      "grad_norm": 20.375,
      "learning_rate": 1.4483020802861607e-05,
      "logits/chosen": -0.18010596930980682,
      "logits/rejected": -0.2111106663942337,
      "logps/chosen": -237.61782836914062,
      "logps/rejected": -239.11837768554688,
      "loss": 0.6305,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.3403284549713135,
      "rewards/margins": 0.3049499988555908,
      "rewards/rejected": -1.6452783346176147,
      "step": 1025
    },
    {
      "epoch": 0.22311274775262646,
      "grad_norm": 19.5,
      "learning_rate": 1.4482504239151469e-05,
      "logits/chosen": -0.2099669724702835,
      "logits/rejected": -0.24200935661792755,
      "logps/chosen": -236.0606689453125,
      "logps/rejected": -243.656982421875,
      "loss": 0.6263,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1930135488510132,
      "rewards/margins": 0.3036869466304779,
      "rewards/rejected": -1.4967005252838135,
      "step": 1030
    },
    {
      "epoch": 0.22419581934365862,
      "grad_norm": 17.625,
      "learning_rate": 1.4481979947610189e-05,
      "logits/chosen": -0.2341732680797577,
      "logits/rejected": -0.2135254442691803,
      "logps/chosen": -256.6346130371094,
      "logps/rejected": -259.16180419921875,
      "loss": 0.6527,
      "rewards/accuracies": 0.6249999403953552,
      "rewards/chosen": -1.6436777114868164,
      "rewards/margins": 0.2948886454105377,
      "rewards/rejected": -1.9385662078857422,
      "step": 1035
    },
    {
      "epoch": 0.22527889093469078,
      "grad_norm": 21.25,
      "learning_rate": 1.4481447928985301e-05,
      "logits/chosen": -0.3024800717830658,
      "logits/rejected": -0.2234015166759491,
      "logps/chosen": -254.27490234375,
      "logps/rejected": -269.741943359375,
      "loss": 0.6096,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": -1.257542610168457,
      "rewards/margins": 0.4916294515132904,
      "rewards/rejected": -1.7491719722747803,
      "step": 1040
    },
    {
      "epoch": 0.22636196252572294,
      "grad_norm": 13.375,
      "learning_rate": 1.4480908184035359e-05,
      "logits/chosen": -0.33951348066329956,
      "logits/rejected": -0.3178249001502991,
      "logps/chosen": -240.4952392578125,
      "logps/rejected": -236.1266326904297,
      "loss": 0.6658,
      "rewards/accuracies": 0.5750001072883606,
      "rewards/chosen": -1.2644952535629272,
      "rewards/margins": 0.2347756326198578,
      "rewards/rejected": -1.4992709159851074,
      "step": 1045
    },
    {
      "epoch": 0.22744503411675512,
      "grad_norm": 17.5,
      "learning_rate": 1.4480360713529926e-05,
      "logits/chosen": -0.33027634024620056,
      "logits/rejected": -0.3020370900630951,
      "logps/chosen": -244.1776580810547,
      "logps/rejected": -256.3446960449219,
      "loss": 0.596,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.0025821924209595,
      "rewards/margins": 0.4145803451538086,
      "rewards/rejected": -1.417162299156189,
      "step": 1050
    },
    {
      "epoch": 0.22852810570778728,
      "grad_norm": 13.375,
      "learning_rate": 1.4479805518249589e-05,
      "logits/chosen": -0.20945307612419128,
      "logits/rejected": -0.199072003364563,
      "logps/chosen": -246.078369140625,
      "logps/rejected": -247.022216796875,
      "loss": 0.6173,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -0.9182292819023132,
      "rewards/margins": 0.34281009435653687,
      "rewards/rejected": -1.2610394954681396,
      "step": 1055
    },
    {
      "epoch": 0.22961117729881944,
      "grad_norm": 12.625,
      "learning_rate": 1.4479242598985942e-05,
      "logits/chosen": -0.3017731308937073,
      "logits/rejected": -0.2625275254249573,
      "logps/chosen": -235.1523895263672,
      "logps/rejected": -231.710205078125,
      "loss": 0.5886,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.071424961090088,
      "rewards/margins": 0.4225679934024811,
      "rewards/rejected": -1.4939926862716675,
      "step": 1060
    },
    {
      "epoch": 0.23069424888985163,
      "grad_norm": 19.25,
      "learning_rate": 1.4478671956541593e-05,
      "logits/chosen": -0.26026105880737305,
      "logits/rejected": -0.2672204375267029,
      "logps/chosen": -253.33163452148438,
      "logps/rejected": -259.33612060546875,
      "loss": 0.6866,
      "rewards/accuracies": 0.5500000715255737,
      "rewards/chosen": -1.0433752536773682,
      "rewards/margins": 0.22085706889629364,
      "rewards/rejected": -1.2642322778701782,
      "step": 1065
    },
    {
      "epoch": 0.2317773204808838,
      "grad_norm": 16.0,
      "learning_rate": 1.4478093591730167e-05,
      "logits/chosen": -0.16519834101200104,
      "logits/rejected": -0.11258391290903091,
      "logps/chosen": -254.2841339111328,
      "logps/rejected": -262.8023681640625,
      "loss": 0.6159,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.7516361474990845,
      "rewards/margins": 0.3074316680431366,
      "rewards/rejected": -1.059067726135254,
      "step": 1070
    },
    {
      "epoch": 0.23286039207191594,
      "grad_norm": 17.125,
      "learning_rate": 1.447750750537629e-05,
      "logits/chosen": -0.24409537017345428,
      "logits/rejected": -0.24347174167633057,
      "logps/chosen": -231.4328155517578,
      "logps/rejected": -236.7353973388672,
      "loss": 0.6071,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -0.8504154086112976,
      "rewards/margins": 0.4080711901187897,
      "rewards/rejected": -1.2584866285324097,
      "step": 1075
    },
    {
      "epoch": 0.23394346366294813,
      "grad_norm": 19.75,
      "learning_rate": 1.4476913698315606e-05,
      "logits/chosen": -0.33128422498703003,
      "logits/rejected": -0.33137640357017517,
      "logps/chosen": -240.5367431640625,
      "logps/rejected": -236.8988800048828,
      "loss": 0.6123,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -1.2547733783721924,
      "rewards/margins": 0.29199546575546265,
      "rewards/rejected": -1.5467689037322998,
      "step": 1080
    },
    {
      "epoch": 0.2350265352539803,
      "grad_norm": 21.5,
      "learning_rate": 1.4476312171394762e-05,
      "logits/chosen": -0.3993833661079407,
      "logits/rejected": -0.36872297525405884,
      "logps/chosen": -234.729736328125,
      "logps/rejected": -243.0828399658203,
      "loss": 0.5935,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -0.998491644859314,
      "rewards/margins": 0.3956199586391449,
      "rewards/rejected": -1.3941116333007812,
      "step": 1085
    },
    {
      "epoch": 0.23610960684501245,
      "grad_norm": 20.5,
      "learning_rate": 1.4475702925471417e-05,
      "logits/chosen": -0.32119816541671753,
      "logits/rejected": -0.31232327222824097,
      "logps/chosen": -245.8464813232422,
      "logps/rejected": -262.10003662109375,
      "loss": 0.6435,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -1.1440627574920654,
      "rewards/margins": 0.3457971215248108,
      "rewards/rejected": -1.4898595809936523,
      "step": 1090
    },
    {
      "epoch": 0.23719267843604463,
      "grad_norm": 18.125,
      "learning_rate": 1.447508596141423e-05,
      "logits/chosen": -0.42148056626319885,
      "logits/rejected": -0.378412663936615,
      "logps/chosen": -236.88925170898438,
      "logps/rejected": -232.6885528564453,
      "loss": 0.6276,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.1969094276428223,
      "rewards/margins": 0.28676682710647583,
      "rewards/rejected": -1.4836761951446533,
      "step": 1095
    },
    {
      "epoch": 0.2382757500270768,
      "grad_norm": 14.5,
      "learning_rate": 1.4474461280102864e-05,
      "logits/chosen": -0.3717447519302368,
      "logits/rejected": -0.3840111196041107,
      "logps/chosen": -245.3320770263672,
      "logps/rejected": -245.1776885986328,
      "loss": 0.5221,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.8653903007507324,
      "rewards/margins": 0.6731550097465515,
      "rewards/rejected": -1.5385452508926392,
      "step": 1100
    },
    {
      "epoch": 0.23935882161810895,
      "grad_norm": 19.0,
      "learning_rate": 1.4473828882427996e-05,
      "logits/chosen": -0.4841481149196625,
      "logits/rejected": -0.4946522116661072,
      "logps/chosen": -249.90170288085938,
      "logps/rejected": -261.15582275390625,
      "loss": 0.6298,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.2928674221038818,
      "rewards/margins": 0.3710517883300781,
      "rewards/rejected": -1.663919448852539,
      "step": 1105
    },
    {
      "epoch": 0.2404418932091411,
      "grad_norm": 21.875,
      "learning_rate": 1.4473188769291294e-05,
      "logits/chosen": -0.4904871881008148,
      "logits/rejected": -0.4675755500793457,
      "logps/chosen": -239.60769653320312,
      "logps/rejected": -242.98629760742188,
      "loss": 0.6283,
      "rewards/accuracies": 0.6416667699813843,
      "rewards/chosen": -1.055094599723816,
      "rewards/margins": 0.4389069974422455,
      "rewards/rejected": -1.4940016269683838,
      "step": 1110
    },
    {
      "epoch": 0.2415249648001733,
      "grad_norm": 16.125,
      "learning_rate": 1.4472540941605428e-05,
      "logits/chosen": -0.5172803997993469,
      "logits/rejected": -0.5164453387260437,
      "logps/chosen": -240.3188018798828,
      "logps/rejected": -251.76583862304688,
      "loss": 0.6041,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -1.2359693050384521,
      "rewards/margins": 0.4470987319946289,
      "rewards/rejected": -1.683068037033081,
      "step": 1115
    },
    {
      "epoch": 0.24260803639120546,
      "grad_norm": 22.375,
      "learning_rate": 1.4471885400294071e-05,
      "logits/chosen": -0.5119749307632446,
      "logits/rejected": -0.44268184900283813,
      "logps/chosen": -255.2139434814453,
      "logps/rejected": -266.2033996582031,
      "loss": 0.622,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -1.3650729656219482,
      "rewards/margins": 0.4449044167995453,
      "rewards/rejected": -1.8099775314331055,
      "step": 1120
    },
    {
      "epoch": 0.24369110798223761,
      "grad_norm": 20.5,
      "learning_rate": 1.4471222146291892e-05,
      "logits/chosen": -0.5113047361373901,
      "logits/rejected": -0.4599158763885498,
      "logps/chosen": -251.11474609375,
      "logps/rejected": -254.35073852539062,
      "loss": 0.5725,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.3309074640274048,
      "rewards/margins": 0.5434951782226562,
      "rewards/rejected": -1.874402403831482,
      "step": 1125
    },
    {
      "epoch": 0.2447741795732698,
      "grad_norm": 19.875,
      "learning_rate": 1.4470551180544559e-05,
      "logits/chosen": -0.3730575442314148,
      "logits/rejected": -0.29822057485580444,
      "logps/chosen": -267.28448486328125,
      "logps/rejected": -274.5223388671875,
      "loss": 0.5687,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -0.9145733118057251,
      "rewards/margins": 0.5894616842269897,
      "rewards/rejected": -1.5040351152420044,
      "step": 1130
    },
    {
      "epoch": 0.24585725116430196,
      "grad_norm": 17.75,
      "learning_rate": 1.446987250400873e-05,
      "logits/chosen": -0.3773976266384125,
      "logits/rejected": -0.36462196707725525,
      "logps/chosen": -241.5471649169922,
      "logps/rejected": -244.08364868164062,
      "loss": 0.6156,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1303387880325317,
      "rewards/margins": 0.4140016436576843,
      "rewards/rejected": -1.5443403720855713,
      "step": 1135
    },
    {
      "epoch": 0.24694032275533412,
      "grad_norm": 14.9375,
      "learning_rate": 1.4469186117652067e-05,
      "logits/chosen": -0.3660892844200134,
      "logits/rejected": -0.3163079619407654,
      "logps/chosen": -252.24014282226562,
      "logps/rejected": -255.2248077392578,
      "loss": 0.5887,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -1.1350467205047607,
      "rewards/margins": 0.4643896520137787,
      "rewards/rejected": -1.5994364023208618,
      "step": 1140
    },
    {
      "epoch": 0.2480233943463663,
      "grad_norm": 14.9375,
      "learning_rate": 1.4468492022453209e-05,
      "logits/chosen": -0.3543713688850403,
      "logits/rejected": -0.35839956998825073,
      "logps/chosen": -234.11474609375,
      "logps/rejected": -240.86160278320312,
      "loss": 0.6038,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.2518047094345093,
      "rewards/margins": 0.47415781021118164,
      "rewards/rejected": -1.7259624004364014,
      "step": 1145
    },
    {
      "epoch": 0.24910646593739846,
      "grad_norm": 20.375,
      "learning_rate": 1.4467790219401806e-05,
      "logits/chosen": -0.34249550104141235,
      "logits/rejected": -0.3057665228843689,
      "logps/chosen": -255.42868041992188,
      "logps/rejected": -263.44244384765625,
      "loss": 0.6014,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9333832859992981,
      "rewards/margins": 0.5383116006851196,
      "rewards/rejected": -1.4716947078704834,
      "step": 1150
    },
    {
      "epoch": 0.25018953752843065,
      "grad_norm": 19.0,
      "learning_rate": 1.4467080709498477e-05,
      "logits/chosen": -0.3547367453575134,
      "logits/rejected": -0.3053904175758362,
      "logps/chosen": -257.95599365234375,
      "logps/rejected": -269.3160095214844,
      "loss": 0.6512,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -1.2556036710739136,
      "rewards/margins": 0.27444136142730713,
      "rewards/rejected": -1.5300449132919312,
      "step": 1155
    },
    {
      "epoch": 0.2512726091194628,
      "grad_norm": 17.375,
      "learning_rate": 1.4466363493754848e-05,
      "logits/chosen": -0.3365868926048279,
      "logits/rejected": -0.27202171087265015,
      "logps/chosen": -238.39053344726562,
      "logps/rejected": -248.0595245361328,
      "loss": 0.5793,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.036049246788025,
      "rewards/margins": 0.5190224647521973,
      "rewards/rejected": -1.5550717115402222,
      "step": 1160
    },
    {
      "epoch": 0.25235568071049497,
      "grad_norm": 18.75,
      "learning_rate": 1.4465638573193521e-05,
      "logits/chosen": -0.4079698920249939,
      "logits/rejected": -0.331678569316864,
      "logps/chosen": -243.7015380859375,
      "logps/rejected": -246.6396942138672,
      "loss": 0.5842,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -1.1038620471954346,
      "rewards/margins": 0.4908418655395508,
      "rewards/rejected": -1.5947039127349854,
      "step": 1165
    },
    {
      "epoch": 0.2534387523015271,
      "grad_norm": 16.0,
      "learning_rate": 1.4464905948848083e-05,
      "logits/chosen": -0.3740221858024597,
      "logits/rejected": -0.3716724216938019,
      "logps/chosen": -243.5222625732422,
      "logps/rejected": -234.61453247070312,
      "loss": 0.5911,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -1.221592664718628,
      "rewards/margins": 0.3726115822792053,
      "rewards/rejected": -1.594204306602478,
      "step": 1170
    },
    {
      "epoch": 0.2545218238925593,
      "grad_norm": 19.125,
      "learning_rate": 1.4464165621763112e-05,
      "logits/chosen": -0.4048662781715393,
      "logits/rejected": -0.3977678716182709,
      "logps/chosen": -249.8496551513672,
      "logps/rejected": -258.94439697265625,
      "loss": 0.6166,
      "rewards/accuracies": 0.6083332896232605,
      "rewards/chosen": -1.3169597387313843,
      "rewards/margins": 0.41529184579849243,
      "rewards/rejected": -1.732251524925232,
      "step": 1175
    },
    {
      "epoch": 0.25560489548359144,
      "grad_norm": 20.25,
      "learning_rate": 1.4463417592994166e-05,
      "logits/chosen": -0.49555402994155884,
      "logits/rejected": -0.4768499433994293,
      "logps/chosen": -248.6404266357422,
      "logps/rejected": -243.39492797851562,
      "loss": 0.6016,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3658021688461304,
      "rewards/margins": 0.4190761148929596,
      "rewards/rejected": -1.784878134727478,
      "step": 1180
    },
    {
      "epoch": 0.25668796707462366,
      "grad_norm": 10.75,
      "learning_rate": 1.4462661863607777e-05,
      "logits/chosen": -0.43655967712402344,
      "logits/rejected": -0.3893892765045166,
      "logps/chosen": -255.0498809814453,
      "logps/rejected": -261.12451171875,
      "loss": 0.5843,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -1.320417046546936,
      "rewards/margins": 0.4712238311767578,
      "rewards/rejected": -1.7916408777236938,
      "step": 1185
    },
    {
      "epoch": 0.2577710386656558,
      "grad_norm": 18.125,
      "learning_rate": 1.446189843468147e-05,
      "logits/chosen": -0.3769032955169678,
      "logits/rejected": -0.3995923399925232,
      "logps/chosen": -254.8723602294922,
      "logps/rejected": -267.49151611328125,
      "loss": 0.6263,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.3146369457244873,
      "rewards/margins": 0.3686372637748718,
      "rewards/rejected": -1.683274269104004,
      "step": 1190
    },
    {
      "epoch": 0.258854110256688,
      "grad_norm": 15.5,
      "learning_rate": 1.4461127307303738e-05,
      "logits/chosen": -0.4728514552116394,
      "logits/rejected": -0.438041627407074,
      "logps/chosen": -246.63742065429688,
      "logps/rejected": -252.6478729248047,
      "loss": 0.6318,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.3911083936691284,
      "rewards/margins": 0.3603866696357727,
      "rewards/rejected": -1.751495122909546,
      "step": 1195
    },
    {
      "epoch": 0.25993718184772013,
      "grad_norm": 14.4375,
      "learning_rate": 1.446034848257405e-05,
      "logits/chosen": -0.46418166160583496,
      "logits/rejected": -0.3781721889972687,
      "logps/chosen": -238.732177734375,
      "logps/rejected": -254.7250518798828,
      "loss": 0.5212,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -1.0931901931762695,
      "rewards/margins": 0.691379189491272,
      "rewards/rejected": -1.7845691442489624,
      "step": 1200
    },
    {
      "epoch": 0.2610202534387523,
      "grad_norm": 15.75,
      "learning_rate": 1.4459561961602854e-05,
      "logits/chosen": -0.46494239568710327,
      "logits/rejected": -0.43728137016296387,
      "logps/chosen": -257.2491760253906,
      "logps/rejected": -264.63189697265625,
      "loss": 0.5282,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.2677032947540283,
      "rewards/margins": 0.6639944911003113,
      "rewards/rejected": -1.9316980838775635,
      "step": 1205
    },
    {
      "epoch": 0.26210332502978445,
      "grad_norm": 17.0,
      "learning_rate": 1.445876774551157e-05,
      "logits/chosen": -0.4016745984554291,
      "logits/rejected": -0.3860119581222534,
      "logps/chosen": -260.63360595703125,
      "logps/rejected": -266.27398681640625,
      "loss": 0.5642,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.490104079246521,
      "rewards/margins": 0.5680019855499268,
      "rewards/rejected": -2.0581061840057373,
      "step": 1210
    },
    {
      "epoch": 0.2631863966208166,
      "grad_norm": 20.875,
      "learning_rate": 1.4457965835432591e-05,
      "logits/chosen": -0.44888386130332947,
      "logits/rejected": -0.4794653058052063,
      "logps/chosen": -263.62176513671875,
      "logps/rejected": -253.90829467773438,
      "loss": 0.6147,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.3366336822509766,
      "rewards/margins": 0.46011966466903687,
      "rewards/rejected": -1.796753168106079,
      "step": 1215
    },
    {
      "epoch": 0.2642694682118488,
      "grad_norm": 16.75,
      "learning_rate": 1.4457156232509278e-05,
      "logits/chosen": -0.3686813712120056,
      "logits/rejected": -0.41304463148117065,
      "logps/chosen": -255.06350708007812,
      "logps/rejected": -251.86087036132812,
      "loss": 0.6832,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -1.3440033197402954,
      "rewards/margins": 0.2563704550266266,
      "rewards/rejected": -1.60037362575531,
      "step": 1220
    },
    {
      "epoch": 0.265352539802881,
      "grad_norm": 14.9375,
      "learning_rate": 1.4456338937895962e-05,
      "logits/chosen": -0.5161653757095337,
      "logits/rejected": -0.5035422444343567,
      "logps/chosen": -245.6992645263672,
      "logps/rejected": -249.9958953857422,
      "loss": 0.6143,
      "rewards/accuracies": 0.5916666388511658,
      "rewards/chosen": -1.198022484779358,
      "rewards/margins": 0.4806770384311676,
      "rewards/rejected": -1.6786997318267822,
      "step": 1225
    },
    {
      "epoch": 0.26643561139391314,
      "grad_norm": 14.0625,
      "learning_rate": 1.4455513952757937e-05,
      "logits/chosen": -0.49110642075538635,
      "logits/rejected": -0.46951571106910706,
      "logps/chosen": -224.458251953125,
      "logps/rejected": -238.232421875,
      "loss": 0.5505,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.1941207647323608,
      "rewards/margins": 0.6065016984939575,
      "rewards/rejected": -1.800622582435608,
      "step": 1230
    },
    {
      "epoch": 0.2675186829849453,
      "grad_norm": 16.75,
      "learning_rate": 1.4454681278271469e-05,
      "logits/chosen": -0.5082752108573914,
      "logits/rejected": -0.4804894030094147,
      "logps/chosen": -242.7399444580078,
      "logps/rejected": -240.49978637695312,
      "loss": 0.5768,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.0470778942108154,
      "rewards/margins": 0.5367264151573181,
      "rewards/rejected": -1.5838042497634888,
      "step": 1235
    },
    {
      "epoch": 0.26860175457597746,
      "grad_norm": 16.25,
      "learning_rate": 1.4453840915623783e-05,
      "logits/chosen": -0.46206218004226685,
      "logits/rejected": -0.47607356309890747,
      "logps/chosen": -249.61141967773438,
      "logps/rejected": -251.66091918945312,
      "loss": 0.5915,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9758882522583008,
      "rewards/margins": 0.4817824959754944,
      "rewards/rejected": -1.4576706886291504,
      "step": 1240
    },
    {
      "epoch": 0.2696848261670096,
      "grad_norm": 18.375,
      "learning_rate": 1.4452992866013062e-05,
      "logits/chosen": -0.4872070848941803,
      "logits/rejected": -0.5097359418869019,
      "logps/chosen": -231.3965606689453,
      "logps/rejected": -237.9442138671875,
      "loss": 0.5694,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -0.9674919247627258,
      "rewards/margins": 0.5408203601837158,
      "rewards/rejected": -1.508312463760376,
      "step": 1245
    },
    {
      "epoch": 0.27076789775804183,
      "grad_norm": 16.625,
      "learning_rate": 1.4452137130648459e-05,
      "logits/chosen": -0.5078374743461609,
      "logits/rejected": -0.5280264616012573,
      "logps/chosen": -245.2581329345703,
      "logps/rejected": -243.20297241210938,
      "loss": 0.601,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.286068081855774,
      "rewards/margins": 0.42304572463035583,
      "rewards/rejected": -1.7091137170791626,
      "step": 1250
    },
    {
      "epoch": 0.271850969349074,
      "grad_norm": 14.125,
      "learning_rate": 1.4451273710750077e-05,
      "logits/chosen": -0.6155784130096436,
      "logits/rejected": -0.47567301988601685,
      "logps/chosen": -237.3800506591797,
      "logps/rejected": -255.06021118164062,
      "loss": 0.5648,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2842419147491455,
      "rewards/margins": 0.637798011302948,
      "rewards/rejected": -1.9220399856567383,
      "step": 1255
    },
    {
      "epoch": 0.27293404094010615,
      "grad_norm": 16.0,
      "learning_rate": 1.445040260754898e-05,
      "logits/chosen": -0.5020845532417297,
      "logits/rejected": -0.5234459042549133,
      "logps/chosen": -254.8231201171875,
      "logps/rejected": -257.53466796875,
      "loss": 0.5862,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.4333362579345703,
      "rewards/margins": 0.5947860479354858,
      "rewards/rejected": -2.0281224250793457,
      "step": 1260
    },
    {
      "epoch": 0.2740171125311383,
      "grad_norm": 18.125,
      "learning_rate": 1.4449523822287184e-05,
      "logits/chosen": -0.5211073756217957,
      "logits/rejected": -0.500187873840332,
      "logps/chosen": -263.5560302734375,
      "logps/rejected": -252.15542602539062,
      "loss": 0.6195,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3445098400115967,
      "rewards/margins": 0.5118342638015747,
      "rewards/rejected": -1.85634446144104,
      "step": 1265
    },
    {
      "epoch": 0.27510018412217047,
      "grad_norm": 15.3125,
      "learning_rate": 1.4448637356217657e-05,
      "logits/chosen": -0.5691152215003967,
      "logits/rejected": -0.5512626767158508,
      "logps/chosen": -262.17364501953125,
      "logps/rejected": -275.0877685546875,
      "loss": 0.5463,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3156284093856812,
      "rewards/margins": 0.5637227296829224,
      "rewards/rejected": -1.879351258277893,
      "step": 1270
    },
    {
      "epoch": 0.2761832557132026,
      "grad_norm": 19.25,
      "learning_rate": 1.4447743210604324e-05,
      "logits/chosen": -0.5546935200691223,
      "logits/rejected": -0.5743864178657532,
      "logps/chosen": -260.68121337890625,
      "logps/rejected": -261.41455078125,
      "loss": 0.6159,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -1.622615098953247,
      "rewards/margins": 0.3839835226535797,
      "rewards/rejected": -2.006598711013794,
      "step": 1275
    },
    {
      "epoch": 0.2772663273042348,
      "grad_norm": 16.0,
      "learning_rate": 1.4446841386722054e-05,
      "logits/chosen": -0.507060706615448,
      "logits/rejected": -0.4705331325531006,
      "logps/chosen": -244.816650390625,
      "logps/rejected": -254.683837890625,
      "loss": 0.6511,
      "rewards/accuracies": 0.5833333730697632,
      "rewards/chosen": -1.4318010807037354,
      "rewards/margins": 0.2951648533344269,
      "rewards/rejected": -1.7269656658172607,
      "step": 1280
    },
    {
      "epoch": 0.278349398895267,
      "grad_norm": 17.875,
      "learning_rate": 1.4445931885856668e-05,
      "logits/chosen": -0.6143358945846558,
      "logits/rejected": -0.5769041180610657,
      "logps/chosen": -232.3941192626953,
      "logps/rejected": -244.92691040039062,
      "loss": 0.6028,
      "rewards/accuracies": 0.6749999523162842,
      "rewards/chosen": -1.1580665111541748,
      "rewards/margins": 0.5100372433662415,
      "rewards/rejected": -1.668103814125061,
      "step": 1285
    },
    {
      "epoch": 0.27943247048629916,
      "grad_norm": 16.375,
      "learning_rate": 1.4445014709304929e-05,
      "logits/chosen": -0.4089280068874359,
      "logits/rejected": -0.3986991345882416,
      "logps/chosen": -254.57095336914062,
      "logps/rejected": -251.0946807861328,
      "loss": 0.5959,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.0377693176269531,
      "rewards/margins": 0.48036742210388184,
      "rewards/rejected": -1.518136739730835,
      "step": 1290
    },
    {
      "epoch": 0.2805155420773313,
      "grad_norm": 15.8125,
      "learning_rate": 1.4444089858374545e-05,
      "logits/chosen": -0.5114402174949646,
      "logits/rejected": -0.4272480607032776,
      "logps/chosen": -240.0308380126953,
      "logps/rejected": -248.14517211914062,
      "loss": 0.5419,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.1364825963974,
      "rewards/margins": 0.5807336568832397,
      "rewards/rejected": -1.7172162532806396,
      "step": 1295
    },
    {
      "epoch": 0.2815986136683635,
      "grad_norm": 12.4375,
      "learning_rate": 1.4443157334384167e-05,
      "logits/chosen": -0.5158824920654297,
      "logits/rejected": -0.4748362600803375,
      "logps/chosen": -224.3240203857422,
      "logps/rejected": -251.85812377929688,
      "loss": 0.5836,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -1.1857417821884155,
      "rewards/margins": 0.4920528829097748,
      "rewards/rejected": -1.6777946949005127,
      "step": 1300
    },
    {
      "epoch": 0.28268168525939563,
      "grad_norm": 15.5625,
      "learning_rate": 1.4442217138663388e-05,
      "logits/chosen": -0.5246096849441528,
      "logits/rejected": -0.4979622960090637,
      "logps/chosen": -238.0264129638672,
      "logps/rejected": -248.31637573242188,
      "loss": 0.6107,
      "rewards/accuracies": 0.6166667342185974,
      "rewards/chosen": -1.2169640064239502,
      "rewards/margins": 0.4694593846797943,
      "rewards/rejected": -1.6864233016967773,
      "step": 1305
    },
    {
      "epoch": 0.2837647568504278,
      "grad_norm": 13.875,
      "learning_rate": 1.4441269272552733e-05,
      "logits/chosen": -0.6010515689849854,
      "logits/rejected": -0.5610944032669067,
      "logps/chosen": -256.004150390625,
      "logps/rejected": -255.4950714111328,
      "loss": 0.6034,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.366471529006958,
      "rewards/margins": 0.4509966969490051,
      "rewards/rejected": -1.817468285560608,
      "step": 1310
    },
    {
      "epoch": 0.28484782844146,
      "grad_norm": 18.25,
      "learning_rate": 1.444031373740367e-05,
      "logits/chosen": -0.5681723356246948,
      "logits/rejected": -0.5273127555847168,
      "logps/chosen": -240.03396606445312,
      "logps/rejected": -254.19570922851562,
      "loss": 0.5862,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -1.2317006587982178,
      "rewards/margins": 0.5569983720779419,
      "rewards/rejected": -1.7886989116668701,
      "step": 1315
    },
    {
      "epoch": 0.28593090003249216,
      "grad_norm": 17.5,
      "learning_rate": 1.44393505345786e-05,
      "logits/chosen": -0.47849661111831665,
      "logits/rejected": -0.50385582447052,
      "logps/chosen": -261.42584228515625,
      "logps/rejected": -264.90777587890625,
      "loss": 0.6623,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.1721395254135132,
      "rewards/margins": 0.3402039110660553,
      "rewards/rejected": -1.5123435258865356,
      "step": 1320
    },
    {
      "epoch": 0.2870139716235243,
      "grad_norm": 15.4375,
      "learning_rate": 1.443837966545085e-05,
      "logits/chosen": -0.6166875958442688,
      "logits/rejected": -0.6256123185157776,
      "logps/chosen": -231.16232299804688,
      "logps/rejected": -245.72677612304688,
      "loss": 0.5608,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1487951278686523,
      "rewards/margins": 0.5118417143821716,
      "rewards/rejected": -1.6606369018554688,
      "step": 1325
    },
    {
      "epoch": 0.2880970432145565,
      "grad_norm": 17.0,
      "learning_rate": 1.4437401131404693e-05,
      "logits/chosen": -0.6968683004379272,
      "logits/rejected": -0.6488800048828125,
      "logps/chosen": -247.2900848388672,
      "logps/rejected": -255.0655517578125,
      "loss": 0.6191,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -1.0748927593231201,
      "rewards/margins": 0.4207105040550232,
      "rewards/rejected": -1.495603322982788,
      "step": 1330
    },
    {
      "epoch": 0.28918011480558864,
      "grad_norm": 16.0,
      "learning_rate": 1.4436414933835314e-05,
      "logits/chosen": -0.5430472493171692,
      "logits/rejected": -0.5298408269882202,
      "logps/chosen": -248.35055541992188,
      "logps/rejected": -245.87451171875,
      "loss": 0.5819,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.8152984380722046,
      "rewards/margins": 0.5555189847946167,
      "rewards/rejected": -1.3708174228668213,
      "step": 1335
    },
    {
      "epoch": 0.2902631863966208,
      "grad_norm": 13.5,
      "learning_rate": 1.4435421074148832e-05,
      "logits/chosen": -0.5704728364944458,
      "logits/rejected": -0.516574501991272,
      "logps/chosen": -238.6221466064453,
      "logps/rejected": -249.9164276123047,
      "loss": 0.5315,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.813635528087616,
      "rewards/margins": 0.9009323120117188,
      "rewards/rejected": -1.71456778049469,
      "step": 1340
    },
    {
      "epoch": 0.29134625798765296,
      "grad_norm": 16.5,
      "learning_rate": 1.4434419553762293e-05,
      "logits/chosen": -0.5784732103347778,
      "logits/rejected": -0.5957360863685608,
      "logps/chosen": -254.53103637695312,
      "logps/rejected": -254.6358184814453,
      "loss": 0.6012,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.6741352081298828,
      "rewards/margins": 0.424616277217865,
      "rewards/rejected": -2.0987515449523926,
      "step": 1345
    },
    {
      "epoch": 0.29242932957868517,
      "grad_norm": 16.375,
      "learning_rate": 1.4433410374103659e-05,
      "logits/chosen": -0.5991619825363159,
      "logits/rejected": -0.6263461112976074,
      "logps/chosen": -246.0270538330078,
      "logps/rejected": -242.42758178710938,
      "loss": 0.5493,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.6526308059692383,
      "rewards/margins": 0.5500800609588623,
      "rewards/rejected": -2.2027108669281006,
      "step": 1350
    },
    {
      "epoch": 0.29351240116971733,
      "grad_norm": 15.1875,
      "learning_rate": 1.443239353661182e-05,
      "logits/chosen": -0.5655080080032349,
      "logits/rejected": -0.5632392168045044,
      "logps/chosen": -240.6619110107422,
      "logps/rejected": -261.197265625,
      "loss": 0.5572,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.4692254066467285,
      "rewards/margins": 0.627414345741272,
      "rewards/rejected": -2.09663987159729,
      "step": 1355
    },
    {
      "epoch": 0.2945954727607495,
      "grad_norm": 16.375,
      "learning_rate": 1.4431369042736579e-05,
      "logits/chosen": -0.46668070554733276,
      "logits/rejected": -0.45769065618515015,
      "logps/chosen": -256.13128662109375,
      "logps/rejected": -262.73956298828125,
      "loss": 0.6194,
      "rewards/accuracies": 0.6166667342185974,
      "rewards/chosen": -1.43289315700531,
      "rewards/margins": 0.4682067036628723,
      "rewards/rejected": -1.9010998010635376,
      "step": 1360
    },
    {
      "epoch": 0.29567854435178165,
      "grad_norm": 15.9375,
      "learning_rate": 1.4430336893938655e-05,
      "logits/chosen": -0.5955272912979126,
      "logits/rejected": -0.560431957244873,
      "logps/chosen": -241.2655029296875,
      "logps/rejected": -249.26168823242188,
      "loss": 0.586,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.0885109901428223,
      "rewards/margins": 0.5934677720069885,
      "rewards/rejected": -1.6819788217544556,
      "step": 1365
    },
    {
      "epoch": 0.2967616159428138,
      "grad_norm": 16.125,
      "learning_rate": 1.4429297091689688e-05,
      "logits/chosen": -0.6510792970657349,
      "logits/rejected": -0.6872649788856506,
      "logps/chosen": -236.6271514892578,
      "logps/rejected": -245.9168243408203,
      "loss": 0.5805,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.7641708850860596,
      "rewards/margins": 0.48506030440330505,
      "rewards/rejected": -2.2492308616638184,
      "step": 1370
    },
    {
      "epoch": 0.29784468753384596,
      "grad_norm": 15.25,
      "learning_rate": 1.4428249637472227e-05,
      "logits/chosen": -0.5923226475715637,
      "logits/rejected": -0.565222442150116,
      "logps/chosen": -252.11758422851562,
      "logps/rejected": -251.66586303710938,
      "loss": 0.6884,
      "rewards/accuracies": 0.5583333373069763,
      "rewards/chosen": -1.8444595336914062,
      "rewards/margins": 0.1747516244649887,
      "rewards/rejected": -2.0192112922668457,
      "step": 1375
    },
    {
      "epoch": 0.2989277591248782,
      "grad_norm": 14.875,
      "learning_rate": 1.4427194532779722e-05,
      "logits/chosen": -0.6984888315200806,
      "logits/rejected": -0.629934549331665,
      "logps/chosen": -264.48345947265625,
      "logps/rejected": -272.3121337890625,
      "loss": 0.5859,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.6355692148208618,
      "rewards/margins": 0.44280409812927246,
      "rewards/rejected": -2.078373432159424,
      "step": 1380
    },
    {
      "epoch": 0.30001083071591034,
      "grad_norm": 15.75,
      "learning_rate": 1.4426131779116548e-05,
      "logits/chosen": -0.7119425535202026,
      "logits/rejected": -0.6713353991508484,
      "logps/chosen": -250.349853515625,
      "logps/rejected": -256.99908447265625,
      "loss": 0.5992,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.510819673538208,
      "rewards/margins": 0.49119892716407776,
      "rewards/rejected": -2.002018451690674,
      "step": 1385
    },
    {
      "epoch": 0.3010939023069425,
      "grad_norm": 16.125,
      "learning_rate": 1.4425061377997972e-05,
      "logits/chosen": -0.7213162183761597,
      "logits/rejected": -0.6450712084770203,
      "logps/chosen": -241.02084350585938,
      "logps/rejected": -244.45938110351562,
      "loss": 0.6184,
      "rewards/accuracies": 0.6083333492279053,
      "rewards/chosen": -1.5106629133224487,
      "rewards/margins": 0.35203322768211365,
      "rewards/rejected": -1.8626960515975952,
      "step": 1390
    },
    {
      "epoch": 0.30217697389797465,
      "grad_norm": 15.75,
      "learning_rate": 1.4423983330950172e-05,
      "logits/chosen": -0.6954799890518188,
      "logits/rejected": -0.5659809112548828,
      "logps/chosen": -241.64822387695312,
      "logps/rejected": -245.6189727783203,
      "loss": 0.6446,
      "rewards/accuracies": 0.6166666746139526,
      "rewards/chosen": -1.2866425514221191,
      "rewards/margins": 0.3782268166542053,
      "rewards/rejected": -1.6648693084716797,
      "step": 1395
    },
    {
      "epoch": 0.3032600454890068,
      "grad_norm": 16.0,
      "learning_rate": 1.4422897639510223e-05,
      "logits/chosen": -0.598487377166748,
      "logits/rejected": -0.6343046426773071,
      "logps/chosen": -248.1034698486328,
      "logps/rejected": -251.57968139648438,
      "loss": 0.5699,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.3525736331939697,
      "rewards/margins": 0.5582513809204102,
      "rewards/rejected": -1.9108250141143799,
      "step": 1400
    },
    {
      "epoch": 0.30434311708003897,
      "grad_norm": 15.6875,
      "learning_rate": 1.4421804305226106e-05,
      "logits/chosen": -0.4892473816871643,
      "logits/rejected": -0.5243555903434753,
      "logps/chosen": -259.87677001953125,
      "logps/rejected": -255.03353881835938,
      "loss": 0.6368,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3752844333648682,
      "rewards/margins": 0.35168778896331787,
      "rewards/rejected": -1.726972222328186,
      "step": 1405
    },
    {
      "epoch": 0.30542618867107113,
      "grad_norm": 11.0625,
      "learning_rate": 1.4420703329656689e-05,
      "logits/chosen": -0.5579140782356262,
      "logits/rejected": -0.4707837998867035,
      "logps/chosen": -246.27001953125,
      "logps/rejected": -263.4816589355469,
      "loss": 0.5974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1487230062484741,
      "rewards/margins": 0.6172156929969788,
      "rewards/rejected": -1.7659389972686768,
      "step": 1410
    },
    {
      "epoch": 0.30650926026210334,
      "grad_norm": 12.9375,
      "learning_rate": 1.4419594714371745e-05,
      "logits/chosen": -0.5933759808540344,
      "logits/rejected": -0.6154261827468872,
      "logps/chosen": -247.9116668701172,
      "logps/rejected": -236.80990600585938,
      "loss": 0.55,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -1.0102132558822632,
      "rewards/margins": 0.5321445465087891,
      "rewards/rejected": -1.5423579216003418,
      "step": 1415
    },
    {
      "epoch": 0.3075923318531355,
      "grad_norm": 13.25,
      "learning_rate": 1.4418478460951934e-05,
      "logits/chosen": -0.5502112507820129,
      "logits/rejected": -0.5615054965019226,
      "logps/chosen": -250.1300506591797,
      "logps/rejected": -257.5440979003906,
      "loss": 0.5751,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.3259202241897583,
      "rewards/margins": 0.4574640393257141,
      "rewards/rejected": -1.783384084701538,
      "step": 1420
    },
    {
      "epoch": 0.30867540344416766,
      "grad_norm": 15.3125,
      "learning_rate": 1.4417354570988807e-05,
      "logits/chosen": -0.6957350969314575,
      "logits/rejected": -0.6432775259017944,
      "logps/chosen": -244.6309814453125,
      "logps/rejected": -239.6031036376953,
      "loss": 0.5714,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -1.1991641521453857,
      "rewards/margins": 0.6053909063339233,
      "rewards/rejected": -1.8045551776885986,
      "step": 1425
    },
    {
      "epoch": 0.3097584750351998,
      "grad_norm": 13.6875,
      "learning_rate": 1.4416223046084807e-05,
      "logits/chosen": -0.5636551380157471,
      "logits/rejected": -0.5427783131599426,
      "logps/chosen": -241.48757934570312,
      "logps/rejected": -246.3280792236328,
      "loss": 0.5744,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.0983613729476929,
      "rewards/margins": 0.46347302198410034,
      "rewards/rejected": -1.5618343353271484,
      "step": 1430
    },
    {
      "epoch": 0.310841546626232,
      "grad_norm": 17.875,
      "learning_rate": 1.4415083887853256e-05,
      "logits/chosen": -0.5777626633644104,
      "logits/rejected": -0.6474701166152954,
      "logps/chosen": -244.59469604492188,
      "logps/rejected": -245.38345336914062,
      "loss": 0.5631,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.267704963684082,
      "rewards/margins": 0.5692669153213501,
      "rewards/rejected": -1.836971640586853,
      "step": 1435
    },
    {
      "epoch": 0.31192461821726414,
      "grad_norm": 15.5625,
      "learning_rate": 1.4413937097918368e-05,
      "logits/chosen": -0.592932939529419,
      "logits/rejected": -0.565041184425354,
      "logps/chosen": -233.88589477539062,
      "logps/rejected": -244.78201293945312,
      "loss": 0.627,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.0634102821350098,
      "rewards/margins": 0.312079519033432,
      "rewards/rejected": -1.3754899501800537,
      "step": 1440
    },
    {
      "epoch": 0.31300768980829635,
      "grad_norm": 17.875,
      "learning_rate": 1.441278267791523e-05,
      "logits/chosen": -0.6233593225479126,
      "logits/rejected": -0.5936845541000366,
      "logps/chosen": -245.8883819580078,
      "logps/rejected": -254.26809692382812,
      "loss": 0.6019,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0832197666168213,
      "rewards/margins": 0.4697085916996002,
      "rewards/rejected": -1.5529283285140991,
      "step": 1445
    },
    {
      "epoch": 0.3140907613993285,
      "grad_norm": 15.6875,
      "learning_rate": 1.4411620629489813e-05,
      "logits/chosen": -0.6749738454818726,
      "logits/rejected": -0.6023078560829163,
      "logps/chosen": -227.64859008789062,
      "logps/rejected": -236.81454467773438,
      "loss": 0.593,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.947696328163147,
      "rewards/margins": 0.5259412527084351,
      "rewards/rejected": -1.4736374616622925,
      "step": 1450
    },
    {
      "epoch": 0.31517383299036067,
      "grad_norm": 14.4375,
      "learning_rate": 1.4410450954298964e-05,
      "logits/chosen": -0.7580444812774658,
      "logits/rejected": -0.692359447479248,
      "logps/chosen": -240.12060546875,
      "logps/rejected": -239.4538116455078,
      "loss": 0.6284,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -0.8310526609420776,
      "rewards/margins": 0.43997153639793396,
      "rewards/rejected": -1.271024227142334,
      "step": 1455
    },
    {
      "epoch": 0.31625690458139283,
      "grad_norm": 15.0,
      "learning_rate": 1.44092736540104e-05,
      "logits/chosen": -0.606439471244812,
      "logits/rejected": -0.627894401550293,
      "logps/chosen": -240.01583862304688,
      "logps/rejected": -257.87799072265625,
      "loss": 0.5849,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.5840397477149963,
      "rewards/margins": 0.5615087747573853,
      "rewards/rejected": -1.1455485820770264,
      "step": 1460
    },
    {
      "epoch": 0.317339976172425,
      "grad_norm": 19.25,
      "learning_rate": 1.4408088730302719e-05,
      "logits/chosen": -0.6168618202209473,
      "logits/rejected": -0.63998943567276,
      "logps/chosen": -230.70272827148438,
      "logps/rejected": -244.4351348876953,
      "loss": 0.5682,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -0.5898928046226501,
      "rewards/margins": 0.620522141456604,
      "rewards/rejected": -1.2104148864746094,
      "step": 1465
    },
    {
      "epoch": 0.31842304776345715,
      "grad_norm": 17.125,
      "learning_rate": 1.440689618486538e-05,
      "logits/chosen": -0.699774444103241,
      "logits/rejected": -0.725190281867981,
      "logps/chosen": -238.32522583007812,
      "logps/rejected": -235.05807495117188,
      "loss": 0.5392,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5573367476463318,
      "rewards/margins": 0.5908375382423401,
      "rewards/rejected": -1.1481744050979614,
      "step": 1470
    },
    {
      "epoch": 0.3195061193544893,
      "grad_norm": 13.0,
      "learning_rate": 1.4405696019398712e-05,
      "logits/chosen": -0.7366136312484741,
      "logits/rejected": -0.7358259558677673,
      "logps/chosen": -228.74075317382812,
      "logps/rejected": -234.7905731201172,
      "loss": 0.5507,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -0.5542409420013428,
      "rewards/margins": 0.6732238531112671,
      "rewards/rejected": -1.2274647951126099,
      "step": 1475
    },
    {
      "epoch": 0.3205891909455215,
      "grad_norm": 15.875,
      "learning_rate": 1.4404488235613908e-05,
      "logits/chosen": -0.7639018297195435,
      "logits/rejected": -0.6668921709060669,
      "logps/chosen": -246.5464630126953,
      "logps/rejected": -256.819091796875,
      "loss": 0.5536,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -1.0976835489273071,
      "rewards/margins": 0.6624487042427063,
      "rewards/rejected": -1.760132074356079,
      "step": 1480
    },
    {
      "epoch": 0.3216722625365537,
      "grad_norm": 14.875,
      "learning_rate": 1.4403272835233026e-05,
      "logits/chosen": -0.7574801445007324,
      "logits/rejected": -0.671824038028717,
      "logps/chosen": -240.0860595703125,
      "logps/rejected": -263.8565673828125,
      "loss": 0.5053,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -0.9391301870346069,
      "rewards/margins": 0.7307391166687012,
      "rewards/rejected": -1.6698691844940186,
      "step": 1485
    },
    {
      "epoch": 0.32275533412758584,
      "grad_norm": 20.75,
      "learning_rate": 1.4402049819988977e-05,
      "logits/chosen": -0.789374053478241,
      "logits/rejected": -0.8069036602973938,
      "logps/chosen": -253.9874267578125,
      "logps/rejected": -261.1774597167969,
      "loss": 0.6245,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -1.1837316751480103,
      "rewards/margins": 0.47176846861839294,
      "rewards/rejected": -1.655500054359436,
      "step": 1490
    },
    {
      "epoch": 0.323838405718618,
      "grad_norm": 21.625,
      "learning_rate": 1.4400819191625536e-05,
      "logits/chosen": -0.7575956583023071,
      "logits/rejected": -0.7921116352081299,
      "logps/chosen": -261.2274475097656,
      "logps/rejected": -275.56524658203125,
      "loss": 0.5658,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.0369621515274048,
      "rewards/margins": 0.5858994722366333,
      "rewards/rejected": -1.6228615045547485,
      "step": 1495
    },
    {
      "epoch": 0.32492147730965015,
      "grad_norm": 16.125,
      "learning_rate": 1.4399580951897334e-05,
      "logits/chosen": -0.8490041494369507,
      "logits/rejected": -0.8307839632034302,
      "logps/chosen": -251.42727661132812,
      "logps/rejected": -263.2121887207031,
      "loss": 0.5898,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -1.2656245231628418,
      "rewards/margins": 0.5278169512748718,
      "rewards/rejected": -1.7934414148330688,
      "step": 1500
    },
    {
      "epoch": 0.32492147730965015,
      "eval_logits/chosen": -0.7698143720626831,
      "eval_logits/rejected": -0.7063764333724976,
      "eval_logps/chosen": -236.15750122070312,
      "eval_logps/rejected": -251.66665649414062,
      "eval_loss": 0.5790660977363586,
      "eval_rewards/accuracies": 0.6650000214576721,
      "eval_rewards/chosen": -1.0978119373321533,
      "eval_rewards/margins": 0.6026651263237,
      "eval_rewards/rejected": -1.7004770040512085,
      "eval_runtime": 14.3426,
      "eval_samples_per_second": 13.944,
      "eval_steps_per_second": 13.944,
      "step": 1500
    },
    {
      "epoch": 0.3260045489006823,
      "grad_norm": 15.9375,
      "learning_rate": 1.4398335102569843e-05,
      "logits/chosen": -0.7808436155319214,
      "logits/rejected": -0.803728461265564,
      "logps/chosen": -251.05154418945312,
      "logps/rejected": -249.83749389648438,
      "loss": 0.5394,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -1.4117839336395264,
      "rewards/margins": 0.6640549898147583,
      "rewards/rejected": -2.075839042663574,
      "step": 1505
    },
    {
      "epoch": 0.3270876204917145,
      "grad_norm": 15.0625,
      "learning_rate": 1.4397081645419402e-05,
      "logits/chosen": -0.9379908442497253,
      "logits/rejected": -0.8321506381034851,
      "logps/chosen": -240.9103546142578,
      "logps/rejected": -242.66738891601562,
      "loss": 0.5725,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -1.7034637928009033,
      "rewards/margins": 0.565796971321106,
      "rewards/rejected": -2.2692606449127197,
      "step": 1510
    },
    {
      "epoch": 0.3281706920827467,
      "grad_norm": 14.625,
      "learning_rate": 1.4395820582233179e-05,
      "logits/chosen": -0.7179431319236755,
      "logits/rejected": -0.6575448513031006,
      "logps/chosen": -241.9539794921875,
      "logps/rejected": -262.21392822265625,
      "loss": 0.58,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -1.1376620531082153,
      "rewards/margins": 0.5075725317001343,
      "rewards/rejected": -1.6452343463897705,
      "step": 1515
    },
    {
      "epoch": 0.32925376367377884,
      "grad_norm": 16.875,
      "learning_rate": 1.43945519148092e-05,
      "logits/chosen": -0.826134204864502,
      "logits/rejected": -0.8400985598564148,
      "logps/chosen": -241.2794952392578,
      "logps/rejected": -249.81295776367188,
      "loss": 0.5815,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -1.47860848903656,
      "rewards/margins": 0.5351995229721069,
      "rewards/rejected": -2.013808012008667,
      "step": 1520
    },
    {
      "epoch": 0.330336835264811,
      "grad_norm": 16.75,
      "learning_rate": 1.4393275644956327e-05,
      "logits/chosen": -0.8826882243156433,
      "logits/rejected": -0.8853017687797546,
      "logps/chosen": -247.3861083984375,
      "logps/rejected": -248.76010131835938,
      "loss": 0.5836,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.6095476150512695,
      "rewards/margins": 0.4934720993041992,
      "rewards/rejected": -2.1030197143554688,
      "step": 1525
    },
    {
      "epoch": 0.33141990685584316,
      "grad_norm": 13.9375,
      "learning_rate": 1.4391991774494265e-05,
      "logits/chosen": -0.8769704699516296,
      "logits/rejected": -0.8219575881958008,
      "logps/chosen": -246.958251953125,
      "logps/rejected": -265.528076171875,
      "loss": 0.5308,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.5736918449401855,
      "rewards/margins": 0.756668746471405,
      "rewards/rejected": -2.3303606510162354,
      "step": 1530
    },
    {
      "epoch": 0.3325029784468753,
      "grad_norm": 12.875,
      "learning_rate": 1.4390700305253548e-05,
      "logits/chosen": -0.8962342143058777,
      "logits/rejected": -0.867231011390686,
      "logps/chosen": -243.0257568359375,
      "logps/rejected": -241.2760467529297,
      "loss": 0.5705,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -1.2071378231048584,
      "rewards/margins": 0.5363125801086426,
      "rewards/rejected": -1.743450403213501,
      "step": 1535
    },
    {
      "epoch": 0.33358605003790753,
      "grad_norm": 16.125,
      "learning_rate": 1.438940123907556e-05,
      "logits/chosen": -0.7648013234138489,
      "logits/rejected": -0.7742846608161926,
      "logps/chosen": -251.01351928710938,
      "logps/rejected": -266.6339111328125,
      "loss": 0.6188,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.8583425283432007,
      "rewards/margins": 0.5878626704216003,
      "rewards/rejected": -2.4462053775787354,
      "step": 1540
    },
    {
      "epoch": 0.3346691216289397,
      "grad_norm": 13.25,
      "learning_rate": 1.4388094577812497e-05,
      "logits/chosen": -0.8430393934249878,
      "logits/rejected": -0.8603261709213257,
      "logps/chosen": -251.91726684570312,
      "logps/rejected": -253.66494750976562,
      "loss": 0.5084,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -1.582794189453125,
      "rewards/margins": 0.7729362845420837,
      "rewards/rejected": -2.3557305335998535,
      "step": 1545
    },
    {
      "epoch": 0.33575219321997185,
      "grad_norm": 12.875,
      "learning_rate": 1.4386780323327402e-05,
      "logits/chosen": -0.8400848507881165,
      "logits/rejected": -0.8231026530265808,
      "logps/chosen": -241.6647491455078,
      "logps/rejected": -263.5063781738281,
      "loss": 0.5144,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.3831050395965576,
      "rewards/margins": 0.7762373685836792,
      "rewards/rejected": -2.1593422889709473,
      "step": 1550
    },
    {
      "epoch": 0.336835264811004,
      "grad_norm": 14.75,
      "learning_rate": 1.4385458477494131e-05,
      "logits/chosen": -0.8008373975753784,
      "logits/rejected": -0.7686402797698975,
      "logps/chosen": -260.930419921875,
      "logps/rejected": -256.41845703125,
      "loss": 0.5857,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -1.7162854671478271,
      "rewards/margins": 0.5956167578697205,
      "rewards/rejected": -2.3119025230407715,
      "step": 1555
    },
    {
      "epoch": 0.33791833640203617,
      "grad_norm": 15.9375,
      "learning_rate": 1.4384129042197373e-05,
      "logits/chosen": -0.8686250448226929,
      "logits/rejected": -0.9085467457771301,
      "logps/chosen": -262.4096374511719,
      "logps/rejected": -259.72467041015625,
      "loss": 0.5697,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.7560207843780518,
      "rewards/margins": 0.551972508430481,
      "rewards/rejected": -2.307993173599243,
      "step": 1560
    },
    {
      "epoch": 0.3390014079930683,
      "grad_norm": 15.375,
      "learning_rate": 1.438279201933263e-05,
      "logits/chosen": -0.9393653869628906,
      "logits/rejected": -0.8762472867965698,
      "logps/chosen": -243.59005737304688,
      "logps/rejected": -253.8372802734375,
      "loss": 0.5978,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.7532646656036377,
      "rewards/margins": 0.5034219026565552,
      "rewards/rejected": -2.256686210632324,
      "step": 1565
    },
    {
      "epoch": 0.3400844795841005,
      "grad_norm": 19.75,
      "learning_rate": 1.4381447410806229e-05,
      "logits/chosen": -0.8999065160751343,
      "logits/rejected": -0.7927647233009338,
      "logps/chosen": -243.7126007080078,
      "logps/rejected": -247.8617706298828,
      "loss": 0.6205,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.5799543857574463,
      "rewards/margins": 0.4597190320491791,
      "rewards/rejected": -2.0396735668182373,
      "step": 1570
    },
    {
      "epoch": 0.3411675511751327,
      "grad_norm": 15.8125,
      "learning_rate": 1.4380095218535311e-05,
      "logits/chosen": -0.7943632006645203,
      "logits/rejected": -0.8102038502693176,
      "logps/chosen": -244.7377471923828,
      "logps/rejected": -260.08123779296875,
      "loss": 0.5614,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.4313576221466064,
      "rewards/margins": 0.6523400545120239,
      "rewards/rejected": -2.083697557449341,
      "step": 1575
    },
    {
      "epoch": 0.34225062276616486,
      "grad_norm": 21.125,
      "learning_rate": 1.4378735444447827e-05,
      "logits/chosen": -0.8175361752510071,
      "logits/rejected": -0.8434522747993469,
      "logps/chosen": -255.1641387939453,
      "logps/rejected": -257.0054016113281,
      "loss": 0.5784,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -1.312989592552185,
      "rewards/margins": 0.5815103650093079,
      "rewards/rejected": -1.8945000171661377,
      "step": 1580
    },
    {
      "epoch": 0.343333694357197,
      "grad_norm": 15.1875,
      "learning_rate": 1.4377368090482541e-05,
      "logits/chosen": -0.8236476182937622,
      "logits/rejected": -0.7447627186775208,
      "logps/chosen": -245.7028045654297,
      "logps/rejected": -259.3045959472656,
      "loss": 0.6151,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.2261545658111572,
      "rewards/margins": 0.447501003742218,
      "rewards/rejected": -1.6736555099487305,
      "step": 1585
    },
    {
      "epoch": 0.3444167659482292,
      "grad_norm": 17.625,
      "learning_rate": 1.4375993158589025e-05,
      "logits/chosen": -0.7927995324134827,
      "logits/rejected": -0.7226306200027466,
      "logps/chosen": -255.1035614013672,
      "logps/rejected": -255.2381134033203,
      "loss": 0.5618,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.3348073959350586,
      "rewards/margins": 0.6086773872375488,
      "rewards/rejected": -1.9434845447540283,
      "step": 1590
    },
    {
      "epoch": 0.34549983753926133,
      "grad_norm": 17.0,
      "learning_rate": 1.437461065072765e-05,
      "logits/chosen": -0.7136057019233704,
      "logits/rejected": -0.6481436491012573,
      "logps/chosen": -245.6250457763672,
      "logps/rejected": -268.1473083496094,
      "loss": 0.5436,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.9531529545783997,
      "rewards/margins": 0.6842986345291138,
      "rewards/rejected": -1.6374515295028687,
      "step": 1595
    },
    {
      "epoch": 0.3465829091302935,
      "grad_norm": 14.9375,
      "learning_rate": 1.4373220568869599e-05,
      "logits/chosen": -0.7463101744651794,
      "logits/rejected": -0.7445545792579651,
      "logps/chosen": -240.4030303955078,
      "logps/rejected": -241.94003295898438,
      "loss": 0.5561,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -0.9913017153739929,
      "rewards/margins": 0.5909088253974915,
      "rewards/rejected": -1.5822105407714844,
      "step": 1600
    },
    {
      "epoch": 0.3476659807213257,
      "grad_norm": 15.0,
      "learning_rate": 1.4371822914996842e-05,
      "logits/chosen": -0.7306896448135376,
      "logits/rejected": -0.671475887298584,
      "logps/chosen": -250.3460693359375,
      "logps/rejected": -264.11077880859375,
      "loss": 0.5049,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -1.3729971647262573,
      "rewards/margins": 0.6684837937355042,
      "rewards/rejected": -2.0414812564849854,
      "step": 1605
    },
    {
      "epoch": 0.34874905231235787,
      "grad_norm": 15.9375,
      "learning_rate": 1.4370417691102154e-05,
      "logits/chosen": -0.7869521379470825,
      "logits/rejected": -0.7783654928207397,
      "logps/chosen": -243.66909790039062,
      "logps/rejected": -257.4124755859375,
      "loss": 0.5868,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.4343074560165405,
      "rewards/margins": 0.5055076479911804,
      "rewards/rejected": -1.9398151636123657,
      "step": 1610
    },
    {
      "epoch": 0.34983212390339,
      "grad_norm": 19.375,
      "learning_rate": 1.43690048991891e-05,
      "logits/chosen": -0.7937682271003723,
      "logits/rejected": -0.7648131847381592,
      "logps/chosen": -241.5159149169922,
      "logps/rejected": -254.619384765625,
      "loss": 0.5788,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.3814547061920166,
      "rewards/margins": 0.5361670255661011,
      "rewards/rejected": -1.9176218509674072,
      "step": 1615
    },
    {
      "epoch": 0.3509151954944222,
      "grad_norm": 16.5,
      "learning_rate": 1.4367584541272035e-05,
      "logits/chosen": -0.8191035389900208,
      "logits/rejected": -0.7689509391784668,
      "logps/chosen": -264.5345153808594,
      "logps/rejected": -268.8923034667969,
      "loss": 0.5819,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.8680301904678345,
      "rewards/margins": 0.5284021496772766,
      "rewards/rejected": -2.396432399749756,
      "step": 1620
    },
    {
      "epoch": 0.35199826708545434,
      "grad_norm": 19.625,
      "learning_rate": 1.4366156619376106e-05,
      "logits/chosen": -0.8134651184082031,
      "logits/rejected": -0.7805220484733582,
      "logps/chosen": -258.7095642089844,
      "logps/rejected": -265.3482971191406,
      "loss": 0.6367,
      "rewards/accuracies": 0.5916666984558105,
      "rewards/chosen": -1.7680692672729492,
      "rewards/margins": 0.5624444484710693,
      "rewards/rejected": -2.3305137157440186,
      "step": 1625
    },
    {
      "epoch": 0.3530813386764865,
      "grad_norm": 19.0,
      "learning_rate": 1.4364721135537237e-05,
      "logits/chosen": -0.7601271867752075,
      "logits/rejected": -0.8506206274032593,
      "logps/chosen": -266.62445068359375,
      "logps/rejected": -269.8436279296875,
      "loss": 0.5707,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.5660700798034668,
      "rewards/margins": 0.563513457775116,
      "rewards/rejected": -2.1295835971832275,
      "step": 1630
    },
    {
      "epoch": 0.35416441026751866,
      "grad_norm": 15.375,
      "learning_rate": 1.4363278091802139e-05,
      "logits/chosen": -0.9103348851203918,
      "logits/rejected": -0.8567415475845337,
      "logps/chosen": -239.5276336669922,
      "logps/rejected": -258.72857666015625,
      "loss": 0.5849,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -1.9460055828094482,
      "rewards/margins": 0.6296418905258179,
      "rewards/rejected": -2.5756478309631348,
      "step": 1635
    },
    {
      "epoch": 0.3552474818585509,
      "grad_norm": 14.625,
      "learning_rate": 1.4361827490228301e-05,
      "logits/chosen": -0.9166059494018555,
      "logits/rejected": -0.8184159398078918,
      "logps/chosen": -245.803466796875,
      "logps/rejected": -251.2508544921875,
      "loss": 0.5737,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.7664594650268555,
      "rewards/margins": 0.5762557983398438,
      "rewards/rejected": -2.342715263366699,
      "step": 1640
    },
    {
      "epoch": 0.35633055344958303,
      "grad_norm": 18.0,
      "learning_rate": 1.436036933288399e-05,
      "logits/chosen": -0.9291666746139526,
      "logits/rejected": -0.9329745173454285,
      "logps/chosen": -255.29257202148438,
      "logps/rejected": -260.3198547363281,
      "loss": 0.5681,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4910407066345215,
      "rewards/margins": 0.5827184915542603,
      "rewards/rejected": -2.0737593173980713,
      "step": 1645
    },
    {
      "epoch": 0.3574136250406152,
      "grad_norm": 18.375,
      "learning_rate": 1.4358903621848241e-05,
      "logits/chosen": -0.7955695390701294,
      "logits/rejected": -0.7286661267280579,
      "logps/chosen": -240.4210662841797,
      "logps/rejected": -247.48583984375,
      "loss": 0.6219,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4027900695800781,
      "rewards/margins": 0.43387454748153687,
      "rewards/rejected": -1.8366645574569702,
      "step": 1650
    },
    {
      "epoch": 0.35849669663164735,
      "grad_norm": 15.1875,
      "learning_rate": 1.4357430359210862e-05,
      "logits/chosen": -0.7245470881462097,
      "logits/rejected": -0.6891970038414001,
      "logps/chosen": -269.0987854003906,
      "logps/rejected": -269.3560791015625,
      "loss": 0.5373,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.5384552478790283,
      "rewards/margins": 0.6190378069877625,
      "rewards/rejected": -2.1574931144714355,
      "step": 1655
    },
    {
      "epoch": 0.3595797682226795,
      "grad_norm": 15.3125,
      "learning_rate": 1.4355949547072431e-05,
      "logits/chosen": -0.8090751767158508,
      "logits/rejected": -0.7634453177452087,
      "logps/chosen": -246.5767059326172,
      "logps/rejected": -271.2801818847656,
      "loss": 0.5412,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.4386533498764038,
      "rewards/margins": 0.7636958360671997,
      "rewards/rejected": -2.2023491859436035,
      "step": 1660
    },
    {
      "epoch": 0.36066283981371167,
      "grad_norm": 14.4375,
      "learning_rate": 1.4354461187544283e-05,
      "logits/chosen": -0.7492874264717102,
      "logits/rejected": -0.7318927645683289,
      "logps/chosen": -241.72415161132812,
      "logps/rejected": -264.73675537109375,
      "loss": 0.5797,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.6577650308609009,
      "rewards/margins": 0.761431097984314,
      "rewards/rejected": -2.419196128845215,
      "step": 1665
    },
    {
      "epoch": 0.3617459114047439,
      "grad_norm": 17.25,
      "learning_rate": 1.435296528274852e-05,
      "logits/chosen": -0.7650780081748962,
      "logits/rejected": -0.7713699340820312,
      "logps/chosen": -260.8956298828125,
      "logps/rejected": -277.87884521484375,
      "loss": 0.5962,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.21903657913208,
      "rewards/margins": 0.4144924283027649,
      "rewards/rejected": -2.6335291862487793,
      "step": 1670
    },
    {
      "epoch": 0.36282898299577604,
      "grad_norm": 16.75,
      "learning_rate": 1.4351461834817998e-05,
      "logits/chosen": -0.7323184609413147,
      "logits/rejected": -0.7845991849899292,
      "logps/chosen": -262.36846923828125,
      "logps/rejected": -269.56170654296875,
      "loss": 0.5963,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7677571773529053,
      "rewards/margins": 0.5768184661865234,
      "rewards/rejected": -2.3445756435394287,
      "step": 1675
    },
    {
      "epoch": 0.3639120545868082,
      "grad_norm": 14.3125,
      "learning_rate": 1.4349950845896334e-05,
      "logits/chosen": -0.7835540771484375,
      "logits/rejected": -0.7663801312446594,
      "logps/chosen": -287.282470703125,
      "logps/rejected": -299.693603515625,
      "loss": 0.4965,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.809848427772522,
      "rewards/margins": 0.8933761715888977,
      "rewards/rejected": -2.7032246589660645,
      "step": 1680
    },
    {
      "epoch": 0.36499512617784036,
      "grad_norm": 16.75,
      "learning_rate": 1.434843231813789e-05,
      "logits/chosen": -0.7474247813224792,
      "logits/rejected": -0.7350304126739502,
      "logps/chosen": -244.92422485351562,
      "logps/rejected": -269.8217468261719,
      "loss": 0.5263,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.443716049194336,
      "rewards/margins": 0.8991066813468933,
      "rewards/rejected": -2.342822551727295,
      "step": 1685
    },
    {
      "epoch": 0.3660781977688725,
      "grad_norm": 16.5,
      "learning_rate": 1.434690625370778e-05,
      "logits/chosen": -0.9301311373710632,
      "logits/rejected": -0.8647225499153137,
      "logps/chosen": -283.62200927734375,
      "logps/rejected": -270.02960205078125,
      "loss": 0.6013,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -1.945951223373413,
      "rewards/margins": 0.6038025617599487,
      "rewards/rejected": -2.5497539043426514,
      "step": 1690
    },
    {
      "epoch": 0.3671612693599047,
      "grad_norm": 17.75,
      "learning_rate": 1.4345372654781866e-05,
      "logits/chosen": -0.8023127317428589,
      "logits/rejected": -0.7701441049575806,
      "logps/chosen": -253.9690399169922,
      "logps/rejected": -256.8511047363281,
      "loss": 0.5636,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.6204639673233032,
      "rewards/margins": 0.7497421503067017,
      "rewards/rejected": -2.370206117630005,
      "step": 1695
    },
    {
      "epoch": 0.36824434095093683,
      "grad_norm": 15.0625,
      "learning_rate": 1.4343831523546746e-05,
      "logits/chosen": -0.8486154675483704,
      "logits/rejected": -0.8230484127998352,
      "logps/chosen": -244.89755249023438,
      "logps/rejected": -251.294189453125,
      "loss": 0.5456,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.7985655069351196,
      "rewards/margins": 0.7019620537757874,
      "rewards/rejected": -2.5005276203155518,
      "step": 1700
    },
    {
      "epoch": 0.36932741254196905,
      "grad_norm": 18.0,
      "learning_rate": 1.4342282862199766e-05,
      "logits/chosen": -0.8262039422988892,
      "logits/rejected": -0.7609850168228149,
      "logps/chosen": -244.5858154296875,
      "logps/rejected": -264.0906982421875,
      "loss": 0.5608,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.7649965286254883,
      "rewards/margins": 0.6384950876235962,
      "rewards/rejected": -2.403491735458374,
      "step": 1705
    },
    {
      "epoch": 0.3704104841330012,
      "grad_norm": 21.5,
      "learning_rate": 1.4340726672949004e-05,
      "logits/chosen": -0.7364564538002014,
      "logits/rejected": -0.683843731880188,
      "logps/chosen": -241.48828125,
      "logps/rejected": -257.00213623046875,
      "loss": 0.5744,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2724498510360718,
      "rewards/margins": 0.6260164976119995,
      "rewards/rejected": -1.8984663486480713,
      "step": 1710
    },
    {
      "epoch": 0.37149355572403336,
      "grad_norm": 15.875,
      "learning_rate": 1.4339162958013267e-05,
      "logits/chosen": -0.8050317764282227,
      "logits/rejected": -0.8031495809555054,
      "logps/chosen": -233.16226196289062,
      "logps/rejected": -235.34408569335938,
      "loss": 0.5476,
      "rewards/accuracies": 0.658333420753479,
      "rewards/chosen": -1.1783653497695923,
      "rewards/margins": 0.6681190729141235,
      "rewards/rejected": -1.8464844226837158,
      "step": 1715
    },
    {
      "epoch": 0.3725766273150655,
      "grad_norm": 15.875,
      "learning_rate": 1.43375917196221e-05,
      "logits/chosen": -0.7514585256576538,
      "logits/rejected": -0.7010458707809448,
      "logps/chosen": -258.61529541015625,
      "logps/rejected": -266.9736022949219,
      "loss": 0.5587,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.3012707233428955,
      "rewards/margins": 0.5747076869010925,
      "rewards/rejected": -1.8759782314300537,
      "step": 1720
    },
    {
      "epoch": 0.3736596989060977,
      "grad_norm": 15.625,
      "learning_rate": 1.4336012960015771e-05,
      "logits/chosen": -0.7792027592658997,
      "logits/rejected": -0.7167314291000366,
      "logps/chosen": -255.29843139648438,
      "logps/rejected": -257.6869201660156,
      "loss": 0.5579,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.5927726030349731,
      "rewards/margins": 0.596457839012146,
      "rewards/rejected": -2.189230442047119,
      "step": 1725
    },
    {
      "epoch": 0.37474277049712984,
      "grad_norm": 17.625,
      "learning_rate": 1.4334426681445274e-05,
      "logits/chosen": -0.8607381582260132,
      "logits/rejected": -0.8287590742111206,
      "logps/chosen": -243.6457977294922,
      "logps/rejected": -245.32614135742188,
      "loss": 0.5194,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4758601188659668,
      "rewards/margins": 0.7023521661758423,
      "rewards/rejected": -2.1782124042510986,
      "step": 1730
    },
    {
      "epoch": 0.37582584208816205,
      "grad_norm": 18.875,
      "learning_rate": 1.4332832886172319e-05,
      "logits/chosen": -0.6903751492500305,
      "logits/rejected": -0.6881506443023682,
      "logps/chosen": -283.07672119140625,
      "logps/rejected": -273.8947448730469,
      "loss": 0.5826,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -1.6646430492401123,
      "rewards/margins": 0.5841519236564636,
      "rewards/rejected": -2.2487950325012207,
      "step": 1735
    },
    {
      "epoch": 0.3769089136791942,
      "grad_norm": 18.625,
      "learning_rate": 1.4331231576469337e-05,
      "logits/chosen": -0.804062008857727,
      "logits/rejected": -0.7574133276939392,
      "logps/chosen": -267.8149108886719,
      "logps/rejected": -263.1063232421875,
      "loss": 0.6126,
      "rewards/accuracies": 0.6416666507720947,
      "rewards/chosen": -1.7099298238754272,
      "rewards/margins": 0.517378568649292,
      "rewards/rejected": -2.227308511734009,
      "step": 1740
    },
    {
      "epoch": 0.37799198527022637,
      "grad_norm": 15.4375,
      "learning_rate": 1.4329622754619473e-05,
      "logits/chosen": -0.8214477300643921,
      "logits/rejected": -0.7503783106803894,
      "logps/chosen": -265.1673583984375,
      "logps/rejected": -257.5434265136719,
      "loss": 0.5752,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4019553661346436,
      "rewards/margins": 0.6368986368179321,
      "rewards/rejected": -2.0388541221618652,
      "step": 1745
    },
    {
      "epoch": 0.37907505686125853,
      "grad_norm": 13.75,
      "learning_rate": 1.4328006422916583e-05,
      "logits/chosen": -0.8422417640686035,
      "logits/rejected": -0.8086522221565247,
      "logps/chosen": -247.6966094970703,
      "logps/rejected": -247.59793090820312,
      "loss": 0.5683,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.6661173105239868,
      "rewards/margins": 0.5536462664604187,
      "rewards/rejected": -2.21976375579834,
      "step": 1750
    },
    {
      "epoch": 0.3801581284522907,
      "grad_norm": 15.625,
      "learning_rate": 1.4326382583665225e-05,
      "logits/chosen": -0.820252537727356,
      "logits/rejected": -0.8056613802909851,
      "logps/chosen": -243.6663360595703,
      "logps/rejected": -272.8218994140625,
      "loss": 0.5332,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.6312023401260376,
      "rewards/margins": 0.7586012482643127,
      "rewards/rejected": -2.389803409576416,
      "step": 1755
    },
    {
      "epoch": 0.38124120004332285,
      "grad_norm": 11.3125,
      "learning_rate": 1.4324751239180669e-05,
      "logits/chosen": -0.9093878865242004,
      "logits/rejected": -0.8234380483627319,
      "logps/chosen": -250.14572143554688,
      "logps/rejected": -259.7788391113281,
      "loss": 0.5284,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.5777201652526855,
      "rewards/margins": 0.6898738145828247,
      "rewards/rejected": -2.2675938606262207,
      "step": 1760
    },
    {
      "epoch": 0.382324271634355,
      "grad_norm": 16.875,
      "learning_rate": 1.4323112391788884e-05,
      "logits/chosen": -0.8133150339126587,
      "logits/rejected": -0.837994396686554,
      "logps/chosen": -254.4798126220703,
      "logps/rejected": -264.84356689453125,
      "loss": 0.4836,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -1.2342332601547241,
      "rewards/margins": 0.8355211019515991,
      "rewards/rejected": -2.0697543621063232,
      "step": 1765
    },
    {
      "epoch": 0.3834073432253872,
      "grad_norm": 17.75,
      "learning_rate": 1.4321466043826532e-05,
      "logits/chosen": -0.8741892576217651,
      "logits/rejected": -0.8299840092658997,
      "logps/chosen": -255.9772186279297,
      "logps/rejected": -261.34820556640625,
      "loss": 0.5724,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -1.752385139465332,
      "rewards/margins": 0.5848693251609802,
      "rewards/rejected": -2.337254524230957,
      "step": 1770
    },
    {
      "epoch": 0.3844904148164194,
      "grad_norm": 19.125,
      "learning_rate": 1.4319812197640977e-05,
      "logits/chosen": -0.9445692896842957,
      "logits/rejected": -0.9134804606437683,
      "logps/chosen": -257.8325500488281,
      "logps/rejected": -272.64093017578125,
      "loss": 0.5564,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -1.9265165328979492,
      "rewards/margins": 0.7564727663993835,
      "rewards/rejected": -2.6829893589019775,
      "step": 1775
    },
    {
      "epoch": 0.38557348640745154,
      "grad_norm": 20.5,
      "learning_rate": 1.4318150855590263e-05,
      "logits/chosen": -0.9926680326461792,
      "logits/rejected": -0.9144555330276489,
      "logps/chosen": -253.60061645507812,
      "logps/rejected": -277.44970703125,
      "loss": 0.5722,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -2.0734031200408936,
      "rewards/margins": 0.6091244220733643,
      "rewards/rejected": -2.682527542114258,
      "step": 1780
    },
    {
      "epoch": 0.3866565579984837,
      "grad_norm": 16.0,
      "learning_rate": 1.4316482020043134e-05,
      "logits/chosen": -0.9891721606254578,
      "logits/rejected": -1.047491431236267,
      "logps/chosen": -258.0538024902344,
      "logps/rejected": -265.71221923828125,
      "loss": 0.5763,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.124316930770874,
      "rewards/margins": 0.5561020970344543,
      "rewards/rejected": -2.6804187297821045,
      "step": 1785
    },
    {
      "epoch": 0.38773962958951586,
      "grad_norm": 13.5625,
      "learning_rate": 1.431480569337901e-05,
      "logits/chosen": -0.9109566807746887,
      "logits/rejected": -0.9067762494087219,
      "logps/chosen": -258.1340637207031,
      "logps/rejected": -269.58160400390625,
      "loss": 0.5424,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -1.859055519104004,
      "rewards/margins": 0.6815401315689087,
      "rewards/rejected": -2.540595531463623,
      "step": 1790
    },
    {
      "epoch": 0.388822701180548,
      "grad_norm": 18.625,
      "learning_rate": 1.4313121877987994e-05,
      "logits/chosen": -0.9715259671211243,
      "logits/rejected": -0.8908200263977051,
      "logps/chosen": -250.42507934570312,
      "logps/rejected": -256.8019714355469,
      "loss": 0.5602,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9144432544708252,
      "rewards/margins": 0.6172842979431152,
      "rewards/rejected": -2.5317275524139404,
      "step": 1795
    },
    {
      "epoch": 0.38990577277158023,
      "grad_norm": 15.0625,
      "learning_rate": 1.4311430576270867e-05,
      "logits/chosen": -0.9778540730476379,
      "logits/rejected": -1.013545036315918,
      "logps/chosen": -262.44512939453125,
      "logps/rejected": -270.87939453125,
      "loss": 0.5103,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.788222074508667,
      "rewards/margins": 0.7859984636306763,
      "rewards/rejected": -2.574220657348633,
      "step": 1800
    },
    {
      "epoch": 0.3909888443626124,
      "grad_norm": 15.75,
      "learning_rate": 1.4309731790639085e-05,
      "logits/chosen": -1.032776951789856,
      "logits/rejected": -0.9696511030197144,
      "logps/chosen": -265.00787353515625,
      "logps/rejected": -255.80184936523438,
      "loss": 0.5586,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1160244941711426,
      "rewards/margins": 0.7099727988243103,
      "rewards/rejected": -2.8259975910186768,
      "step": 1805
    },
    {
      "epoch": 0.39207191595364455,
      "grad_norm": 17.125,
      "learning_rate": 1.430802552351477e-05,
      "logits/chosen": -0.9286937713623047,
      "logits/rejected": -0.8849109411239624,
      "logps/chosen": -267.50213623046875,
      "logps/rejected": -290.356689453125,
      "loss": 0.5312,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5792936086654663,
      "rewards/margins": 0.929097056388855,
      "rewards/rejected": -2.5083906650543213,
      "step": 1810
    },
    {
      "epoch": 0.3931549875446767,
      "grad_norm": 19.0,
      "learning_rate": 1.4306311777330718e-05,
      "logits/chosen": -0.9539271593093872,
      "logits/rejected": -0.9180781245231628,
      "logps/chosen": -250.67214965820312,
      "logps/rejected": -259.4949035644531,
      "loss": 0.5282,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.9430339336395264,
      "rewards/margins": 0.7518062591552734,
      "rewards/rejected": -2.6948401927948,
      "step": 1815
    },
    {
      "epoch": 0.39423805913570886,
      "grad_norm": 19.75,
      "learning_rate": 1.4304590554530381e-05,
      "logits/chosen": -0.9625054597854614,
      "logits/rejected": -0.8163393139839172,
      "logps/chosen": -280.7355651855469,
      "logps/rejected": -306.98785400390625,
      "loss": 0.5289,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -1.8992325067520142,
      "rewards/margins": 0.6473459005355835,
      "rewards/rejected": -2.5465784072875977,
      "step": 1820
    },
    {
      "epoch": 0.395321130726741,
      "grad_norm": 16.0,
      "learning_rate": 1.4302861857567879e-05,
      "logits/chosen": -1.1041854619979858,
      "logits/rejected": -1.0521636009216309,
      "logps/chosen": -273.2971496582031,
      "logps/rejected": -294.34356689453125,
      "loss": 0.547,
      "rewards/accuracies": 0.6749999523162842,
      "rewards/chosen": -2.3539204597473145,
      "rewards/margins": 0.8410186767578125,
      "rewards/rejected": -3.194939136505127,
      "step": 1825
    },
    {
      "epoch": 0.3964042023177732,
      "grad_norm": 18.75,
      "learning_rate": 1.430112568890798e-05,
      "logits/chosen": -1.0009628534317017,
      "logits/rejected": -0.9801994562149048,
      "logps/chosen": -258.5396728515625,
      "logps/rejected": -274.12860107421875,
      "loss": 0.5382,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.0172197818756104,
      "rewards/margins": 0.7753834128379822,
      "rewards/rejected": -2.792603015899658,
      "step": 1830
    },
    {
      "epoch": 0.3974872739088054,
      "grad_norm": 17.125,
      "learning_rate": 1.4299382051026115e-05,
      "logits/chosen": -1.0534178018569946,
      "logits/rejected": -0.9875997304916382,
      "logps/chosen": -276.4689025878906,
      "logps/rejected": -285.1141052246094,
      "loss": 0.5375,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.9849212169647217,
      "rewards/margins": 0.8962751626968384,
      "rewards/rejected": -2.8811964988708496,
      "step": 1835
    },
    {
      "epoch": 0.39857034549983755,
      "grad_norm": 20.5,
      "learning_rate": 1.4297630946408356e-05,
      "logits/chosen": -1.1312570571899414,
      "logits/rejected": -1.0951857566833496,
      "logps/chosen": -275.78277587890625,
      "logps/rejected": -296.00311279296875,
      "loss": 0.5533,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -2.289161205291748,
      "rewards/margins": 0.7568100690841675,
      "rewards/rejected": -3.045971393585205,
      "step": 1840
    },
    {
      "epoch": 0.3996534170908697,
      "grad_norm": 15.75,
      "learning_rate": 1.4295872377551424e-05,
      "logits/chosen": -1.1903177499771118,
      "logits/rejected": -1.1679670810699463,
      "logps/chosen": -264.39752197265625,
      "logps/rejected": -281.11456298828125,
      "loss": 0.544,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -2.340266704559326,
      "rewards/margins": 0.8635807037353516,
      "rewards/rejected": -3.2038474082946777,
      "step": 1845
    },
    {
      "epoch": 0.40073648868190187,
      "grad_norm": 13.5625,
      "learning_rate": 1.4294106346962687e-05,
      "logits/chosen": -1.0396606922149658,
      "logits/rejected": -1.0430612564086914,
      "logps/chosen": -268.11480712890625,
      "logps/rejected": -273.5795593261719,
      "loss": 0.6253,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -2.317070484161377,
      "rewards/margins": 0.5530928373336792,
      "rewards/rejected": -2.870163679122925,
      "step": 1850
    },
    {
      "epoch": 0.40181956027293403,
      "grad_norm": 14.1875,
      "learning_rate": 1.4292332857160142e-05,
      "logits/chosen": -1.205608606338501,
      "logits/rejected": -1.1454917192459106,
      "logps/chosen": -266.9917907714844,
      "logps/rejected": -284.72271728515625,
      "loss": 0.5289,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -2.3792409896850586,
      "rewards/margins": 0.8432963490486145,
      "rewards/rejected": -3.2225372791290283,
      "step": 1855
    },
    {
      "epoch": 0.4029026318639662,
      "grad_norm": 17.5,
      "learning_rate": 1.429055191067243e-05,
      "logits/chosen": -0.9919408559799194,
      "logits/rejected": -0.9897691607475281,
      "logps/chosen": -272.062255859375,
      "logps/rejected": -281.4119873046875,
      "loss": 0.5597,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.986047387123108,
      "rewards/margins": 0.7284066677093506,
      "rewards/rejected": -2.714454174041748,
      "step": 1860
    },
    {
      "epoch": 0.4039857034549984,
      "grad_norm": 13.5625,
      "learning_rate": 1.4288763510038822e-05,
      "logits/chosen": -1.041102409362793,
      "logits/rejected": -1.022080421447754,
      "logps/chosen": -262.74444580078125,
      "logps/rejected": -280.5736083984375,
      "loss": 0.5513,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.9411256313323975,
      "rewards/margins": 0.8370518684387207,
      "rewards/rejected": -2.778177261352539,
      "step": 1865
    },
    {
      "epoch": 0.40506877504603056,
      "grad_norm": 21.625,
      "learning_rate": 1.4286967657809214e-05,
      "logits/chosen": -1.1026147603988647,
      "logits/rejected": -1.0489275455474854,
      "logps/chosen": -278.0836486816406,
      "logps/rejected": -277.1592102050781,
      "loss": 0.6362,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.164580821990967,
      "rewards/margins": 0.44028791785240173,
      "rewards/rejected": -2.6048684120178223,
      "step": 1870
    },
    {
      "epoch": 0.4061518466370627,
      "grad_norm": 19.0,
      "learning_rate": 1.4285164356544131e-05,
      "logits/chosen": -1.0778573751449585,
      "logits/rejected": -1.0087100267410278,
      "logps/chosen": -260.9560241699219,
      "logps/rejected": -270.3584899902344,
      "loss": 0.5507,
      "rewards/accuracies": 0.7166667580604553,
      "rewards/chosen": -1.9336029291152954,
      "rewards/margins": 0.7482233643531799,
      "rewards/rejected": -2.68182635307312,
      "step": 1875
    },
    {
      "epoch": 0.4072349182280949,
      "grad_norm": 13.5,
      "learning_rate": 1.4283353608814717e-05,
      "logits/chosen": -1.053283929824829,
      "logits/rejected": -0.9979074597358704,
      "logps/chosen": -256.1917419433594,
      "logps/rejected": -267.677734375,
      "loss": 0.5096,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.022473096847534,
      "rewards/margins": 0.7590644955635071,
      "rewards/rejected": -2.7815377712249756,
      "step": 1880
    },
    {
      "epoch": 0.40831798981912704,
      "grad_norm": 15.25,
      "learning_rate": 1.4281535417202728e-05,
      "logits/chosen": -0.9265245199203491,
      "logits/rejected": -0.8376182317733765,
      "logps/chosen": -276.27325439453125,
      "logps/rejected": -275.8667297363281,
      "loss": 0.5505,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.8130671977996826,
      "rewards/margins": 0.643242359161377,
      "rewards/rejected": -2.4563097953796387,
      "step": 1885
    },
    {
      "epoch": 0.4094010614101592,
      "grad_norm": 18.875,
      "learning_rate": 1.4279709784300541e-05,
      "logits/chosen": -0.9629146456718445,
      "logits/rejected": -0.993911862373352,
      "logps/chosen": -256.91668701171875,
      "logps/rejected": -278.061767578125,
      "loss": 0.5454,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -2.140218496322632,
      "rewards/margins": 0.9393094778060913,
      "rewards/rejected": -3.0795280933380127,
      "step": 1890
    },
    {
      "epoch": 0.41048413300119135,
      "grad_norm": 20.75,
      "learning_rate": 1.4277876712711137e-05,
      "logits/chosen": -0.9897246360778809,
      "logits/rejected": -0.951362133026123,
      "logps/chosen": -263.43084716796875,
      "logps/rejected": -261.61004638671875,
      "loss": 0.6345,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.862463355064392,
      "rewards/margins": 0.5305992960929871,
      "rewards/rejected": -2.3930625915527344,
      "step": 1895
    },
    {
      "epoch": 0.41156720459222357,
      "grad_norm": 15.6875,
      "learning_rate": 1.427603620504811e-05,
      "logits/chosen": -1.0203770399093628,
      "logits/rejected": -0.8562126159667969,
      "logps/chosen": -270.5472106933594,
      "logps/rejected": -315.7224426269531,
      "loss": 0.5211,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.1127970218658447,
      "rewards/margins": 0.8911916613578796,
      "rewards/rejected": -3.003988742828369,
      "step": 1900
    },
    {
      "epoch": 0.4126502761832557,
      "grad_norm": 18.625,
      "learning_rate": 1.427418826393565e-05,
      "logits/chosen": -0.9441285133361816,
      "logits/rejected": -0.9544242024421692,
      "logps/chosen": -273.19720458984375,
      "logps/rejected": -279.7838439941406,
      "loss": 0.5392,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.0584540367126465,
      "rewards/margins": 0.8157480359077454,
      "rewards/rejected": -2.874202251434326,
      "step": 1905
    },
    {
      "epoch": 0.4137333477742879,
      "grad_norm": 18.375,
      "learning_rate": 1.4272332892008547e-05,
      "logits/chosen": -1.0234196186065674,
      "logits/rejected": -1.019460916519165,
      "logps/chosen": -263.07904052734375,
      "logps/rejected": -264.47821044921875,
      "loss": 0.5156,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.9769010543823242,
      "rewards/margins": 0.7016016244888306,
      "rewards/rejected": -2.6785025596618652,
      "step": 1910
    },
    {
      "epoch": 0.41481641936532004,
      "grad_norm": 13.0625,
      "learning_rate": 1.4270470091912188e-05,
      "logits/chosen": -0.9733136296272278,
      "logits/rejected": -0.9830517768859863,
      "logps/chosen": -253.81643676757812,
      "logps/rejected": -267.4310607910156,
      "loss": 0.5115,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -1.6857759952545166,
      "rewards/margins": 0.8132553100585938,
      "rewards/rejected": -2.4990315437316895,
      "step": 1915
    },
    {
      "epoch": 0.4158994909563522,
      "grad_norm": 16.0,
      "learning_rate": 1.4268599866302545e-05,
      "logits/chosen": -1.0656806230545044,
      "logits/rejected": -1.0674000978469849,
      "logps/chosen": -242.0073699951172,
      "logps/rejected": -257.4451599121094,
      "loss": 0.5381,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -1.9972209930419922,
      "rewards/margins": 0.666935920715332,
      "rewards/rejected": -2.664156675338745,
      "step": 1920
    },
    {
      "epoch": 0.41698256254738436,
      "grad_norm": 14.4375,
      "learning_rate": 1.4266722217846192e-05,
      "logits/chosen": -1.03227698802948,
      "logits/rejected": -1.056180715560913,
      "logps/chosen": -244.2303924560547,
      "logps/rejected": -255.6460723876953,
      "loss": 0.518,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -2.0615005493164062,
      "rewards/margins": 0.8818842768669128,
      "rewards/rejected": -2.943384885787964,
      "step": 1925
    },
    {
      "epoch": 0.4180656341384166,
      "grad_norm": 16.375,
      "learning_rate": 1.4264837149220267e-05,
      "logits/chosen": -1.067704677581787,
      "logits/rejected": -1.0836942195892334,
      "logps/chosen": -268.334228515625,
      "logps/rejected": -271.4444274902344,
      "loss": 0.5244,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -2.0181918144226074,
      "rewards/margins": 0.7087535262107849,
      "rewards/rejected": -2.726944923400879,
      "step": 1930
    },
    {
      "epoch": 0.41914870572944873,
      "grad_norm": 22.25,
      "learning_rate": 1.42629446631125e-05,
      "logits/chosen": -1.2101020812988281,
      "logits/rejected": -1.1427984237670898,
      "logps/chosen": -260.3104553222656,
      "logps/rejected": -284.74066162109375,
      "loss": 0.5848,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.917384386062622,
      "rewards/margins": 0.8219369649887085,
      "rewards/rejected": -2.73932147026062,
      "step": 1935
    },
    {
      "epoch": 0.4202317773204809,
      "grad_norm": 18.625,
      "learning_rate": 1.4261044762221195e-05,
      "logits/chosen": -1.195650339126587,
      "logits/rejected": -1.1372077465057373,
      "logps/chosen": -251.9102783203125,
      "logps/rejected": -263.85186767578125,
      "loss": 0.58,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -2.0369815826416016,
      "rewards/margins": 0.6531511545181274,
      "rewards/rejected": -2.6901326179504395,
      "step": 1940
    },
    {
      "epoch": 0.42131484891151305,
      "grad_norm": 16.75,
      "learning_rate": 1.4259137449255226e-05,
      "logits/chosen": -1.185638427734375,
      "logits/rejected": -1.1696209907531738,
      "logps/chosen": -272.0119934082031,
      "logps/rejected": -269.25262451171875,
      "loss": 0.5483,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.987105131149292,
      "rewards/margins": 0.6459411382675171,
      "rewards/rejected": -2.6330461502075195,
      "step": 1945
    },
    {
      "epoch": 0.4223979205025452,
      "grad_norm": 19.375,
      "learning_rate": 1.4257222726934034e-05,
      "logits/chosen": -1.091447114944458,
      "logits/rejected": -1.0760146379470825,
      "logps/chosen": -245.0991668701172,
      "logps/rejected": -255.92367553710938,
      "loss": 0.4969,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -1.2826719284057617,
      "rewards/margins": 0.918591320514679,
      "rewards/rejected": -2.201262950897217,
      "step": 1950
    },
    {
      "epoch": 0.42348099209357737,
      "grad_norm": 17.25,
      "learning_rate": 1.4255300597987628e-05,
      "logits/chosen": -1.122521162033081,
      "logits/rejected": -1.1729309558868408,
      "logps/chosen": -253.58389282226562,
      "logps/rejected": -258.7366027832031,
      "loss": 0.5389,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.9177796840667725,
      "rewards/margins": 0.7694286108016968,
      "rewards/rejected": -2.6872081756591797,
      "step": 1955
    },
    {
      "epoch": 0.4245640636846095,
      "grad_norm": 15.9375,
      "learning_rate": 1.425337106515658e-05,
      "logits/chosen": -1.2881218194961548,
      "logits/rejected": -1.268127679824829,
      "logps/chosen": -250.99496459960938,
      "logps/rejected": -256.59735107421875,
      "loss": 0.5602,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.9378992319107056,
      "rewards/margins": 0.6664783954620361,
      "rewards/rejected": -2.6043777465820312,
      "step": 1960
    },
    {
      "epoch": 0.42564713527564174,
      "grad_norm": 21.75,
      "learning_rate": 1.4251434131192007e-05,
      "logits/chosen": -1.221762776374817,
      "logits/rejected": -1.1776920557022095,
      "logps/chosen": -268.42657470703125,
      "logps/rejected": -273.28131103515625,
      "loss": 0.5869,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -2.2619712352752686,
      "rewards/margins": 0.7669438123703003,
      "rewards/rejected": -3.0289149284362793,
      "step": 1965
    },
    {
      "epoch": 0.4267302068666739,
      "grad_norm": 24.75,
      "learning_rate": 1.4249489798855588e-05,
      "logits/chosen": -1.1473724842071533,
      "logits/rejected": -1.1308038234710693,
      "logps/chosen": -273.7420654296875,
      "logps/rejected": -286.30169677734375,
      "loss": 0.643,
      "rewards/accuracies": 0.6583333015441895,
      "rewards/chosen": -2.2917141914367676,
      "rewards/margins": 0.5234671831130981,
      "rewards/rejected": -2.8151814937591553,
      "step": 1970
    },
    {
      "epoch": 0.42781327845770606,
      "grad_norm": 16.5,
      "learning_rate": 1.4247538070919549e-05,
      "logits/chosen": -1.2063887119293213,
      "logits/rejected": -1.1237660646438599,
      "logps/chosen": -259.2772521972656,
      "logps/rejected": -296.8221435546875,
      "loss": 0.4987,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.042222499847412,
      "rewards/margins": 0.9468919634819031,
      "rewards/rejected": -2.989114284515381,
      "step": 1975
    },
    {
      "epoch": 0.4288963500487382,
      "grad_norm": 16.5,
      "learning_rate": 1.424557895016666e-05,
      "logits/chosen": -1.2214343547821045,
      "logits/rejected": -1.2705655097961426,
      "logps/chosen": -264.330322265625,
      "logps/rejected": -276.0326843261719,
      "loss": 0.5315,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.302483081817627,
      "rewards/margins": 0.8173856735229492,
      "rewards/rejected": -3.1198689937591553,
      "step": 1980
    },
    {
      "epoch": 0.4299794216397704,
      "grad_norm": 16.125,
      "learning_rate": 1.424361243939023e-05,
      "logits/chosen": -1.2979151010513306,
      "logits/rejected": -1.2273836135864258,
      "logps/chosen": -242.4459228515625,
      "logps/rejected": -270.24139404296875,
      "loss": 0.4975,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.149850368499756,
      "rewards/margins": 1.007805585861206,
      "rewards/rejected": -3.1576554775238037,
      "step": 1985
    },
    {
      "epoch": 0.43106249323080253,
      "grad_norm": 18.375,
      "learning_rate": 1.4241638541394108e-05,
      "logits/chosen": -1.1768356561660767,
      "logits/rejected": -1.178406000137329,
      "logps/chosen": -264.08056640625,
      "logps/rejected": -263.80511474609375,
      "loss": 0.4987,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.4395089149475098,
      "rewards/margins": 0.806128203868866,
      "rewards/rejected": -3.2456371784210205,
      "step": 1990
    },
    {
      "epoch": 0.43214556482183475,
      "grad_norm": 16.0,
      "learning_rate": 1.4239657258992673e-05,
      "logits/chosen": -1.2476823329925537,
      "logits/rejected": -1.1616411209106445,
      "logps/chosen": -252.94375610351562,
      "logps/rejected": -286.2205505371094,
      "loss": 0.4832,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.882537603378296,
      "rewards/margins": 1.1707959175109863,
      "rewards/rejected": -3.0533335208892822,
      "step": 1995
    },
    {
      "epoch": 0.4332286364128669,
      "grad_norm": 14.1875,
      "learning_rate": 1.4237668595010828e-05,
      "logits/chosen": -1.2135419845581055,
      "logits/rejected": -1.1948378086090088,
      "logps/chosen": -293.0056457519531,
      "logps/rejected": -292.49200439453125,
      "loss": 0.5371,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.566795825958252,
      "rewards/margins": 0.7983666658401489,
      "rewards/rejected": -3.3651626110076904,
      "step": 2000
    },
    {
      "epoch": 0.4332286364128669,
      "eval_logits/chosen": -1.2104902267456055,
      "eval_logits/rejected": -1.146630048751831,
      "eval_logps/chosen": -258.84039306640625,
      "eval_logps/rejected": -277.5482177734375,
      "eval_loss": 0.5572131872177124,
      "eval_rewards/accuracies": 0.7099999785423279,
      "eval_rewards/chosen": -2.2319560050964355,
      "eval_rewards/margins": 0.7625982165336609,
      "eval_rewards/rejected": -2.994554042816162,
      "eval_runtime": 14.4385,
      "eval_samples_per_second": 13.852,
      "eval_steps_per_second": 13.852,
      "step": 2000
    },
    {
      "epoch": 0.43431170800389907,
      "grad_norm": 19.25,
      "learning_rate": 1.4235672552284016e-05,
      "logits/chosen": -1.1842247247695923,
      "logits/rejected": -1.1853526830673218,
      "logps/chosen": -285.78564453125,
      "logps/rejected": -298.64739990234375,
      "loss": 0.586,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.3116095066070557,
      "rewards/margins": 0.7583860754966736,
      "rewards/rejected": -3.069995403289795,
      "step": 2005
    },
    {
      "epoch": 0.4353947795949312,
      "grad_norm": 15.375,
      "learning_rate": 1.4233669133658185e-05,
      "logits/chosen": -1.2832882404327393,
      "logits/rejected": -1.2594715356826782,
      "logps/chosen": -280.316650390625,
      "logps/rejected": -307.14337158203125,
      "loss": 0.4991,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.7319655418395996,
      "rewards/margins": 1.038526177406311,
      "rewards/rejected": -3.770491361618042,
      "step": 2010
    },
    {
      "epoch": 0.4364778511859634,
      "grad_norm": 15.5,
      "learning_rate": 1.4231658341989806e-05,
      "logits/chosen": -1.2171415090560913,
      "logits/rejected": -1.1634011268615723,
      "logps/chosen": -273.9981689453125,
      "logps/rejected": -291.4281311035156,
      "loss": 0.5706,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.400667428970337,
      "rewards/margins": 0.6216941475868225,
      "rewards/rejected": -3.022361993789673,
      "step": 2015
    },
    {
      "epoch": 0.43756092277699554,
      "grad_norm": 16.875,
      "learning_rate": 1.4229640180145862e-05,
      "logits/chosen": -1.181121587753296,
      "logits/rejected": -1.0852130651474,
      "logps/chosen": -268.360595703125,
      "logps/rejected": -279.0284729003906,
      "loss": 0.5251,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -1.9367672204971313,
      "rewards/margins": 0.8869361877441406,
      "rewards/rejected": -2.8237032890319824,
      "step": 2020
    },
    {
      "epoch": 0.4386439943680277,
      "grad_norm": 14.8125,
      "learning_rate": 1.4227614651003844e-05,
      "logits/chosen": -1.2820897102355957,
      "logits/rejected": -1.1549463272094727,
      "logps/chosen": -254.28622436523438,
      "logps/rejected": -270.4839172363281,
      "loss": 0.5275,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.287902593612671,
      "rewards/margins": 0.8442327380180359,
      "rewards/rejected": -3.1321353912353516,
      "step": 2025
    },
    {
      "epoch": 0.4397270659590599,
      "grad_norm": 15.4375,
      "learning_rate": 1.4225581757451747e-05,
      "logits/chosen": -1.2645975351333618,
      "logits/rejected": -1.1777074337005615,
      "logps/chosen": -241.990234375,
      "logps/rejected": -267.09619140625,
      "loss": 0.5543,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -1.8209717273712158,
      "rewards/margins": 0.682286262512207,
      "rewards/rejected": -2.5032577514648438,
      "step": 2030
    },
    {
      "epoch": 0.4408101375500921,
      "grad_norm": 22.0,
      "learning_rate": 1.4223541502388066e-05,
      "logits/chosen": -1.2397762537002563,
      "logits/rejected": -1.2071638107299805,
      "logps/chosen": -249.40731811523438,
      "logps/rejected": -252.5460968017578,
      "loss": 0.5742,
      "rewards/accuracies": 0.7250000834465027,
      "rewards/chosen": -2.1652770042419434,
      "rewards/margins": 0.6437378525733948,
      "rewards/rejected": -2.8090150356292725,
      "step": 2035
    },
    {
      "epoch": 0.44189320914112423,
      "grad_norm": 14.5,
      "learning_rate": 1.4221493888721795e-05,
      "logits/chosen": -1.292372465133667,
      "logits/rejected": -1.2626291513442993,
      "logps/chosen": -249.47195434570312,
      "logps/rejected": -273.8794250488281,
      "loss": 0.5302,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.9357191324234009,
      "rewards/margins": 0.6794013977050781,
      "rewards/rejected": -2.6151204109191895,
      "step": 2040
    },
    {
      "epoch": 0.4429762807321564,
      "grad_norm": 12.1875,
      "learning_rate": 1.4219438919372414e-05,
      "logits/chosen": -1.2505440711975098,
      "logits/rejected": -1.313697338104248,
      "logps/chosen": -251.885986328125,
      "logps/rejected": -252.15286254882812,
      "loss": 0.5375,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.801847219467163,
      "rewards/margins": 0.6834757328033447,
      "rewards/rejected": -2.485322952270508,
      "step": 2045
    },
    {
      "epoch": 0.44405935232318855,
      "grad_norm": 20.25,
      "learning_rate": 1.4217376597269897e-05,
      "logits/chosen": -1.1873468160629272,
      "logits/rejected": -1.1258594989776611,
      "logps/chosen": -251.7368927001953,
      "logps/rejected": -259.4183349609375,
      "loss": 0.5782,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.4482823610305786,
      "rewards/margins": 0.6196103692054749,
      "rewards/rejected": -2.067892551422119,
      "step": 2050
    },
    {
      "epoch": 0.4451424239142207,
      "grad_norm": 16.0,
      "learning_rate": 1.42153069253547e-05,
      "logits/chosen": -1.1961177587509155,
      "logits/rejected": -1.1399773359298706,
      "logps/chosen": -259.83062744140625,
      "logps/rejected": -270.72113037109375,
      "loss": 0.5088,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -1.2817747592926025,
      "rewards/margins": 0.7484508752822876,
      "rewards/rejected": -2.0302252769470215,
      "step": 2055
    },
    {
      "epoch": 0.4462254955052529,
      "grad_norm": 16.5,
      "learning_rate": 1.4213229906577751e-05,
      "logits/chosen": -1.0878088474273682,
      "logits/rejected": -1.0876624584197998,
      "logps/chosen": -241.24673461914062,
      "logps/rejected": -258.19781494140625,
      "loss": 0.5519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3116743564605713,
      "rewards/margins": 0.6923326849937439,
      "rewards/rejected": -2.00400710105896,
      "step": 2060
    },
    {
      "epoch": 0.4473085670962851,
      "grad_norm": 19.625,
      "learning_rate": 1.4211145543900465e-05,
      "logits/chosen": -1.1023634672164917,
      "logits/rejected": -1.1103194952011108,
      "logps/chosen": -239.29983520507812,
      "logps/rejected": -250.05081176757812,
      "loss": 0.4987,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -1.2776429653167725,
      "rewards/margins": 0.712195873260498,
      "rewards/rejected": -1.9898388385772705,
      "step": 2065
    },
    {
      "epoch": 0.44839163868731724,
      "grad_norm": 19.125,
      "learning_rate": 1.4209053840294721e-05,
      "logits/chosen": -1.1117581129074097,
      "logits/rejected": -1.0653401613235474,
      "logps/chosen": -264.3855895996094,
      "logps/rejected": -267.7663879394531,
      "loss": 0.5624,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.562840223312378,
      "rewards/margins": 0.6381114721298218,
      "rewards/rejected": -2.20095157623291,
      "step": 2070
    },
    {
      "epoch": 0.4494747102783494,
      "grad_norm": 19.625,
      "learning_rate": 1.4206954798742863e-05,
      "logits/chosen": -1.188055396080017,
      "logits/rejected": -1.1374149322509766,
      "logps/chosen": -258.683837890625,
      "logps/rejected": -271.45831298828125,
      "loss": 0.524,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -1.9835294485092163,
      "rewards/margins": 0.8773692846298218,
      "rewards/rejected": -2.860898733139038,
      "step": 2075
    },
    {
      "epoch": 0.45055778186938156,
      "grad_norm": 15.6875,
      "learning_rate": 1.4204848422237702e-05,
      "logits/chosen": -1.1508060693740845,
      "logits/rejected": -1.1090443134307861,
      "logps/chosen": -259.94989013671875,
      "logps/rejected": -265.5556945800781,
      "loss": 0.5522,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7336318492889404,
      "rewards/margins": 0.7835192084312439,
      "rewards/rejected": -2.517151117324829,
      "step": 2080
    },
    {
      "epoch": 0.4516408534604137,
      "grad_norm": 16.0,
      "learning_rate": 1.4202734713782506e-05,
      "logits/chosen": -1.0439239740371704,
      "logits/rejected": -1.0233309268951416,
      "logps/chosen": -264.865966796875,
      "logps/rejected": -275.1529235839844,
      "loss": 0.566,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.075441360473633,
      "rewards/margins": 0.7400392293930054,
      "rewards/rejected": -2.8154807090759277,
      "step": 2085
    },
    {
      "epoch": 0.4527239250514459,
      "grad_norm": 18.625,
      "learning_rate": 1.4200613676390996e-05,
      "logits/chosen": -1.2045377492904663,
      "logits/rejected": -1.1562758684158325,
      "logps/chosen": -275.26556396484375,
      "logps/rejected": -273.3468322753906,
      "loss": 0.5869,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.4841525554656982,
      "rewards/margins": 0.6901721358299255,
      "rewards/rejected": -3.1743245124816895,
      "step": 2090
    },
    {
      "epoch": 0.4538069966424781,
      "grad_norm": 18.625,
      "learning_rate": 1.4198485313087344e-05,
      "logits/chosen": -0.9803377389907837,
      "logits/rejected": -0.9364644885063171,
      "logps/chosen": -240.749267578125,
      "logps/rejected": -264.2745056152344,
      "loss": 0.558,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.693170189857483,
      "rewards/margins": 0.677627444267273,
      "rewards/rejected": -2.370797634124756,
      "step": 2095
    },
    {
      "epoch": 0.45489006823351025,
      "grad_norm": 15.5,
      "learning_rate": 1.4196349626906161e-05,
      "logits/chosen": -0.9023561477661133,
      "logits/rejected": -0.9026093482971191,
      "logps/chosen": -279.14520263671875,
      "logps/rejected": -296.344482421875,
      "loss": 0.5548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.8759441375732422,
      "rewards/margins": 0.7438714504241943,
      "rewards/rejected": -2.6198153495788574,
      "step": 2100
    },
    {
      "epoch": 0.4559731398245424,
      "grad_norm": 15.4375,
      "learning_rate": 1.4194206620892513e-05,
      "logits/chosen": -0.9601651430130005,
      "logits/rejected": -0.8899949193000793,
      "logps/chosen": -248.9342041015625,
      "logps/rejected": -265.3130798339844,
      "loss": 0.5117,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.5226675271987915,
      "rewards/margins": 0.889045238494873,
      "rewards/rejected": -2.411712646484375,
      "step": 2105
    },
    {
      "epoch": 0.45705621141557456,
      "grad_norm": 18.25,
      "learning_rate": 1.4192056298101883e-05,
      "logits/chosen": -1.0255472660064697,
      "logits/rejected": -0.9738146662712097,
      "logps/chosen": -260.7536926269531,
      "logps/rejected": -284.48455810546875,
      "loss": 0.4995,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.9915273189544678,
      "rewards/margins": 1.033369779586792,
      "rewards/rejected": -3.0248970985412598,
      "step": 2110
    },
    {
      "epoch": 0.4581392830066067,
      "grad_norm": 16.375,
      "learning_rate": 1.4189898661600205e-05,
      "logits/chosen": -1.0441701412200928,
      "logits/rejected": -0.9495380520820618,
      "logps/chosen": -254.0312042236328,
      "logps/rejected": -262.9917907714844,
      "loss": 0.5433,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7246090173721313,
      "rewards/margins": 0.7144724726676941,
      "rewards/rejected": -2.4390814304351807,
      "step": 2115
    },
    {
      "epoch": 0.4592223545976389,
      "grad_norm": 23.25,
      "learning_rate": 1.418773371446383e-05,
      "logits/chosen": -0.9391103982925415,
      "logits/rejected": -0.9798518419265747,
      "logps/chosen": -257.6258544921875,
      "logps/rejected": -268.31243896484375,
      "loss": 0.598,
      "rewards/accuracies": 0.6416667699813843,
      "rewards/chosen": -1.7239885330200195,
      "rewards/margins": 0.6502836346626282,
      "rewards/rejected": -2.374272346496582,
      "step": 2120
    },
    {
      "epoch": 0.4603054261886711,
      "grad_norm": 16.75,
      "learning_rate": 1.4185561459779537e-05,
      "logits/chosen": -0.9756108522415161,
      "logits/rejected": -0.9992986917495728,
      "logps/chosen": -257.5811462402344,
      "logps/rejected": -268.79827880859375,
      "loss": 0.5562,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -1.5930001735687256,
      "rewards/margins": 0.8086250424385071,
      "rewards/rejected": -2.401625156402588,
      "step": 2125
    },
    {
      "epoch": 0.46138849777970325,
      "grad_norm": 17.875,
      "learning_rate": 1.4183381900644519e-05,
      "logits/chosen": -0.9533130526542664,
      "logits/rejected": -0.8954323530197144,
      "logps/chosen": -253.1155242919922,
      "logps/rejected": -269.67645263671875,
      "loss": 0.4635,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.6793445348739624,
      "rewards/margins": 0.9818909764289856,
      "rewards/rejected": -2.6612355709075928,
      "step": 2130
    },
    {
      "epoch": 0.4624715693707354,
      "grad_norm": 18.125,
      "learning_rate": 1.4181195040166389e-05,
      "logits/chosen": -0.9924600720405579,
      "logits/rejected": -0.9212808609008789,
      "logps/chosen": -243.1766357421875,
      "logps/rejected": -270.96112060546875,
      "loss": 0.4858,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -1.6525799036026,
      "rewards/margins": 0.9283296465873718,
      "rewards/rejected": -2.5809097290039062,
      "step": 2135
    },
    {
      "epoch": 0.4635546409617676,
      "grad_norm": 17.875,
      "learning_rate": 1.4179000881463169e-05,
      "logits/chosen": -0.9078903198242188,
      "logits/rejected": -0.9550788998603821,
      "logps/chosen": -251.2039031982422,
      "logps/rejected": -265.02484130859375,
      "loss": 0.6043,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -1.8805698156356812,
      "rewards/margins": 0.5531989932060242,
      "rewards/rejected": -2.4337687492370605,
      "step": 2140
    },
    {
      "epoch": 0.46463771255279973,
      "grad_norm": 19.75,
      "learning_rate": 1.4176799427663283e-05,
      "logits/chosen": -1.0581812858581543,
      "logits/rejected": -1.0737016201019287,
      "logps/chosen": -255.6433563232422,
      "logps/rejected": -268.5458068847656,
      "loss": 0.5257,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.0578837394714355,
      "rewards/margins": 0.8624227643013,
      "rewards/rejected": -2.9203062057495117,
      "step": 2145
    },
    {
      "epoch": 0.4657207841438319,
      "grad_norm": 17.5,
      "learning_rate": 1.417459068190556e-05,
      "logits/chosen": -1.007395625114441,
      "logits/rejected": -0.9687100648880005,
      "logps/chosen": -281.9477233886719,
      "logps/rejected": -288.5154113769531,
      "loss": 0.5159,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.1309609413146973,
      "rewards/margins": 0.838647723197937,
      "rewards/rejected": -2.969608783721924,
      "step": 2150
    },
    {
      "epoch": 0.46680385573486405,
      "grad_norm": 13.125,
      "learning_rate": 1.417237464733923e-05,
      "logits/chosen": -1.0564011335372925,
      "logits/rejected": -1.056102991104126,
      "logps/chosen": -256.55621337890625,
      "logps/rejected": -261.243896484375,
      "loss": 0.4949,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -2.232473373413086,
      "rewards/margins": 0.8864701986312866,
      "rewards/rejected": -3.118943691253662,
      "step": 2155
    },
    {
      "epoch": 0.46788692732589626,
      "grad_norm": 16.875,
      "learning_rate": 1.4170151327123907e-05,
      "logits/chosen": -0.9337573051452637,
      "logits/rejected": -0.9398282766342163,
      "logps/chosen": -261.5662536621094,
      "logps/rejected": -274.001953125,
      "loss": 0.5025,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.206019639968872,
      "rewards/margins": 0.8775808215141296,
      "rewards/rejected": -3.0836005210876465,
      "step": 2160
    },
    {
      "epoch": 0.4689699989169284,
      "grad_norm": 18.625,
      "learning_rate": 1.4167920724429599e-05,
      "logits/chosen": -1.099552869796753,
      "logits/rejected": -0.9784445762634277,
      "logps/chosen": -280.387939453125,
      "logps/rejected": -286.44775390625,
      "loss": 0.4968,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.1955366134643555,
      "rewards/margins": 0.946280837059021,
      "rewards/rejected": -3.141817569732666,
      "step": 2165
    },
    {
      "epoch": 0.4700530705079606,
      "grad_norm": 20.875,
      "learning_rate": 1.416568284243669e-05,
      "logits/chosen": -0.9167990684509277,
      "logits/rejected": -0.8952016830444336,
      "logps/chosen": -272.943359375,
      "logps/rejected": -298.64373779296875,
      "loss": 0.5937,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.391833543777466,
      "rewards/margins": 0.6508208513259888,
      "rewards/rejected": -3.042654514312744,
      "step": 2170
    },
    {
      "epoch": 0.47113614209899274,
      "grad_norm": 17.5,
      "learning_rate": 1.416343768433595e-05,
      "logits/chosen": -1.0681031942367554,
      "logits/rejected": -1.0986872911453247,
      "logps/chosen": -273.28936767578125,
      "logps/rejected": -281.49066162109375,
      "loss": 0.5322,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.350847005844116,
      "rewards/margins": 0.7273110151290894,
      "rewards/rejected": -3.078158378601074,
      "step": 2175
    },
    {
      "epoch": 0.4722192136900249,
      "grad_norm": 13.375,
      "learning_rate": 1.4161185253328523e-05,
      "logits/chosen": -1.089279055595398,
      "logits/rejected": -1.0746757984161377,
      "logps/chosen": -263.72662353515625,
      "logps/rejected": -277.2108154296875,
      "loss": 0.5299,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.069314479827881,
      "rewards/margins": 0.809256374835968,
      "rewards/rejected": -2.878570795059204,
      "step": 2180
    },
    {
      "epoch": 0.47330228528105706,
      "grad_norm": 20.375,
      "learning_rate": 1.4158925552625917e-05,
      "logits/chosen": -1.0702050924301147,
      "logits/rejected": -1.082779884338379,
      "logps/chosen": -279.0769348144531,
      "logps/rejected": -291.29315185546875,
      "loss": 0.5128,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.4228997230529785,
      "rewards/margins": 0.9039355516433716,
      "rewards/rejected": -3.3268356323242188,
      "step": 2185
    },
    {
      "epoch": 0.47438535687208927,
      "grad_norm": 15.4375,
      "learning_rate": 1.4156658585450011e-05,
      "logits/chosen": -1.147486686706543,
      "logits/rejected": -1.1569888591766357,
      "logps/chosen": -253.35787963867188,
      "logps/rejected": -276.037109375,
      "loss": 0.4788,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -1.9070899486541748,
      "rewards/margins": 0.9946557879447937,
      "rewards/rejected": -2.9017457962036133,
      "step": 2190
    },
    {
      "epoch": 0.47546842846312143,
      "grad_norm": 16.25,
      "learning_rate": 1.4154384355033044e-05,
      "logits/chosen": -1.071740746498108,
      "logits/rejected": -1.1287717819213867,
      "logps/chosen": -259.92633056640625,
      "logps/rejected": -262.55120849609375,
      "loss": 0.5377,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.9569623470306396,
      "rewards/margins": 0.7173613905906677,
      "rewards/rejected": -2.674323558807373,
      "step": 2195
    },
    {
      "epoch": 0.4765515000541536,
      "grad_norm": 22.0,
      "learning_rate": 1.4152102864617607e-05,
      "logits/chosen": -0.971031665802002,
      "logits/rejected": -0.9972405433654785,
      "logps/chosen": -283.39385986328125,
      "logps/rejected": -291.48345947265625,
      "loss": 0.6073,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -2.007230043411255,
      "rewards/margins": 0.7387665510177612,
      "rewards/rejected": -2.7459967136383057,
      "step": 2200
    },
    {
      "epoch": 0.47763457164518575,
      "grad_norm": 14.6875,
      "learning_rate": 1.4149814117456642e-05,
      "logits/chosen": -1.209607481956482,
      "logits/rejected": -1.2080358266830444,
      "logps/chosen": -251.32528686523438,
      "logps/rejected": -264.61956787109375,
      "loss": 0.506,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4415740966796875,
      "rewards/margins": 1.042561650276184,
      "rewards/rejected": -3.484135389328003,
      "step": 2205
    },
    {
      "epoch": 0.4787176432362179,
      "grad_norm": 14.4375,
      "learning_rate": 1.4147518116813442e-05,
      "logits/chosen": -1.01505446434021,
      "logits/rejected": -1.0311514139175415,
      "logps/chosen": -277.7330322265625,
      "logps/rejected": -287.32769775390625,
      "loss": 0.5724,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.549198627471924,
      "rewards/margins": 0.8022290468215942,
      "rewards/rejected": -3.3514270782470703,
      "step": 2210
    },
    {
      "epoch": 0.47980071482725006,
      "grad_norm": 16.25,
      "learning_rate": 1.4145214865961643e-05,
      "logits/chosen": -0.9706375002861023,
      "logits/rejected": -0.9814649820327759,
      "logps/chosen": -270.646240234375,
      "logps/rejected": -273.70611572265625,
      "loss": 0.5265,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.289354085922241,
      "rewards/margins": 0.8449188470840454,
      "rewards/rejected": -3.1342735290527344,
      "step": 2215
    },
    {
      "epoch": 0.4808837864182822,
      "grad_norm": 15.0625,
      "learning_rate": 1.4142904368185213e-05,
      "logits/chosen": -1.0752966403961182,
      "logits/rejected": -1.0429799556732178,
      "logps/chosen": -245.03866577148438,
      "logps/rejected": -259.91033935546875,
      "loss": 0.5621,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1850104331970215,
      "rewards/margins": 0.7081643342971802,
      "rewards/rejected": -2.893174648284912,
      "step": 2220
    },
    {
      "epoch": 0.48196685800931444,
      "grad_norm": 15.6875,
      "learning_rate": 1.4140586626778451e-05,
      "logits/chosen": -0.9486624002456665,
      "logits/rejected": -0.9573456048965454,
      "logps/chosen": -265.75469970703125,
      "logps/rejected": -277.4949951171875,
      "loss": 0.5298,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.082129955291748,
      "rewards/margins": 0.9398897290229797,
      "rewards/rejected": -3.022019624710083,
      "step": 2225
    },
    {
      "epoch": 0.4830499296003466,
      "grad_norm": 15.6875,
      "learning_rate": 1.4138261645045998e-05,
      "logits/chosen": -0.8969489336013794,
      "logits/rejected": -0.8995112180709839,
      "logps/chosen": -270.11932373046875,
      "logps/rejected": -285.37921142578125,
      "loss": 0.5135,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.1500511169433594,
      "rewards/margins": 1.0443367958068848,
      "rewards/rejected": -3.1943881511688232,
      "step": 2230
    },
    {
      "epoch": 0.48413300119137875,
      "grad_norm": 21.625,
      "learning_rate": 1.4135929426302798e-05,
      "logits/chosen": -0.9076678156852722,
      "logits/rejected": -0.8600123524665833,
      "logps/chosen": -269.4095764160156,
      "logps/rejected": -287.6859436035156,
      "loss": 0.547,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.425595760345459,
      "rewards/margins": 0.7590791583061218,
      "rewards/rejected": -3.184675455093384,
      "step": 2235
    },
    {
      "epoch": 0.4852160727824109,
      "grad_norm": 14.0625,
      "learning_rate": 1.413358997387413e-05,
      "logits/chosen": -0.8245556950569153,
      "logits/rejected": -0.7453678250312805,
      "logps/chosen": -268.6802673339844,
      "logps/rejected": -279.0674743652344,
      "loss": 0.5043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1309525966644287,
      "rewards/margins": 0.7537658214569092,
      "rewards/rejected": -2.884718418121338,
      "step": 2240
    },
    {
      "epoch": 0.48629914437344307,
      "grad_norm": 19.875,
      "learning_rate": 1.413124329109558e-05,
      "logits/chosen": -0.8362806439399719,
      "logits/rejected": -0.8203395009040833,
      "logps/chosen": -262.59173583984375,
      "logps/rejected": -271.2304382324219,
      "loss": 0.5378,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.0512378215789795,
      "rewards/margins": 0.8086967468261719,
      "rewards/rejected": -2.8599343299865723,
      "step": 2245
    },
    {
      "epoch": 0.48738221596447523,
      "grad_norm": 15.625,
      "learning_rate": 1.4128889381313041e-05,
      "logits/chosen": -0.7915539145469666,
      "logits/rejected": -0.8081375956535339,
      "logps/chosen": -273.72943115234375,
      "logps/rejected": -297.0674743652344,
      "loss": 0.5471,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -2.2960784435272217,
      "rewards/margins": 0.8215058445930481,
      "rewards/rejected": -3.117584228515625,
      "step": 2250
    },
    {
      "epoch": 0.48846528755550744,
      "grad_norm": 14.9375,
      "learning_rate": 1.4126528247882714e-05,
      "logits/chosen": -0.8134397268295288,
      "logits/rejected": -0.8210073709487915,
      "logps/chosen": -249.0049285888672,
      "logps/rejected": -275.869384765625,
      "loss": 0.5304,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -1.5039196014404297,
      "rewards/margins": 0.9747133255004883,
      "rewards/rejected": -2.478632688522339,
      "step": 2255
    },
    {
      "epoch": 0.4895483591465396,
      "grad_norm": 17.375,
      "learning_rate": 1.41241598941711e-05,
      "logits/chosen": -0.7836515307426453,
      "logits/rejected": -0.6972833871841431,
      "logps/chosen": -251.0963592529297,
      "logps/rejected": -268.1458740234375,
      "loss": 0.5006,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.6276918649673462,
      "rewards/margins": 0.8516240119934082,
      "rewards/rejected": -2.479315996170044,
      "step": 2260
    },
    {
      "epoch": 0.49063143073757176,
      "grad_norm": 14.9375,
      "learning_rate": 1.412178432355499e-05,
      "logits/chosen": -0.9138312339782715,
      "logits/rejected": -0.8763836622238159,
      "logps/chosen": -254.766845703125,
      "logps/rejected": -264.5461120605469,
      "loss": 0.5281,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -1.9505252838134766,
      "rewards/margins": 0.7908778786659241,
      "rewards/rejected": -2.741403102874756,
      "step": 2265
    },
    {
      "epoch": 0.4917145023286039,
      "grad_norm": 19.5,
      "learning_rate": 1.4119401539421472e-05,
      "logits/chosen": -0.8379949331283569,
      "logits/rejected": -0.8086563944816589,
      "logps/chosen": -246.809326171875,
      "logps/rejected": -275.21954345703125,
      "loss": 0.5832,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -1.6563701629638672,
      "rewards/margins": 0.6442514657974243,
      "rewards/rejected": -2.300621747970581,
      "step": 2270
    },
    {
      "epoch": 0.4927975739196361,
      "grad_norm": 20.125,
      "learning_rate": 1.4117011545167909e-05,
      "logits/chosen": -0.9023043513298035,
      "logits/rejected": -0.8305550813674927,
      "logps/chosen": -258.28033447265625,
      "logps/rejected": -275.7666320800781,
      "loss": 0.5576,
      "rewards/accuracies": 0.716666579246521,
      "rewards/chosen": -1.7249521017074585,
      "rewards/margins": 0.7701574563980103,
      "rewards/rejected": -2.4951093196868896,
      "step": 2275
    },
    {
      "epoch": 0.49388064551066824,
      "grad_norm": 17.125,
      "learning_rate": 1.4114614344201952e-05,
      "logits/chosen": -0.9320415258407593,
      "logits/rejected": -0.9000755548477173,
      "logps/chosen": -245.9698028564453,
      "logps/rejected": -261.4046630859375,
      "loss": 0.6083,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -1.8807865381240845,
      "rewards/margins": 0.6565802693367004,
      "rewards/rejected": -2.5373668670654297,
      "step": 2280
    },
    {
      "epoch": 0.4949637171017004,
      "grad_norm": 14.75,
      "learning_rate": 1.411220993994153e-05,
      "logits/chosen": -1.00065016746521,
      "logits/rejected": -1.02578866481781,
      "logps/chosen": -242.2409210205078,
      "logps/rejected": -255.9967498779297,
      "loss": 0.5295,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -1.9171231985092163,
      "rewards/margins": 0.7963953614234924,
      "rewards/rejected": -2.7135186195373535,
      "step": 2285
    },
    {
      "epoch": 0.4960467886927326,
      "grad_norm": 18.75,
      "learning_rate": 1.4109798335814833e-05,
      "logits/chosen": -0.9709134101867676,
      "logits/rejected": -0.9430945515632629,
      "logps/chosen": -252.3046112060547,
      "logps/rejected": -267.477294921875,
      "loss": 0.5576,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.1372759342193604,
      "rewards/margins": 0.69530189037323,
      "rewards/rejected": -2.83257794380188,
      "step": 2290
    },
    {
      "epoch": 0.49712986028376477,
      "grad_norm": 18.0,
      "learning_rate": 1.4107379535260322e-05,
      "logits/chosen": -1.0049405097961426,
      "logits/rejected": -0.8439800143241882,
      "logps/chosen": -260.6210021972656,
      "logps/rejected": -290.84210205078125,
      "loss": 0.5298,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -1.8548133373260498,
      "rewards/margins": 0.859087347984314,
      "rewards/rejected": -2.7139010429382324,
      "step": 2295
    },
    {
      "epoch": 0.4982129318747969,
      "grad_norm": 14.375,
      "learning_rate": 1.4104953541726717e-05,
      "logits/chosen": -1.1072927713394165,
      "logits/rejected": -1.0745035409927368,
      "logps/chosen": -247.9993133544922,
      "logps/rejected": -271.0910339355469,
      "loss": 0.5175,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -1.9358408451080322,
      "rewards/margins": 0.834145724773407,
      "rewards/rejected": -2.769986629486084,
      "step": 2300
    },
    {
      "epoch": 0.4992960034658291,
      "grad_norm": 17.0,
      "learning_rate": 1.4102520358672999e-05,
      "logits/chosen": -1.1340234279632568,
      "logits/rejected": -1.112038493156433,
      "logps/chosen": -269.7098083496094,
      "logps/rejected": -284.02130126953125,
      "loss": 0.593,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -2.2439723014831543,
      "rewards/margins": 0.663448691368103,
      "rewards/rejected": -2.9074206352233887,
      "step": 2305
    },
    {
      "epoch": 0.5003790750568613,
      "grad_norm": 22.5,
      "learning_rate": 1.4100079989568392e-05,
      "logits/chosen": -1.0629037618637085,
      "logits/rejected": -1.007076621055603,
      "logps/chosen": -256.56756591796875,
      "logps/rejected": -273.8661193847656,
      "loss": 0.5039,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -1.9678175449371338,
      "rewards/margins": 0.8905720710754395,
      "rewards/rejected": -2.858389377593994,
      "step": 2310
    },
    {
      "epoch": 0.5014621466478935,
      "grad_norm": 15.125,
      "learning_rate": 1.4097632437892369e-05,
      "logits/chosen": -0.9884664416313171,
      "logits/rejected": -1.0358843803405762,
      "logps/chosen": -247.41311645507812,
      "logps/rejected": -262.13104248046875,
      "loss": 0.5137,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -1.6019569635391235,
      "rewards/margins": 0.8375879526138306,
      "rewards/rejected": -2.439545154571533,
      "step": 2315
    },
    {
      "epoch": 0.5025452182389256,
      "grad_norm": 16.75,
      "learning_rate": 1.4095177707134645e-05,
      "logits/chosen": -1.0005900859832764,
      "logits/rejected": -1.0318191051483154,
      "logps/chosen": -259.00286865234375,
      "logps/rejected": -273.0606384277344,
      "loss": 0.5025,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -1.523640513420105,
      "rewards/margins": 0.8045549392700195,
      "rewards/rejected": -2.328195571899414,
      "step": 2320
    },
    {
      "epoch": 0.5036282898299578,
      "grad_norm": 19.0,
      "learning_rate": 1.4092715800795173e-05,
      "logits/chosen": -0.9152177572250366,
      "logits/rejected": -0.9422056078910828,
      "logps/chosen": -260.77972412109375,
      "logps/rejected": -270.85137939453125,
      "loss": 0.5446,
      "rewards/accuracies": 0.6833333969116211,
      "rewards/chosen": -1.4271609783172607,
      "rewards/margins": 0.7416859865188599,
      "rewards/rejected": -2.16884708404541,
      "step": 2325
    },
    {
      "epoch": 0.5047113614209899,
      "grad_norm": 14.0625,
      "learning_rate": 1.4090246722384128e-05,
      "logits/chosen": -1.1880407333374023,
      "logits/rejected": -1.1500743627548218,
      "logps/chosen": -255.13375854492188,
      "logps/rejected": -260.45977783203125,
      "loss": 0.4827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7729116678237915,
      "rewards/margins": 0.952061653137207,
      "rewards/rejected": -2.724973201751709,
      "step": 2330
    },
    {
      "epoch": 0.5057944330120221,
      "grad_norm": 21.625,
      "learning_rate": 1.4087770475421923e-05,
      "logits/chosen": -1.1579668521881104,
      "logits/rejected": -1.1729729175567627,
      "logps/chosen": -260.23309326171875,
      "logps/rejected": -274.0466613769531,
      "loss": 0.5218,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -2.089104413986206,
      "rewards/margins": 0.73902428150177,
      "rewards/rejected": -2.8281285762786865,
      "step": 2335
    },
    {
      "epoch": 0.5068775046030543,
      "grad_norm": 17.125,
      "learning_rate": 1.4085287063439183e-05,
      "logits/chosen": -1.1692168712615967,
      "logits/rejected": -1.1263477802276611,
      "logps/chosen": -269.6738586425781,
      "logps/rejected": -284.6568603515625,
      "loss": 0.5054,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -2.0686168670654297,
      "rewards/margins": 0.8684026002883911,
      "rewards/rejected": -2.937019109725952,
      "step": 2340
    },
    {
      "epoch": 0.5079605761940864,
      "grad_norm": 16.875,
      "learning_rate": 1.4082796489976751e-05,
      "logits/chosen": -1.1720547676086426,
      "logits/rejected": -1.159746766090393,
      "logps/chosen": -262.5434875488281,
      "logps/rejected": -271.96136474609375,
      "loss": 0.5332,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.1618564128875732,
      "rewards/margins": 0.7414556741714478,
      "rewards/rejected": -2.9033122062683105,
      "step": 2345
    },
    {
      "epoch": 0.5090436477851186,
      "grad_norm": 16.375,
      "learning_rate": 1.4080298758585686e-05,
      "logits/chosen": -1.2488313913345337,
      "logits/rejected": -1.170011281967163,
      "logps/chosen": -253.569580078125,
      "logps/rejected": -272.914306640625,
      "loss": 0.4861,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -1.9460361003875732,
      "rewards/margins": 1.0654993057250977,
      "rewards/rejected": -3.01153564453125,
      "step": 2350
    },
    {
      "epoch": 0.5101267193761507,
      "grad_norm": 17.5,
      "learning_rate": 1.4077793872827242e-05,
      "logits/chosen": -1.2818422317504883,
      "logits/rejected": -1.3502733707427979,
      "logps/chosen": -261.7262878417969,
      "logps/rejected": -274.4891052246094,
      "loss": 0.5538,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -2.6964211463928223,
      "rewards/margins": 0.7361631989479065,
      "rewards/rejected": -3.432584047317505,
      "step": 2355
    },
    {
      "epoch": 0.5112097909671829,
      "grad_norm": 17.375,
      "learning_rate": 1.4075281836272886e-05,
      "logits/chosen": -1.1733360290527344,
      "logits/rejected": -1.198250651359558,
      "logps/chosen": -270.322265625,
      "logps/rejected": -290.5069580078125,
      "loss": 0.5127,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.6180355548858643,
      "rewards/margins": 0.9724578857421875,
      "rewards/rejected": -3.590493679046631,
      "step": 2360
    },
    {
      "epoch": 0.512292862558215,
      "grad_norm": 21.0,
      "learning_rate": 1.4072762652504272e-05,
      "logits/chosen": -1.2406384944915771,
      "logits/rejected": -1.1846983432769775,
      "logps/chosen": -279.33648681640625,
      "logps/rejected": -293.31146240234375,
      "loss": 0.5488,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.866607189178467,
      "rewards/margins": 0.8488700985908508,
      "rewards/rejected": -3.715477466583252,
      "step": 2365
    },
    {
      "epoch": 0.5133759341492473,
      "grad_norm": 18.875,
      "learning_rate": 1.4070236325113246e-05,
      "logits/chosen": -1.2921102046966553,
      "logits/rejected": -1.1708396673202515,
      "logps/chosen": -255.14169311523438,
      "logps/rejected": -270.57110595703125,
      "loss": 0.5071,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.173652172088623,
      "rewards/margins": 0.8671897649765015,
      "rewards/rejected": -3.040842056274414,
      "step": 2370
    },
    {
      "epoch": 0.5144590057402795,
      "grad_norm": 17.375,
      "learning_rate": 1.4067702857701844e-05,
      "logits/chosen": -1.2056628465652466,
      "logits/rejected": -1.2068570852279663,
      "logps/chosen": -283.60711669921875,
      "logps/rejected": -285.98419189453125,
      "loss": 0.4958,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.6288321018218994,
      "rewards/margins": 1.0367668867111206,
      "rewards/rejected": -3.6655991077423096,
      "step": 2375
    },
    {
      "epoch": 0.5155420773313116,
      "grad_norm": 18.875,
      "learning_rate": 1.4065162253882273e-05,
      "logits/chosen": -1.2193626165390015,
      "logits/rejected": -1.2186553478240967,
      "logps/chosen": -290.9107666015625,
      "logps/rejected": -294.64813232421875,
      "loss": 0.5373,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.6732077598571777,
      "rewards/margins": 0.8831768035888672,
      "rewards/rejected": -3.556384325027466,
      "step": 2380
    },
    {
      "epoch": 0.5166251489223438,
      "grad_norm": 19.625,
      "learning_rate": 1.4062614517276927e-05,
      "logits/chosen": -1.1000338792800903,
      "logits/rejected": -1.1505568027496338,
      "logps/chosen": -287.1487731933594,
      "logps/rejected": -300.97430419921875,
      "loss": 0.5573,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -2.790944814682007,
      "rewards/margins": 0.6882976293563843,
      "rewards/rejected": -3.4792423248291016,
      "step": 2385
    },
    {
      "epoch": 0.517708220513376,
      "grad_norm": 25.375,
      "learning_rate": 1.406005965151836e-05,
      "logits/chosen": -1.273866891860962,
      "logits/rejected": -1.1833178997039795,
      "logps/chosen": -275.7812805175781,
      "logps/rejected": -290.17535400390625,
      "loss": 0.5399,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.8472423553466797,
      "rewards/margins": 0.9444856643676758,
      "rewards/rejected": -3.7917277812957764,
      "step": 2390
    },
    {
      "epoch": 0.5187912921044081,
      "grad_norm": 20.5,
      "learning_rate": 1.4057497660249295e-05,
      "logits/chosen": -1.1401675939559937,
      "logits/rejected": -1.0825154781341553,
      "logps/chosen": -279.6927185058594,
      "logps/rejected": -298.4749450683594,
      "loss": 0.5711,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.4783551692962646,
      "rewards/margins": 0.7835859060287476,
      "rewards/rejected": -3.2619411945343018,
      "step": 2395
    },
    {
      "epoch": 0.5198743636954403,
      "grad_norm": 17.625,
      "learning_rate": 1.4054928547122614e-05,
      "logits/chosen": -1.1344046592712402,
      "logits/rejected": -1.1020578145980835,
      "logps/chosen": -249.3963623046875,
      "logps/rejected": -267.66497802734375,
      "loss": 0.563,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.291019916534424,
      "rewards/margins": 0.7891637086868286,
      "rewards/rejected": -3.080183744430542,
      "step": 2400
    },
    {
      "epoch": 0.5209574352864724,
      "grad_norm": 17.625,
      "learning_rate": 1.4052352315801355e-05,
      "logits/chosen": -1.2270336151123047,
      "logits/rejected": -1.186764121055603,
      "logps/chosen": -266.89508056640625,
      "logps/rejected": -285.8085021972656,
      "loss": 0.5359,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4620561599731445,
      "rewards/margins": 0.7411717176437378,
      "rewards/rejected": -3.203227996826172,
      "step": 2405
    },
    {
      "epoch": 0.5220405068775046,
      "grad_norm": 18.625,
      "learning_rate": 1.4049768969958702e-05,
      "logits/chosen": -1.1717994213104248,
      "logits/rejected": -1.1263314485549927,
      "logps/chosen": -263.16204833984375,
      "logps/rejected": -291.2176818847656,
      "loss": 0.4801,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.164123773574829,
      "rewards/margins": 1.1478230953216553,
      "rewards/rejected": -3.3119468688964844,
      "step": 2410
    },
    {
      "epoch": 0.5231235784685367,
      "grad_norm": 18.0,
      "learning_rate": 1.4047178513277986e-05,
      "logits/chosen": -1.2806308269500732,
      "logits/rejected": -1.2395483255386353,
      "logps/chosen": -258.0469970703125,
      "logps/rejected": -276.2000427246094,
      "loss": 0.5294,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -2.497792959213257,
      "rewards/margins": 0.9103351831436157,
      "rewards/rejected": -3.408128261566162,
      "step": 2415
    },
    {
      "epoch": 0.5242066500595689,
      "grad_norm": 18.875,
      "learning_rate": 1.4044580949452673e-05,
      "logits/chosen": -1.1372497081756592,
      "logits/rejected": -1.086006760597229,
      "logps/chosen": -264.977294921875,
      "logps/rejected": -275.1259460449219,
      "loss": 0.5357,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4277334213256836,
      "rewards/margins": 0.8121234178543091,
      "rewards/rejected": -3.2398571968078613,
      "step": 2420
    },
    {
      "epoch": 0.5252897216506011,
      "grad_norm": 17.25,
      "learning_rate": 1.4041976282186363e-05,
      "logits/chosen": -1.1399552822113037,
      "logits/rejected": -1.18049955368042,
      "logps/chosen": -271.74932861328125,
      "logps/rejected": -282.104736328125,
      "loss": 0.4782,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.4036688804626465,
      "rewards/margins": 1.0835347175598145,
      "rewards/rejected": -3.487203598022461,
      "step": 2425
    },
    {
      "epoch": 0.5263727932416332,
      "grad_norm": 18.375,
      "learning_rate": 1.4039364515192787e-05,
      "logits/chosen": -1.213824987411499,
      "logits/rejected": -1.1768776178359985,
      "logps/chosen": -284.076904296875,
      "logps/rejected": -288.19091796875,
      "loss": 0.5078,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.9531478881835938,
      "rewards/margins": 0.9449865221977234,
      "rewards/rejected": -3.89813494682312,
      "step": 2430
    },
    {
      "epoch": 0.5274558648326655,
      "grad_norm": 23.625,
      "learning_rate": 1.4036745652195798e-05,
      "logits/chosen": -1.1784080266952515,
      "logits/rejected": -1.1619542837142944,
      "logps/chosen": -281.17352294921875,
      "logps/rejected": -305.5094299316406,
      "loss": 0.4927,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.7011005878448486,
      "rewards/margins": 1.0632209777832031,
      "rewards/rejected": -3.7643210887908936,
      "step": 2435
    },
    {
      "epoch": 0.5285389364236976,
      "grad_norm": 17.25,
      "learning_rate": 1.4034119696929367e-05,
      "logits/chosen": -1.1937620639801025,
      "logits/rejected": -1.1498768329620361,
      "logps/chosen": -299.97198486328125,
      "logps/rejected": -309.3303527832031,
      "loss": 0.5457,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.949101209640503,
      "rewards/margins": 0.749502420425415,
      "rewards/rejected": -3.698603868484497,
      "step": 2440
    },
    {
      "epoch": 0.5296220080147298,
      "grad_norm": 20.25,
      "learning_rate": 1.4031486653137573e-05,
      "logits/chosen": -1.2767560482025146,
      "logits/rejected": -1.2219313383102417,
      "logps/chosen": -271.7012939453125,
      "logps/rejected": -296.23638916015625,
      "loss": 0.5231,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": -2.8311269283294678,
      "rewards/margins": 0.9089571237564087,
      "rewards/rejected": -3.740084171295166,
      "step": 2445
    },
    {
      "epoch": 0.530705079605762,
      "grad_norm": 16.75,
      "learning_rate": 1.4028846524574602e-05,
      "logits/chosen": -1.299161672592163,
      "logits/rejected": -1.3088141679763794,
      "logps/chosen": -269.85479736328125,
      "logps/rejected": -278.43145751953125,
      "loss": 0.5572,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.5213124752044678,
      "rewards/margins": 0.7271442413330078,
      "rewards/rejected": -3.2484564781188965,
      "step": 2450
    },
    {
      "epoch": 0.5317881511967941,
      "grad_norm": 19.5,
      "learning_rate": 1.402619931500475e-05,
      "logits/chosen": -1.2652719020843506,
      "logits/rejected": -1.188877820968628,
      "logps/chosen": -292.051513671875,
      "logps/rejected": -289.6279602050781,
      "loss": 0.5433,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.726522207260132,
      "rewards/margins": 0.7785870432853699,
      "rewards/rejected": -3.5051090717315674,
      "step": 2455
    },
    {
      "epoch": 0.5328712227878263,
      "grad_norm": 17.25,
      "learning_rate": 1.4023545028202401e-05,
      "logits/chosen": -1.1737675666809082,
      "logits/rejected": -1.1012470722198486,
      "logps/chosen": -268.63092041015625,
      "logps/rejected": -288.1087646484375,
      "loss": 0.5582,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -2.4834885597229004,
      "rewards/margins": 0.885968804359436,
      "rewards/rejected": -3.369457244873047,
      "step": 2460
    },
    {
      "epoch": 0.5339542943788584,
      "grad_norm": 15.1875,
      "learning_rate": 1.4020883667952027e-05,
      "logits/chosen": -1.135318636894226,
      "logits/rejected": -1.154854655265808,
      "logps/chosen": -263.2058410644531,
      "logps/rejected": -281.10076904296875,
      "loss": 0.4766,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.0308899879455566,
      "rewards/margins": 1.1274945735931396,
      "rewards/rejected": -3.1583850383758545,
      "step": 2465
    },
    {
      "epoch": 0.5350373659698906,
      "grad_norm": 16.75,
      "learning_rate": 1.4018215238048195e-05,
      "logits/chosen": -1.1187195777893066,
      "logits/rejected": -1.0999598503112793,
      "logps/chosen": -257.6288757324219,
      "logps/rejected": -278.5059814453125,
      "loss": 0.481,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -2.1599669456481934,
      "rewards/margins": 0.9838963747024536,
      "rewards/rejected": -3.1438632011413574,
      "step": 2470
    },
    {
      "epoch": 0.5361204375609228,
      "grad_norm": 18.5,
      "learning_rate": 1.4015539742295548e-05,
      "logits/chosen": -1.0990655422210693,
      "logits/rejected": -1.1162766218185425,
      "logps/chosen": -268.9720764160156,
      "logps/rejected": -280.237548828125,
      "loss": 0.5673,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.3344597816467285,
      "rewards/margins": 0.7798570394515991,
      "rewards/rejected": -3.1143171787261963,
      "step": 2475
    },
    {
      "epoch": 0.5372035091519549,
      "grad_norm": 23.625,
      "learning_rate": 1.4012857184508799e-05,
      "logits/chosen": -1.1017783880233765,
      "logits/rejected": -1.0960228443145752,
      "logps/chosen": -279.4531555175781,
      "logps/rejected": -302.75689697265625,
      "loss": 0.5357,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -2.434126138687134,
      "rewards/margins": 0.7007744312286377,
      "rewards/rejected": -3.1349005699157715,
      "step": 2480
    },
    {
      "epoch": 0.5382865807429871,
      "grad_norm": 17.5,
      "learning_rate": 1.4010167568512737e-05,
      "logits/chosen": -1.0801059007644653,
      "logits/rejected": -1.1211836338043213,
      "logps/chosen": -263.80145263671875,
      "logps/rejected": -267.22283935546875,
      "loss": 0.5411,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.491584062576294,
      "rewards/margins": 0.7681187987327576,
      "rewards/rejected": -3.259702682495117,
      "step": 2485
    },
    {
      "epoch": 0.5393696523340192,
      "grad_norm": 14.6875,
      "learning_rate": 1.4007470898142208e-05,
      "logits/chosen": -1.2282084226608276,
      "logits/rejected": -1.1592363119125366,
      "logps/chosen": -264.7639465332031,
      "logps/rejected": -280.2796630859375,
      "loss": 0.589,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -2.526151180267334,
      "rewards/margins": 0.5851426124572754,
      "rewards/rejected": -3.1112937927246094,
      "step": 2490
    },
    {
      "epoch": 0.5404527239250514,
      "grad_norm": 14.375,
      "learning_rate": 1.4004767177242122e-05,
      "logits/chosen": -1.235616683959961,
      "logits/rejected": -1.1085262298583984,
      "logps/chosen": -275.05108642578125,
      "logps/rejected": -287.03594970703125,
      "loss": 0.5088,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.323539972305298,
      "rewards/margins": 0.8854660987854004,
      "rewards/rejected": -3.209005832672119,
      "step": 2495
    },
    {
      "epoch": 0.5415357955160837,
      "grad_norm": 22.125,
      "learning_rate": 1.4002056409667433e-05,
      "logits/chosen": -1.003310203552246,
      "logits/rejected": -1.078144907951355,
      "logps/chosen": -281.384521484375,
      "logps/rejected": -280.98876953125,
      "loss": 0.4946,
      "rewards/accuracies": 0.7666667699813843,
      "rewards/chosen": -2.2032108306884766,
      "rewards/margins": 0.8824064135551453,
      "rewards/rejected": -3.0856173038482666,
      "step": 2500
    },
    {
      "epoch": 0.5415357955160837,
      "eval_logits/chosen": -1.1923750638961792,
      "eval_logits/rejected": -1.125585675239563,
      "eval_logps/chosen": -260.89251708984375,
      "eval_logps/rejected": -280.0216369628906,
      "eval_loss": 0.5688720345497131,
      "eval_rewards/accuracies": 0.6449999809265137,
      "eval_rewards/chosen": -2.334561824798584,
      "eval_rewards/margins": 0.7836636900901794,
      "eval_rewards/rejected": -3.1182260513305664,
      "eval_runtime": 14.4162,
      "eval_samples_per_second": 13.873,
      "eval_steps_per_second": 13.873,
      "step": 2500
    },
    {
      "epoch": 0.5426188671071158,
      "grad_norm": 15.0,
      "learning_rate": 1.3999338599283155e-05,
      "logits/chosen": -1.2879607677459717,
      "logits/rejected": -1.21940016746521,
      "logps/chosen": -275.55352783203125,
      "logps/rejected": -301.01416015625,
      "loss": 0.4971,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.963222026824951,
      "rewards/margins": 0.7738688588142395,
      "rewards/rejected": -3.737091064453125,
      "step": 2505
    },
    {
      "epoch": 0.543701938698148,
      "grad_norm": 20.0,
      "learning_rate": 1.3996613749964332e-05,
      "logits/chosen": -1.1201732158660889,
      "logits/rejected": -1.0357739925384521,
      "logps/chosen": -286.0970764160156,
      "logps/rejected": -295.419921875,
      "loss": 0.5226,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.6683483123779297,
      "rewards/margins": 0.6948339939117432,
      "rewards/rejected": -3.363182544708252,
      "step": 2510
    },
    {
      "epoch": 0.5447850102891801,
      "grad_norm": 19.375,
      "learning_rate": 1.3993881865596053e-05,
      "logits/chosen": -1.1815518140792847,
      "logits/rejected": -1.1239241361618042,
      "logps/chosen": -264.0482482910156,
      "logps/rejected": -296.6741943359375,
      "loss": 0.5046,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.294323205947876,
      "rewards/margins": 1.0019557476043701,
      "rewards/rejected": -3.296279191970825,
      "step": 2515
    },
    {
      "epoch": 0.5458680818802123,
      "grad_norm": 24.0,
      "learning_rate": 1.3991142950073428e-05,
      "logits/chosen": -1.199709177017212,
      "logits/rejected": -1.145442247390747,
      "logps/chosen": -287.12933349609375,
      "logps/rejected": -313.1647644042969,
      "loss": 0.5829,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -2.692422389984131,
      "rewards/margins": 0.7898037433624268,
      "rewards/rejected": -3.4822254180908203,
      "step": 2520
    },
    {
      "epoch": 0.5469511534712445,
      "grad_norm": 20.5,
      "learning_rate": 1.39883970073016e-05,
      "logits/chosen": -1.2963730096817017,
      "logits/rejected": -1.2808277606964111,
      "logps/chosen": -272.4757080078125,
      "logps/rejected": -284.78985595703125,
      "loss": 0.5503,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.756075859069824,
      "rewards/margins": 0.8731797933578491,
      "rewards/rejected": -3.6292552947998047,
      "step": 2525
    },
    {
      "epoch": 0.5480342250622766,
      "grad_norm": 19.5,
      "learning_rate": 1.3985644041195731e-05,
      "logits/chosen": -1.3085216283798218,
      "logits/rejected": -1.281714677810669,
      "logps/chosen": -267.395751953125,
      "logps/rejected": -282.99530029296875,
      "loss": 0.5706,
      "rewards/accuracies": 0.6416667103767395,
      "rewards/chosen": -2.8004801273345947,
      "rewards/margins": 0.6629133820533752,
      "rewards/rejected": -3.463393449783325,
      "step": 2530
    },
    {
      "epoch": 0.5491172966533088,
      "grad_norm": 13.125,
      "learning_rate": 1.3982884055680989e-05,
      "logits/chosen": -1.2547587156295776,
      "logits/rejected": -1.217772126197815,
      "logps/chosen": -263.7702331542969,
      "logps/rejected": -280.29144287109375,
      "loss": 0.5044,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.4308812618255615,
      "rewards/margins": 0.9005125761032104,
      "rewards/rejected": -3.3313934803009033,
      "step": 2535
    },
    {
      "epoch": 0.5502003682443409,
      "grad_norm": 26.5,
      "learning_rate": 1.398011705469256e-05,
      "logits/chosen": -1.2505500316619873,
      "logits/rejected": -1.2608507871627808,
      "logps/chosen": -273.4361267089844,
      "logps/rejected": -283.1285705566406,
      "loss": 0.5694,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.476188898086548,
      "rewards/margins": 0.8091329336166382,
      "rewards/rejected": -3.2853221893310547,
      "step": 2540
    },
    {
      "epoch": 0.5512834398353731,
      "grad_norm": 20.125,
      "learning_rate": 1.3977343042175626e-05,
      "logits/chosen": -1.3015434741973877,
      "logits/rejected": -1.2638928890228271,
      "logps/chosen": -277.33587646484375,
      "logps/rejected": -307.6432800292969,
      "loss": 0.4484,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.278602361679077,
      "rewards/margins": 1.260035514831543,
      "rewards/rejected": -3.53863787651062,
      "step": 2545
    },
    {
      "epoch": 0.5523665114264052,
      "grad_norm": 17.0,
      "learning_rate": 1.3974562022085372e-05,
      "logits/chosen": -1.3354713916778564,
      "logits/rejected": -1.296318531036377,
      "logps/chosen": -282.06488037109375,
      "logps/rejected": -310.17181396484375,
      "loss": 0.4715,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.0964977741241455,
      "rewards/margins": 1.1536476612091064,
      "rewards/rejected": -4.25014591217041,
      "step": 2550
    },
    {
      "epoch": 0.5534495830174374,
      "grad_norm": 16.625,
      "learning_rate": 1.3971773998386966e-05,
      "logits/chosen": -1.463468074798584,
      "logits/rejected": -1.4334781169891357,
      "logps/chosen": -258.1885986328125,
      "logps/rejected": -277.0633544921875,
      "loss": 0.5009,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.643908977508545,
      "rewards/margins": 0.97931969165802,
      "rewards/rejected": -3.6232287883758545,
      "step": 2555
    },
    {
      "epoch": 0.5545326546084696,
      "grad_norm": 16.25,
      "learning_rate": 1.3968978975055571e-05,
      "logits/chosen": -1.4534101486206055,
      "logits/rejected": -1.3638770580291748,
      "logps/chosen": -275.76641845703125,
      "logps/rejected": -293.88397216796875,
      "loss": 0.4671,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -2.82440447807312,
      "rewards/margins": 1.1067661046981812,
      "rewards/rejected": -3.931170701980591,
      "step": 2560
    },
    {
      "epoch": 0.5556157261995018,
      "grad_norm": 18.75,
      "learning_rate": 1.3966176956076325e-05,
      "logits/chosen": -1.4873542785644531,
      "logits/rejected": -1.4627398252487183,
      "logps/chosen": -272.2625427246094,
      "logps/rejected": -290.911865234375,
      "loss": 0.4749,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.287092447280884,
      "rewards/margins": 1.0136659145355225,
      "rewards/rejected": -3.3007583618164062,
      "step": 2565
    },
    {
      "epoch": 0.556698797790534,
      "grad_norm": 24.875,
      "learning_rate": 1.396336794544434e-05,
      "logits/chosen": -1.4713562726974487,
      "logits/rejected": -1.4977824687957764,
      "logps/chosen": -282.0677795410156,
      "logps/rejected": -296.81048583984375,
      "loss": 0.6144,
      "rewards/accuracies": 0.6249999403953552,
      "rewards/chosen": -2.8876712322235107,
      "rewards/margins": 0.7700803279876709,
      "rewards/rejected": -3.6577515602111816,
      "step": 2570
    },
    {
      "epoch": 0.5577818693815662,
      "grad_norm": 20.875,
      "learning_rate": 1.3960551947164694e-05,
      "logits/chosen": -1.545137643814087,
      "logits/rejected": -1.4730534553527832,
      "logps/chosen": -275.91949462890625,
      "logps/rejected": -289.9657897949219,
      "loss": 0.5513,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -3.084303379058838,
      "rewards/margins": 0.9303779602050781,
      "rewards/rejected": -4.014681339263916,
      "step": 2575
    },
    {
      "epoch": 0.5588649409725983,
      "grad_norm": 19.25,
      "learning_rate": 1.3957728965252436e-05,
      "logits/chosen": -1.4221789836883545,
      "logits/rejected": -1.3940889835357666,
      "logps/chosen": -263.9010009765625,
      "logps/rejected": -279.34735107421875,
      "loss": 0.5498,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.757960081100464,
      "rewards/margins": 0.7786729335784912,
      "rewards/rejected": -3.536633014678955,
      "step": 2580
    },
    {
      "epoch": 0.5599480125636305,
      "grad_norm": 15.375,
      "learning_rate": 1.3954899003732569e-05,
      "logits/chosen": -1.493186116218567,
      "logits/rejected": -1.4391547441482544,
      "logps/chosen": -275.31243896484375,
      "logps/rejected": -299.6541748046875,
      "loss": 0.4906,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.5592710971832275,
      "rewards/margins": 1.220603585243225,
      "rewards/rejected": -3.779874324798584,
      "step": 2585
    },
    {
      "epoch": 0.5610310841546626,
      "grad_norm": 13.3125,
      "learning_rate": 1.3952062066640044e-05,
      "logits/chosen": -1.2623193264007568,
      "logits/rejected": -1.2323236465454102,
      "logps/chosen": -270.7505798339844,
      "logps/rejected": -284.2131042480469,
      "loss": 0.4647,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.0983853340148926,
      "rewards/margins": 1.1600486040115356,
      "rewards/rejected": -3.2584335803985596,
      "step": 2590
    },
    {
      "epoch": 0.5621141557456948,
      "grad_norm": 21.25,
      "learning_rate": 1.3949218158019759e-05,
      "logits/chosen": -1.3728193044662476,
      "logits/rejected": -1.3693296909332275,
      "logps/chosen": -270.5417175292969,
      "logps/rejected": -290.536865234375,
      "loss": 0.5221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.617722988128662,
      "rewards/margins": 0.8561124801635742,
      "rewards/rejected": -3.473834991455078,
      "step": 2595
    },
    {
      "epoch": 0.563197227336727,
      "grad_norm": 17.875,
      "learning_rate": 1.3946367281926557e-05,
      "logits/chosen": -1.400892972946167,
      "logits/rejected": -1.383202075958252,
      "logps/chosen": -278.2144775390625,
      "logps/rejected": -288.983642578125,
      "loss": 0.5073,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.450247287750244,
      "rewards/margins": 0.9537274241447449,
      "rewards/rejected": -3.403975009918213,
      "step": 2600
    },
    {
      "epoch": 0.5642802989277591,
      "grad_norm": 21.375,
      "learning_rate": 1.3943509442425208e-05,
      "logits/chosen": -1.3595492839813232,
      "logits/rejected": -1.3609094619750977,
      "logps/chosen": -278.9186706542969,
      "logps/rejected": -286.2938537597656,
      "loss": 0.5316,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.453007221221924,
      "rewards/margins": 0.9979684948921204,
      "rewards/rejected": -3.4509758949279785,
      "step": 2605
    },
    {
      "epoch": 0.5653633705187913,
      "grad_norm": 17.625,
      "learning_rate": 1.3940644643590414e-05,
      "logits/chosen": -1.2683584690093994,
      "logits/rejected": -1.306173324584961,
      "logps/chosen": -275.75732421875,
      "logps/rejected": -277.69696044921875,
      "loss": 0.5133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.0732548236846924,
      "rewards/margins": 0.7988507151603699,
      "rewards/rejected": -2.872105360031128,
      "step": 2610
    },
    {
      "epoch": 0.5664464421098234,
      "grad_norm": 15.4375,
      "learning_rate": 1.3937772889506804e-05,
      "logits/chosen": -1.3713321685791016,
      "logits/rejected": -1.358755111694336,
      "logps/chosen": -269.61932373046875,
      "logps/rejected": -283.7223205566406,
      "loss": 0.4827,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.3608086109161377,
      "rewards/margins": 0.9793470501899719,
      "rewards/rejected": -3.340155839920044,
      "step": 2615
    },
    {
      "epoch": 0.5675295137008556,
      "grad_norm": 21.25,
      "learning_rate": 1.3934894184268914e-05,
      "logits/chosen": -1.3621547222137451,
      "logits/rejected": -1.3839081525802612,
      "logps/chosen": -264.71826171875,
      "logps/rejected": -283.52239990234375,
      "loss": 0.4553,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.4962544441223145,
      "rewards/margins": 1.0829925537109375,
      "rewards/rejected": -3.579247236251831,
      "step": 2620
    },
    {
      "epoch": 0.5686125852918877,
      "grad_norm": 15.6875,
      "learning_rate": 1.3932008531981197e-05,
      "logits/chosen": -1.336822748184204,
      "logits/rejected": -1.299045443534851,
      "logps/chosen": -271.98773193359375,
      "logps/rejected": -282.68896484375,
      "loss": 0.4724,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.326476573944092,
      "rewards/margins": 0.9762721061706543,
      "rewards/rejected": -3.302748918533325,
      "step": 2625
    },
    {
      "epoch": 0.56969565688292,
      "grad_norm": 15.1875,
      "learning_rate": 1.3929115936758014e-05,
      "logits/chosen": -1.3806976079940796,
      "logits/rejected": -1.3436311483383179,
      "logps/chosen": -291.61834716796875,
      "logps/rejected": -303.49493408203125,
      "loss": 0.4352,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -2.2979390621185303,
      "rewards/margins": 1.2696131467819214,
      "rewards/rejected": -3.567552089691162,
      "step": 2630
    },
    {
      "epoch": 0.5707787284739522,
      "grad_norm": 19.0,
      "learning_rate": 1.3926216402723615e-05,
      "logits/chosen": -1.5828664302825928,
      "logits/rejected": -1.5783888101577759,
      "logps/chosen": -276.43365478515625,
      "logps/rejected": -281.97564697265625,
      "loss": 0.5372,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.547935962677002,
      "rewards/margins": 0.9466825723648071,
      "rewards/rejected": -3.4946179389953613,
      "step": 2635
    },
    {
      "epoch": 0.5718618000649843,
      "grad_norm": 18.625,
      "learning_rate": 1.3923309934012156e-05,
      "logits/chosen": -1.51369309425354,
      "logits/rejected": -1.4986236095428467,
      "logps/chosen": -289.58477783203125,
      "logps/rejected": -296.88616943359375,
      "loss": 0.5023,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.577951669692993,
      "rewards/margins": 0.9688816070556641,
      "rewards/rejected": -3.5468335151672363,
      "step": 2640
    },
    {
      "epoch": 0.5729448716560165,
      "grad_norm": 24.375,
      "learning_rate": 1.3920396534767672e-05,
      "logits/chosen": -1.3972022533416748,
      "logits/rejected": -1.3888123035430908,
      "logps/chosen": -270.3268127441406,
      "logps/rejected": -286.6598205566406,
      "loss": 0.5809,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.6432809829711914,
      "rewards/margins": 0.8474627733230591,
      "rewards/rejected": -3.4907443523406982,
      "step": 2645
    },
    {
      "epoch": 0.5740279432470486,
      "grad_norm": 18.375,
      "learning_rate": 1.391747620914408e-05,
      "logits/chosen": -1.338687539100647,
      "logits/rejected": -1.3430861234664917,
      "logps/chosen": -287.9543151855469,
      "logps/rejected": -301.655517578125,
      "loss": 0.5579,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.6566271781921387,
      "rewards/margins": 0.9331556558609009,
      "rewards/rejected": -3.58978271484375,
      "step": 2650
    },
    {
      "epoch": 0.5751110148380808,
      "grad_norm": 25.25,
      "learning_rate": 1.3914548961305176e-05,
      "logits/chosen": -1.6610047817230225,
      "logits/rejected": -1.53459894657135,
      "logps/chosen": -267.0198974609375,
      "logps/rejected": -284.98541259765625,
      "loss": 0.5922,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -2.5829687118530273,
      "rewards/margins": 0.6877350807189941,
      "rewards/rejected": -3.2707037925720215,
      "step": 2655
    },
    {
      "epoch": 0.576194086429113,
      "grad_norm": 16.625,
      "learning_rate": 1.3911614795424623e-05,
      "logits/chosen": -1.5049711465835571,
      "logits/rejected": -1.4424690008163452,
      "logps/chosen": -270.2715148925781,
      "logps/rejected": -289.05072021484375,
      "loss": 0.4641,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.7151341438293457,
      "rewards/margins": 1.025775671005249,
      "rewards/rejected": -3.7409095764160156,
      "step": 2660
    },
    {
      "epoch": 0.5772771580201451,
      "grad_norm": 19.75,
      "learning_rate": 1.3908673715685953e-05,
      "logits/chosen": -1.3408468961715698,
      "logits/rejected": -1.3398351669311523,
      "logps/chosen": -290.7549133300781,
      "logps/rejected": -298.1149597167969,
      "loss": 0.51,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.2451906204223633,
      "rewards/margins": 0.940830409526825,
      "rewards/rejected": -3.186021327972412,
      "step": 2665
    },
    {
      "epoch": 0.5783602296111773,
      "grad_norm": 18.125,
      "learning_rate": 1.3905725726282544e-05,
      "logits/chosen": -1.4507274627685547,
      "logits/rejected": -1.5295366048812866,
      "logps/chosen": -283.5868835449219,
      "logps/rejected": -277.77874755859375,
      "loss": 0.5356,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.5412542819976807,
      "rewards/margins": 0.8176851272583008,
      "rewards/rejected": -3.3589394092559814,
      "step": 2670
    },
    {
      "epoch": 0.5794433012022094,
      "grad_norm": 18.875,
      "learning_rate": 1.3902770831417639e-05,
      "logits/chosen": -1.4301835298538208,
      "logits/rejected": -1.4296001195907593,
      "logps/chosen": -272.76959228515625,
      "logps/rejected": -289.3567810058594,
      "loss": 0.521,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.6084980964660645,
      "rewards/margins": 0.894855797290802,
      "rewards/rejected": -3.5033538341522217,
      "step": 2675
    },
    {
      "epoch": 0.5805263727932416,
      "grad_norm": 16.75,
      "learning_rate": 1.389980903530432e-05,
      "logits/chosen": -1.3886183500289917,
      "logits/rejected": -1.365741491317749,
      "logps/chosen": -263.58697509765625,
      "logps/rejected": -276.1841735839844,
      "loss": 0.5167,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.5915274620056152,
      "rewards/margins": 0.8781619071960449,
      "rewards/rejected": -3.4696896076202393,
      "step": 2680
    },
    {
      "epoch": 0.5816094443842738,
      "grad_norm": 18.875,
      "learning_rate": 1.3896840342165512e-05,
      "logits/chosen": -1.3790405988693237,
      "logits/rejected": -1.3679085969924927,
      "logps/chosen": -283.07977294921875,
      "logps/rejected": -292.3308410644531,
      "loss": 0.5454,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -3.0345335006713867,
      "rewards/margins": 0.8823339343070984,
      "rewards/rejected": -3.91686749458313,
      "step": 2685
    },
    {
      "epoch": 0.5826925159753059,
      "grad_norm": 19.5,
      "learning_rate": 1.3893864756233965e-05,
      "logits/chosen": -1.2537953853607178,
      "logits/rejected": -1.247188687324524,
      "logps/chosen": -265.778564453125,
      "logps/rejected": -278.30450439453125,
      "loss": 0.563,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.606929302215576,
      "rewards/margins": 0.9644933938980103,
      "rewards/rejected": -3.571423292160034,
      "step": 2690
    },
    {
      "epoch": 0.5837755875663382,
      "grad_norm": 19.375,
      "learning_rate": 1.3890882281752271e-05,
      "logits/chosen": -1.2189313173294067,
      "logits/rejected": -1.1534725427627563,
      "logps/chosen": -283.80712890625,
      "logps/rejected": -287.1308898925781,
      "loss": 0.5204,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -2.7186551094055176,
      "rewards/margins": 0.8880051374435425,
      "rewards/rejected": -3.606660842895508,
      "step": 2695
    },
    {
      "epoch": 0.5848586591573703,
      "grad_norm": 18.875,
      "learning_rate": 1.3887892922972833e-05,
      "logits/chosen": -1.2053771018981934,
      "logits/rejected": -1.231943964958191,
      "logps/chosen": -278.5745849609375,
      "logps/rejected": -287.48358154296875,
      "loss": 0.4722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.777101516723633,
      "rewards/margins": 1.0555211305618286,
      "rewards/rejected": -3.832622528076172,
      "step": 2700
    },
    {
      "epoch": 0.5859417307484025,
      "grad_norm": 16.625,
      "learning_rate": 1.3884896684157872e-05,
      "logits/chosen": -1.148075819015503,
      "logits/rejected": -1.1228309869766235,
      "logps/chosen": -277.41839599609375,
      "logps/rejected": -300.2118225097656,
      "loss": 0.5519,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.54902982711792,
      "rewards/margins": 0.9889354705810547,
      "rewards/rejected": -3.5379652976989746,
      "step": 2705
    },
    {
      "epoch": 0.5870248023394347,
      "grad_norm": 13.25,
      "learning_rate": 1.3881893569579419e-05,
      "logits/chosen": -1.0493685007095337,
      "logits/rejected": -1.0710093975067139,
      "logps/chosen": -276.8389892578125,
      "logps/rejected": -296.43109130859375,
      "loss": 0.4475,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -2.7431435585021973,
      "rewards/margins": 1.2357041835784912,
      "rewards/rejected": -3.9788475036621094,
      "step": 2710
    },
    {
      "epoch": 0.5881078739304668,
      "grad_norm": 16.5,
      "learning_rate": 1.3878883583519312e-05,
      "logits/chosen": -1.1014423370361328,
      "logits/rejected": -1.1662561893463135,
      "logps/chosen": -264.5492858886719,
      "logps/rejected": -277.24884033203125,
      "loss": 0.4789,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4977641105651855,
      "rewards/margins": 0.9487814903259277,
      "rewards/rejected": -3.446545362472534,
      "step": 2715
    },
    {
      "epoch": 0.589190945521499,
      "grad_norm": 14.5625,
      "learning_rate": 1.3875866730269178e-05,
      "logits/chosen": -1.1840283870697021,
      "logits/rejected": -1.1631168127059937,
      "logps/chosen": -271.2416687011719,
      "logps/rejected": -287.696533203125,
      "loss": 0.4819,
      "rewards/accuracies": 0.7083333134651184,
      "rewards/chosen": -2.316281795501709,
      "rewards/margins": 0.9205352663993835,
      "rewards/rejected": -3.2368171215057373,
      "step": 2720
    },
    {
      "epoch": 0.5902740171125311,
      "grad_norm": 22.0,
      "learning_rate": 1.3872843014130446e-05,
      "logits/chosen": -1.3021008968353271,
      "logits/rejected": -1.2728103399276733,
      "logps/chosen": -281.10284423828125,
      "logps/rejected": -292.1086730957031,
      "loss": 0.5389,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.9606118202209473,
      "rewards/margins": 0.7361993789672852,
      "rewards/rejected": -3.6968111991882324,
      "step": 2725
    },
    {
      "epoch": 0.5913570887035633,
      "grad_norm": 25.125,
      "learning_rate": 1.386981243941432e-05,
      "logits/chosen": -1.5221951007843018,
      "logits/rejected": -1.4792042970657349,
      "logps/chosen": -275.75970458984375,
      "logps/rejected": -301.8537902832031,
      "loss": 0.468,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.065836191177368,
      "rewards/margins": 1.1738680601119995,
      "rewards/rejected": -4.239704608917236,
      "step": 2730
    },
    {
      "epoch": 0.5924401602945955,
      "grad_norm": 16.125,
      "learning_rate": 1.3866775010441788e-05,
      "logits/chosen": -1.52059006690979,
      "logits/rejected": -1.5820362567901611,
      "logps/chosen": -275.8177490234375,
      "logps/rejected": -286.29412841796875,
      "loss": 0.5152,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.7415173053741455,
      "rewards/margins": 0.9318100214004517,
      "rewards/rejected": -3.6733272075653076,
      "step": 2735
    },
    {
      "epoch": 0.5935232318856276,
      "grad_norm": 16.875,
      "learning_rate": 1.3863730731543611e-05,
      "logits/chosen": -1.6315393447875977,
      "logits/rejected": -1.651049256324768,
      "logps/chosen": -275.1454772949219,
      "logps/rejected": -281.51641845703125,
      "loss": 0.5302,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.983496904373169,
      "rewards/margins": 1.1332619190216064,
      "rewards/rejected": -4.116758823394775,
      "step": 2740
    },
    {
      "epoch": 0.5946063034766598,
      "grad_norm": 15.5,
      "learning_rate": 1.3860679607060318e-05,
      "logits/chosen": -1.6113427877426147,
      "logits/rejected": -1.587233304977417,
      "logps/chosen": -278.07073974609375,
      "logps/rejected": -303.70587158203125,
      "loss": 0.4686,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.902482271194458,
      "rewards/margins": 1.1017439365386963,
      "rewards/rejected": -4.0042266845703125,
      "step": 2745
    },
    {
      "epoch": 0.5956893750676919,
      "grad_norm": 18.75,
      "learning_rate": 1.3857621641342192e-05,
      "logits/chosen": -1.676335096359253,
      "logits/rejected": -1.6617177724838257,
      "logps/chosen": -288.92132568359375,
      "logps/rejected": -307.7901306152344,
      "loss": 0.5272,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.3721988201141357,
      "rewards/margins": 1.0002801418304443,
      "rewards/rejected": -4.372479438781738,
      "step": 2750
    },
    {
      "epoch": 0.5967724466587241,
      "grad_norm": 26.625,
      "learning_rate": 1.3854556838749276e-05,
      "logits/chosen": -1.4591116905212402,
      "logits/rejected": -1.4437226057052612,
      "logps/chosen": -300.22442626953125,
      "logps/rejected": -303.95037841796875,
      "loss": 0.6145,
      "rewards/accuracies": 0.6583333611488342,
      "rewards/chosen": -3.0389740467071533,
      "rewards/margins": 0.7195954918861389,
      "rewards/rejected": -3.7585692405700684,
      "step": 2755
    },
    {
      "epoch": 0.5978555182497564,
      "grad_norm": 22.75,
      "learning_rate": 1.385148520365136e-05,
      "logits/chosen": -1.6038129329681396,
      "logits/rejected": -1.524237036705017,
      "logps/chosen": -275.4639587402344,
      "logps/rejected": -302.1669921875,
      "loss": 0.5028,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -2.959007978439331,
      "rewards/margins": 1.1703730821609497,
      "rewards/rejected": -4.12938117980957,
      "step": 2760
    },
    {
      "epoch": 0.5989385898407885,
      "grad_norm": 14.4375,
      "learning_rate": 1.3848406740427973e-05,
      "logits/chosen": -1.651471495628357,
      "logits/rejected": -1.588604211807251,
      "logps/chosen": -277.19085693359375,
      "logps/rejected": -300.1969299316406,
      "loss": 0.4456,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.263015031814575,
      "rewards/margins": 1.1881943941116333,
      "rewards/rejected": -4.451210021972656,
      "step": 2765
    },
    {
      "epoch": 0.6000216614318207,
      "grad_norm": 17.5,
      "learning_rate": 1.3845321453468387e-05,
      "logits/chosen": -1.6759897470474243,
      "logits/rejected": -1.5946366786956787,
      "logps/chosen": -279.14617919921875,
      "logps/rejected": -305.675048828125,
      "loss": 0.4808,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -3.296905994415283,
      "rewards/margins": 1.0754616260528564,
      "rewards/rejected": -4.372367858886719,
      "step": 2770
    },
    {
      "epoch": 0.6011047330228528,
      "grad_norm": 14.625,
      "learning_rate": 1.3842229347171592e-05,
      "logits/chosen": -1.4791667461395264,
      "logits/rejected": -1.5894081592559814,
      "logps/chosen": -292.38067626953125,
      "logps/rejected": -287.4145202636719,
      "loss": 0.5254,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.1360795497894287,
      "rewards/margins": 1.0325839519500732,
      "rewards/rejected": -4.168663501739502,
      "step": 2775
    },
    {
      "epoch": 0.602187804613885,
      "grad_norm": 18.5,
      "learning_rate": 1.383913042594631e-05,
      "logits/chosen": -1.504873275756836,
      "logits/rejected": -1.504676103591919,
      "logps/chosen": -279.7005615234375,
      "logps/rejected": -283.04974365234375,
      "loss": 0.5404,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -3.2372965812683105,
      "rewards/margins": 0.8499137759208679,
      "rewards/rejected": -4.087210655212402,
      "step": 2780
    },
    {
      "epoch": 0.6032708762049172,
      "grad_norm": 22.625,
      "learning_rate": 1.3836024694210975e-05,
      "logits/chosen": -1.3055124282836914,
      "logits/rejected": -1.347794771194458,
      "logps/chosen": -274.82159423828125,
      "logps/rejected": -286.8946838378906,
      "loss": 0.5256,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.4972121715545654,
      "rewards/margins": 0.9983749389648438,
      "rewards/rejected": -3.495587110519409,
      "step": 2785
    },
    {
      "epoch": 0.6043539477959493,
      "grad_norm": 14.3125,
      "learning_rate": 1.3832912156393735e-05,
      "logits/chosen": -1.389756679534912,
      "logits/rejected": -1.3942017555236816,
      "logps/chosen": -268.50469970703125,
      "logps/rejected": -284.19720458984375,
      "loss": 0.5082,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.8596677780151367,
      "rewards/margins": 0.9721798896789551,
      "rewards/rejected": -3.83184814453125,
      "step": 2790
    },
    {
      "epoch": 0.6054370193869815,
      "grad_norm": 22.125,
      "learning_rate": 1.3829792816932438e-05,
      "logits/chosen": -1.2811977863311768,
      "logits/rejected": -1.3018964529037476,
      "logps/chosen": -277.97039794921875,
      "logps/rejected": -280.02423095703125,
      "loss": 0.5302,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.514758586883545,
      "rewards/margins": 0.8632100224494934,
      "rewards/rejected": -3.3779685497283936,
      "step": 2795
    },
    {
      "epoch": 0.6065200909780136,
      "grad_norm": 23.875,
      "learning_rate": 1.3826666680274633e-05,
      "logits/chosen": -1.2834758758544922,
      "logits/rejected": -1.2562090158462524,
      "logps/chosen": -265.6846923828125,
      "logps/rejected": -287.9786682128906,
      "loss": 0.5517,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -2.030292272567749,
      "rewards/margins": 0.9350635409355164,
      "rewards/rejected": -2.9653561115264893,
      "step": 2800
    },
    {
      "epoch": 0.6076031625690458,
      "grad_norm": 19.75,
      "learning_rate": 1.3823533750877561e-05,
      "logits/chosen": -1.2871074676513672,
      "logits/rejected": -1.3347464799880981,
      "logps/chosen": -281.1221008300781,
      "logps/rejected": -295.24468994140625,
      "loss": 0.479,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.5409634113311768,
      "rewards/margins": 1.0966793298721313,
      "rewards/rejected": -3.6376426219940186,
      "step": 2805
    },
    {
      "epoch": 0.6086862341600779,
      "grad_norm": 21.125,
      "learning_rate": 1.3820394033208143e-05,
      "logits/chosen": -1.197800874710083,
      "logits/rejected": -1.1721227169036865,
      "logps/chosen": -270.4149475097656,
      "logps/rejected": -298.02020263671875,
      "loss": 0.5137,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.356797218322754,
      "rewards/margins": 1.0045530796051025,
      "rewards/rejected": -3.3613502979278564,
      "step": 2810
    },
    {
      "epoch": 0.6097693057511101,
      "grad_norm": 23.625,
      "learning_rate": 1.3817247531742985e-05,
      "logits/chosen": -1.2793552875518799,
      "logits/rejected": -1.3682860136032104,
      "logps/chosen": -281.65203857421875,
      "logps/rejected": -285.86614990234375,
      "loss": 0.504,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.7620139122009277,
      "rewards/margins": 1.0946334600448608,
      "rewards/rejected": -3.8566479682922363,
      "step": 2815
    },
    {
      "epoch": 0.6108523773421423,
      "grad_norm": 22.0,
      "learning_rate": 1.381409425096836e-05,
      "logits/chosen": -1.2764034271240234,
      "logits/rejected": -1.1892130374908447,
      "logps/chosen": -275.1268615722656,
      "logps/rejected": -306.0076904296875,
      "loss": 0.4907,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.888550043106079,
      "rewards/margins": 0.9199217557907104,
      "rewards/rejected": -3.8084716796875,
      "step": 2820
    },
    {
      "epoch": 0.6119354489331745,
      "grad_norm": 16.75,
      "learning_rate": 1.3810934195380213e-05,
      "logits/chosen": -1.4154692888259888,
      "logits/rejected": -1.4166285991668701,
      "logps/chosen": -277.47637939453125,
      "logps/rejected": -294.8193359375,
      "loss": 0.5409,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.9505867958068848,
      "rewards/margins": 0.8565462827682495,
      "rewards/rejected": -3.8071327209472656,
      "step": 2825
    },
    {
      "epoch": 0.6130185205242067,
      "grad_norm": 18.375,
      "learning_rate": 1.3807767369484142e-05,
      "logits/chosen": -1.4217369556427002,
      "logits/rejected": -1.4845248460769653,
      "logps/chosen": -296.5137939453125,
      "logps/rejected": -309.809326171875,
      "loss": 0.5513,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -3.1792900562286377,
      "rewards/margins": 0.7933127284049988,
      "rewards/rejected": -3.9726028442382812,
      "step": 2830
    },
    {
      "epoch": 0.6141015921152388,
      "grad_norm": 18.25,
      "learning_rate": 1.3804593777795401e-05,
      "logits/chosen": -1.4435911178588867,
      "logits/rejected": -1.4227540493011475,
      "logps/chosen": -267.810546875,
      "logps/rejected": -290.75604248046875,
      "loss": 0.4866,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.5946946144104004,
      "rewards/margins": 1.0227010250091553,
      "rewards/rejected": -3.6173954010009766,
      "step": 2835
    },
    {
      "epoch": 0.615184663706271,
      "grad_norm": 19.0,
      "learning_rate": 1.380141342483889e-05,
      "logits/chosen": -1.4302228689193726,
      "logits/rejected": -1.410153865814209,
      "logps/chosen": -298.3344421386719,
      "logps/rejected": -310.2230529785156,
      "loss": 0.5391,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -3.091327428817749,
      "rewards/margins": 0.8997696042060852,
      "rewards/rejected": -3.9910969734191895,
      "step": 2840
    },
    {
      "epoch": 0.6162677352973032,
      "grad_norm": 16.75,
      "learning_rate": 1.3798226315149154e-05,
      "logits/chosen": -1.4603359699249268,
      "logits/rejected": -1.4636735916137695,
      "logps/chosen": -274.28179931640625,
      "logps/rejected": -299.52099609375,
      "loss": 0.4845,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.154053211212158,
      "rewards/margins": 1.1645164489746094,
      "rewards/rejected": -4.318569183349609,
      "step": 2845
    },
    {
      "epoch": 0.6173508068883353,
      "grad_norm": 23.875,
      "learning_rate": 1.3795032453270363e-05,
      "logits/chosen": -1.5164040327072144,
      "logits/rejected": -1.4696426391601562,
      "logps/chosen": -282.083740234375,
      "logps/rejected": -307.83447265625,
      "loss": 0.5164,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -2.8758978843688965,
      "rewards/margins": 0.8612872958183289,
      "rewards/rejected": -3.737185001373291,
      "step": 2850
    },
    {
      "epoch": 0.6184338784793675,
      "grad_norm": 16.125,
      "learning_rate": 1.3791831843756321e-05,
      "logits/chosen": -1.6062462329864502,
      "logits/rejected": -1.5852899551391602,
      "logps/chosen": -264.92657470703125,
      "logps/rejected": -281.7421569824219,
      "loss": 0.5193,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.6711246967315674,
      "rewards/margins": 1.0164783000946045,
      "rewards/rejected": -3.687602996826172,
      "step": 2855
    },
    {
      "epoch": 0.6195169500703996,
      "grad_norm": 12.5625,
      "learning_rate": 1.3788624491170451e-05,
      "logits/chosen": -1.5627707242965698,
      "logits/rejected": -1.4824931621551514,
      "logps/chosen": -271.61578369140625,
      "logps/rejected": -305.1745910644531,
      "loss": 0.4108,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.382671594619751,
      "rewards/margins": 1.3823957443237305,
      "rewards/rejected": -3.7650675773620605,
      "step": 2860
    },
    {
      "epoch": 0.6206000216614318,
      "grad_norm": 17.125,
      "learning_rate": 1.3785410400085788e-05,
      "logits/chosen": -1.5491552352905273,
      "logits/rejected": -1.6214818954467773,
      "logps/chosen": -293.573486328125,
      "logps/rejected": -304.70648193359375,
      "loss": 0.5207,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -3.2887890338897705,
      "rewards/margins": 0.9419960975646973,
      "rewards/rejected": -4.2307844161987305,
      "step": 2865
    },
    {
      "epoch": 0.621683093252464,
      "grad_norm": 18.625,
      "learning_rate": 1.3782189575084978e-05,
      "logits/chosen": -1.6222026348114014,
      "logits/rejected": -1.5725289583206177,
      "logps/chosen": -284.5187072753906,
      "logps/rejected": -300.5948791503906,
      "loss": 0.4981,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -2.9086689949035645,
      "rewards/margins": 1.0967973470687866,
      "rewards/rejected": -4.005466461181641,
      "step": 2870
    },
    {
      "epoch": 0.6227661648434961,
      "grad_norm": 26.25,
      "learning_rate": 1.3778962020760267e-05,
      "logits/chosen": -1.6337058544158936,
      "logits/rejected": -1.6901843547821045,
      "logps/chosen": -291.4362487792969,
      "logps/rejected": -298.7823791503906,
      "loss": 0.4937,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.9355642795562744,
      "rewards/margins": 0.9514894485473633,
      "rewards/rejected": -3.8870537281036377,
      "step": 2875
    },
    {
      "epoch": 0.6238492364345283,
      "grad_norm": 15.0,
      "learning_rate": 1.3775727741713494e-05,
      "logits/chosen": -1.6973212957382202,
      "logits/rejected": -1.6714200973510742,
      "logps/chosen": -256.8191833496094,
      "logps/rejected": -291.0643005371094,
      "loss": 0.4599,
      "rewards/accuracies": 0.7666667699813843,
      "rewards/chosen": -2.2830746173858643,
      "rewards/margins": 1.0445802211761475,
      "rewards/rejected": -3.3276546001434326,
      "step": 2880
    },
    {
      "epoch": 0.6249323080255604,
      "grad_norm": 17.25,
      "learning_rate": 1.3772486742556091e-05,
      "logits/chosen": -1.619218111038208,
      "logits/rejected": -1.6563646793365479,
      "logps/chosen": -286.0284118652344,
      "logps/rejected": -289.1103820800781,
      "loss": 0.5441,
      "rewards/accuracies": 0.6833332777023315,
      "rewards/chosen": -2.4998221397399902,
      "rewards/margins": 0.9064549207687378,
      "rewards/rejected": -3.4062771797180176,
      "step": 2885
    },
    {
      "epoch": 0.6260153796165927,
      "grad_norm": 16.125,
      "learning_rate": 1.3769239027909063e-05,
      "logits/chosen": -1.635730504989624,
      "logits/rejected": -1.6731481552124023,
      "logps/chosen": -277.5523376464844,
      "logps/rejected": -284.1510314941406,
      "loss": 0.4881,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.4354586601257324,
      "rewards/margins": 1.1422059535980225,
      "rewards/rejected": -3.577664852142334,
      "step": 2890
    },
    {
      "epoch": 0.6270984512076249,
      "grad_norm": 20.125,
      "learning_rate": 1.3765984602402998e-05,
      "logits/chosen": -1.475931167602539,
      "logits/rejected": -1.4696238040924072,
      "logps/chosen": -277.9344787597656,
      "logps/rejected": -302.07269287109375,
      "loss": 0.507,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.7832980155944824,
      "rewards/margins": 0.9156006574630737,
      "rewards/rejected": -3.6988983154296875,
      "step": 2895
    },
    {
      "epoch": 0.628181522798657,
      "grad_norm": 16.375,
      "learning_rate": 1.376272347067805e-05,
      "logits/chosen": -1.5383107662200928,
      "logits/rejected": -1.547486662864685,
      "logps/chosen": -290.45477294921875,
      "logps/rejected": -302.28131103515625,
      "loss": 0.5183,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.703857898712158,
      "rewards/margins": 1.1233792304992676,
      "rewards/rejected": -3.827237606048584,
      "step": 2900
    },
    {
      "epoch": 0.6292645943896892,
      "grad_norm": 21.25,
      "learning_rate": 1.375945563738393e-05,
      "logits/chosen": -1.6950457096099854,
      "logits/rejected": -1.6394011974334717,
      "logps/chosen": -275.11834716796875,
      "logps/rejected": -293.47021484375,
      "loss": 0.5643,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.893019676208496,
      "rewards/margins": 0.8549701571464539,
      "rewards/rejected": -3.747990131378174,
      "step": 2905
    },
    {
      "epoch": 0.6303476659807213,
      "grad_norm": 18.375,
      "learning_rate": 1.3756181107179916e-05,
      "logits/chosen": -1.595016598701477,
      "logits/rejected": -1.4789540767669678,
      "logps/chosen": -274.0680847167969,
      "logps/rejected": -299.1567687988281,
      "loss": 0.4694,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.55454683303833,
      "rewards/margins": 1.0033857822418213,
      "rewards/rejected": -3.5579323768615723,
      "step": 2910
    },
    {
      "epoch": 0.6314307375717535,
      "grad_norm": 19.25,
      "learning_rate": 1.3752899884734816e-05,
      "logits/chosen": -1.6267162561416626,
      "logits/rejected": -1.6121059656143188,
      "logps/chosen": -269.3705139160156,
      "logps/rejected": -287.54254150390625,
      "loss": 0.4176,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.474458694458008,
      "rewards/margins": 1.4499486684799194,
      "rewards/rejected": -3.9244072437286377,
      "step": 2915
    },
    {
      "epoch": 0.6325138091627857,
      "grad_norm": 22.375,
      "learning_rate": 1.3749611974727e-05,
      "logits/chosen": -1.5277830362319946,
      "logits/rejected": -1.5375659465789795,
      "logps/chosen": -262.2665100097656,
      "logps/rejected": -290.5980224609375,
      "loss": 0.4958,
      "rewards/accuracies": 0.7250000834465027,
      "rewards/chosen": -2.4088051319122314,
      "rewards/margins": 1.263769507408142,
      "rewards/rejected": -3.672574520111084,
      "step": 2920
    },
    {
      "epoch": 0.6335968807538178,
      "grad_norm": 14.75,
      "learning_rate": 1.374631738184436e-05,
      "logits/chosen": -1.5241698026657104,
      "logits/rejected": -1.5127952098846436,
      "logps/chosen": -267.1932067871094,
      "logps/rejected": -298.84613037109375,
      "loss": 0.4462,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.397836685180664,
      "rewards/margins": 1.239391565322876,
      "rewards/rejected": -3.637228012084961,
      "step": 2925
    },
    {
      "epoch": 0.63467995234485,
      "grad_norm": 20.375,
      "learning_rate": 1.3743016110784318e-05,
      "logits/chosen": -1.6559460163116455,
      "logits/rejected": -1.5480279922485352,
      "logps/chosen": -267.77801513671875,
      "logps/rejected": -294.22467041015625,
      "loss": 0.4671,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.822746992111206,
      "rewards/margins": 1.1495344638824463,
      "rewards/rejected": -3.9722816944122314,
      "step": 2930
    },
    {
      "epoch": 0.6357630239358821,
      "grad_norm": 15.0,
      "learning_rate": 1.3739708166253821e-05,
      "logits/chosen": -1.572543978691101,
      "logits/rejected": -1.6136363744735718,
      "logps/chosen": -283.65240478515625,
      "logps/rejected": -303.16461181640625,
      "loss": 0.4532,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -2.4933362007141113,
      "rewards/margins": 1.1588786840438843,
      "rewards/rejected": -3.652214527130127,
      "step": 2935
    },
    {
      "epoch": 0.6368460955269143,
      "grad_norm": 19.5,
      "learning_rate": 1.373639355296933e-05,
      "logits/chosen": -1.8708076477050781,
      "logits/rejected": -1.8026704788208008,
      "logps/chosen": -263.1170959472656,
      "logps/rejected": -285.75,
      "loss": 0.489,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.8073928356170654,
      "rewards/margins": 1.1648938655853271,
      "rewards/rejected": -3.9722869396209717,
      "step": 2940
    },
    {
      "epoch": 0.6379291671179464,
      "grad_norm": 17.0,
      "learning_rate": 1.3733072275656812e-05,
      "logits/chosen": -1.799320936203003,
      "logits/rejected": -1.8158849477767944,
      "logps/chosen": -281.0562438964844,
      "logps/rejected": -301.7154846191406,
      "loss": 0.5381,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.1631245613098145,
      "rewards/margins": 1.0125484466552734,
      "rewards/rejected": -4.175673007965088,
      "step": 2945
    },
    {
      "epoch": 0.6390122387089786,
      "grad_norm": 19.625,
      "learning_rate": 1.3729744339051743e-05,
      "logits/chosen": -1.8148794174194336,
      "logits/rejected": -1.7439393997192383,
      "logps/chosen": -281.0985412597656,
      "logps/rejected": -293.91571044921875,
      "loss": 0.5547,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.849522113800049,
      "rewards/margins": 0.7939195036888123,
      "rewards/rejected": -3.643441677093506,
      "step": 2950
    },
    {
      "epoch": 0.6400953103000109,
      "grad_norm": 17.25,
      "learning_rate": 1.3726409747899083e-05,
      "logits/chosen": -1.747467279434204,
      "logits/rejected": -1.8003909587860107,
      "logps/chosen": -274.4515380859375,
      "logps/rejected": -281.799072265625,
      "loss": 0.4992,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.751807689666748,
      "rewards/margins": 1.0731908082962036,
      "rewards/rejected": -3.824998378753662,
      "step": 2955
    },
    {
      "epoch": 0.641178381891043,
      "grad_norm": 16.125,
      "learning_rate": 1.3723068506953285e-05,
      "logits/chosen": -1.6388702392578125,
      "logits/rejected": -1.7181205749511719,
      "logps/chosen": -270.6435241699219,
      "logps/rejected": -288.80682373046875,
      "loss": 0.4631,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.5353751182556152,
      "rewards/margins": 1.2181508541107178,
      "rewards/rejected": -3.753526210784912,
      "step": 2960
    },
    {
      "epoch": 0.6422614534820752,
      "grad_norm": 21.0,
      "learning_rate": 1.3719720620978288e-05,
      "logits/chosen": -1.6558841466903687,
      "logits/rejected": -1.663805603981018,
      "logps/chosen": -278.07965087890625,
      "logps/rejected": -299.181396484375,
      "loss": 0.4286,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -2.686361789703369,
      "rewards/margins": 1.3627394437789917,
      "rewards/rejected": -4.049101829528809,
      "step": 2965
    },
    {
      "epoch": 0.6433445250731074,
      "grad_norm": 16.375,
      "learning_rate": 1.3716366094747496e-05,
      "logits/chosen": -1.8111069202423096,
      "logits/rejected": -1.8288215398788452,
      "logps/chosen": -289.9911193847656,
      "logps/rejected": -294.22430419921875,
      "loss": 0.4966,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.875706911087036,
      "rewards/margins": 0.9834769368171692,
      "rewards/rejected": -3.8591842651367188,
      "step": 2970
    },
    {
      "epoch": 0.6444275966641395,
      "grad_norm": 18.125,
      "learning_rate": 1.371300493304379e-05,
      "logits/chosen": -1.797394037246704,
      "logits/rejected": -1.8737672567367554,
      "logps/chosen": -265.7646484375,
      "logps/rejected": -279.76995849609375,
      "loss": 0.4246,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.704695224761963,
      "rewards/margins": 1.1811349391937256,
      "rewards/rejected": -3.8858299255371094,
      "step": 2975
    },
    {
      "epoch": 0.6455106682551717,
      "grad_norm": 20.75,
      "learning_rate": 1.3709637140659505e-05,
      "logits/chosen": -1.9018075466156006,
      "logits/rejected": -1.7704689502716064,
      "logps/chosen": -285.56097412109375,
      "logps/rejected": -298.96270751953125,
      "loss": 0.5367,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.9949326515197754,
      "rewards/margins": 1.0200382471084595,
      "rewards/rejected": -4.014970302581787,
      "step": 2980
    },
    {
      "epoch": 0.6465937398462038,
      "grad_norm": 16.375,
      "learning_rate": 1.3706262722396434e-05,
      "logits/chosen": -1.9438308477401733,
      "logits/rejected": -1.9495327472686768,
      "logps/chosen": -272.4732666015625,
      "logps/rejected": -286.40667724609375,
      "loss": 0.5561,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.947474718093872,
      "rewards/margins": 0.9330391883850098,
      "rewards/rejected": -3.8805136680603027,
      "step": 2985
    },
    {
      "epoch": 0.647676811437236,
      "grad_norm": 16.125,
      "learning_rate": 1.3702881683065814e-05,
      "logits/chosen": -1.904539704322815,
      "logits/rejected": -1.8658336400985718,
      "logps/chosen": -271.55084228515625,
      "logps/rejected": -299.4832458496094,
      "loss": 0.405,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.1269783973693848,
      "rewards/margins": 1.4551976919174194,
      "rewards/rejected": -4.582176685333252,
      "step": 2990
    },
    {
      "epoch": 0.6487598830282681,
      "grad_norm": 19.75,
      "learning_rate": 1.369949402748833e-05,
      "logits/chosen": -1.8373184204101562,
      "logits/rejected": -1.7894256114959717,
      "logps/chosen": -276.3069152832031,
      "logps/rejected": -290.94256591796875,
      "loss": 0.4584,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.4723546504974365,
      "rewards/margins": 1.0197440385818481,
      "rewards/rejected": -3.4920990467071533,
      "step": 2995
    },
    {
      "epoch": 0.6498429546193003,
      "grad_norm": 21.875,
      "learning_rate": 1.3696099760494084e-05,
      "logits/chosen": -1.791242003440857,
      "logits/rejected": -1.7876726388931274,
      "logps/chosen": -272.00286865234375,
      "logps/rejected": -303.8002624511719,
      "loss": 0.4751,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -2.577385663986206,
      "rewards/margins": 1.2868260145187378,
      "rewards/rejected": -3.8642120361328125,
      "step": 3000
    },
    {
      "epoch": 0.6498429546193003,
      "eval_logits/chosen": -1.8320815563201904,
      "eval_logits/rejected": -1.7672909498214722,
      "eval_logps/chosen": -265.4243469238281,
      "eval_logps/rejected": -287.4786071777344,
      "eval_loss": 0.5467246174812317,
      "eval_rewards/accuracies": 0.7200000286102295,
      "eval_rewards/chosen": -2.561154365539551,
      "eval_rewards/margins": 0.929919421672821,
      "eval_rewards/rejected": -3.4910736083984375,
      "eval_runtime": 14.7145,
      "eval_samples_per_second": 13.592,
      "eval_steps_per_second": 13.592,
      "step": 3000
    },
    {
      "epoch": 0.6509260262103325,
      "grad_norm": 17.375,
      "learning_rate": 1.3692698886922625e-05,
      "logits/chosen": -1.8679542541503906,
      "logits/rejected": -1.8159459829330444,
      "logps/chosen": -278.92852783203125,
      "logps/rejected": -285.48028564453125,
      "loss": 0.5136,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.7960433959960938,
      "rewards/margins": 1.0444886684417725,
      "rewards/rejected": -3.840531826019287,
      "step": 3005
    },
    {
      "epoch": 0.6520090978013646,
      "grad_norm": 17.375,
      "learning_rate": 1.3689291411622906e-05,
      "logits/chosen": -1.826972246170044,
      "logits/rejected": -1.8680741786956787,
      "logps/chosen": -279.09375,
      "logps/rejected": -292.02996826171875,
      "loss": 0.4945,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.9093995094299316,
      "rewards/margins": 1.0329554080963135,
      "rewards/rejected": -3.942354679107666,
      "step": 3010
    },
    {
      "epoch": 0.6530921693923969,
      "grad_norm": 22.875,
      "learning_rate": 1.3685877339453303e-05,
      "logits/chosen": -1.9077904224395752,
      "logits/rejected": -1.9193451404571533,
      "logps/chosen": -278.51214599609375,
      "logps/rejected": -295.0505676269531,
      "loss": 0.5261,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -3.284086227416992,
      "rewards/margins": 0.9597852826118469,
      "rewards/rejected": -4.243871212005615,
      "step": 3015
    },
    {
      "epoch": 0.654175240983429,
      "grad_norm": 22.5,
      "learning_rate": 1.3682456675281585e-05,
      "logits/chosen": -1.71354079246521,
      "logits/rejected": -1.7191498279571533,
      "logps/chosen": -263.12030029296875,
      "logps/rejected": -290.1624755859375,
      "loss": 0.5198,
      "rewards/accuracies": 0.7416667342185974,
      "rewards/chosen": -2.796535015106201,
      "rewards/margins": 0.98553866147995,
      "rewards/rejected": -3.782073497772217,
      "step": 3020
    },
    {
      "epoch": 0.6552583125744612,
      "grad_norm": 17.25,
      "learning_rate": 1.3679029423984937e-05,
      "logits/chosen": -1.7576462030410767,
      "logits/rejected": -1.8324012756347656,
      "logps/chosen": -266.7425231933594,
      "logps/rejected": -280.99871826171875,
      "loss": 0.4673,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.7113709449768066,
      "rewards/margins": 1.1048380136489868,
      "rewards/rejected": -3.816209077835083,
      "step": 3025
    },
    {
      "epoch": 0.6563413841654934,
      "grad_norm": 14.5625,
      "learning_rate": 1.3675595590449923e-05,
      "logits/chosen": -1.752943754196167,
      "logits/rejected": -1.7891111373901367,
      "logps/chosen": -271.2606506347656,
      "logps/rejected": -280.41168212890625,
      "loss": 0.4771,
      "rewards/accuracies": 0.7833332419395447,
      "rewards/chosen": -2.592111587524414,
      "rewards/margins": 1.066746473312378,
      "rewards/rejected": -3.65885853767395,
      "step": 3030
    },
    {
      "epoch": 0.6574244557565255,
      "grad_norm": 19.0,
      "learning_rate": 1.3672155179572495e-05,
      "logits/chosen": -1.6843559741973877,
      "logits/rejected": -1.6457220315933228,
      "logps/chosen": -280.5265197753906,
      "logps/rejected": -309.02392578125,
      "loss": 0.5127,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4174563884735107,
      "rewards/margins": 1.063330888748169,
      "rewards/rejected": -3.4807872772216797,
      "step": 3035
    },
    {
      "epoch": 0.6585075273475577,
      "grad_norm": 21.25,
      "learning_rate": 1.3668708196257987e-05,
      "logits/chosen": -1.7882401943206787,
      "logits/rejected": -1.7402969598770142,
      "logps/chosen": -279.5749206542969,
      "logps/rejected": -296.1362609863281,
      "loss": 0.4584,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -2.741730213165283,
      "rewards/margins": 1.0496902465820312,
      "rewards/rejected": -3.7914204597473145,
      "step": 3040
    },
    {
      "epoch": 0.6595905989385898,
      "grad_norm": 19.75,
      "learning_rate": 1.3665254645421098e-05,
      "logits/chosen": -1.6630789041519165,
      "logits/rejected": -1.6228868961334229,
      "logps/chosen": -277.572265625,
      "logps/rejected": -293.7634582519531,
      "loss": 0.5123,
      "rewards/accuracies": 0.716666579246521,
      "rewards/chosen": -2.7124593257904053,
      "rewards/margins": 0.9074892997741699,
      "rewards/rejected": -3.619948625564575,
      "step": 3045
    },
    {
      "epoch": 0.660673670529622,
      "grad_norm": 20.625,
      "learning_rate": 1.3661794531985894e-05,
      "logits/chosen": -1.8709657192230225,
      "logits/rejected": -1.83297860622406,
      "logps/chosen": -279.5922546386719,
      "logps/rejected": -292.22882080078125,
      "loss": 0.4968,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.215095043182373,
      "rewards/margins": 1.2086156606674194,
      "rewards/rejected": -4.423710823059082,
      "step": 3050
    },
    {
      "epoch": 0.6617567421206542,
      "grad_norm": 20.25,
      "learning_rate": 1.36583278608858e-05,
      "logits/chosen": -1.6930201053619385,
      "logits/rejected": -1.7405792474746704,
      "logps/chosen": -277.7146301269531,
      "logps/rejected": -282.87567138671875,
      "loss": 0.4708,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -2.776630401611328,
      "rewards/margins": 1.1227929592132568,
      "rewards/rejected": -3.899423599243164,
      "step": 3055
    },
    {
      "epoch": 0.6628398137116863,
      "grad_norm": 22.875,
      "learning_rate": 1.365485463706359e-05,
      "logits/chosen": -1.7760502099990845,
      "logits/rejected": -1.763878583908081,
      "logps/chosen": -281.74267578125,
      "logps/rejected": -301.83819580078125,
      "loss": 0.4852,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.7023255825042725,
      "rewards/margins": 1.1468009948730469,
      "rewards/rejected": -3.8491263389587402,
      "step": 3060
    },
    {
      "epoch": 0.6639228853027185,
      "grad_norm": 21.5,
      "learning_rate": 1.3651374865471372e-05,
      "logits/chosen": -1.7425386905670166,
      "logits/rejected": -1.7360070943832397,
      "logps/chosen": -289.9761657714844,
      "logps/rejected": -302.460205078125,
      "loss": 0.4873,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -3.1930699348449707,
      "rewards/margins": 1.1455377340316772,
      "rewards/rejected": -4.338608264923096,
      "step": 3065
    },
    {
      "epoch": 0.6650059568937506,
      "grad_norm": 19.5,
      "learning_rate": 1.3647888551070605e-05,
      "logits/chosen": -1.6609899997711182,
      "logits/rejected": -1.6136095523834229,
      "logps/chosen": -298.53985595703125,
      "logps/rejected": -301.50653076171875,
      "loss": 0.5152,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -2.6891732215881348,
      "rewards/margins": 0.9933584928512573,
      "rewards/rejected": -3.6825313568115234,
      "step": 3070
    },
    {
      "epoch": 0.6660890284847828,
      "grad_norm": 19.125,
      "learning_rate": 1.3644395698832065e-05,
      "logits/chosen": -1.7236592769622803,
      "logits/rejected": -1.706477403640747,
      "logps/chosen": -269.04925537109375,
      "logps/rejected": -289.826171875,
      "loss": 0.5134,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.1670966148376465,
      "rewards/margins": 0.9001131057739258,
      "rewards/rejected": -4.067209720611572,
      "step": 3075
    },
    {
      "epoch": 0.6671721000758151,
      "grad_norm": 16.75,
      "learning_rate": 1.364089631373586e-05,
      "logits/chosen": -1.5726789236068726,
      "logits/rejected": -1.553096890449524,
      "logps/chosen": -275.59112548828125,
      "logps/rejected": -295.48333740234375,
      "loss": 0.5037,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.6021697521209717,
      "rewards/margins": 0.9546161890029907,
      "rewards/rejected": -3.556786298751831,
      "step": 3080
    },
    {
      "epoch": 0.6682551716668472,
      "grad_norm": 21.875,
      "learning_rate": 1.3637390400771396e-05,
      "logits/chosen": -1.4050966501235962,
      "logits/rejected": -1.3956589698791504,
      "logps/chosen": -270.86419677734375,
      "logps/rejected": -301.73101806640625,
      "loss": 0.5112,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.274183750152588,
      "rewards/margins": 0.9602338671684265,
      "rewards/rejected": -3.234417676925659,
      "step": 3085
    },
    {
      "epoch": 0.6693382432578794,
      "grad_norm": 16.5,
      "learning_rate": 1.3633877964937403e-05,
      "logits/chosen": -1.6259323358535767,
      "logits/rejected": -1.5977429151535034,
      "logps/chosen": -266.1186218261719,
      "logps/rejected": -294.1722412109375,
      "loss": 0.4742,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.6182987689971924,
      "rewards/margins": 1.0431417226791382,
      "rewards/rejected": -3.66144061088562,
      "step": 3090
    },
    {
      "epoch": 0.6704213148489115,
      "grad_norm": 16.25,
      "learning_rate": 1.3630359011241905e-05,
      "logits/chosen": -1.6027300357818604,
      "logits/rejected": -1.6050968170166016,
      "logps/chosen": -266.3509521484375,
      "logps/rejected": -302.23504638671875,
      "loss": 0.4303,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.525815486907959,
      "rewards/margins": 1.4119174480438232,
      "rewards/rejected": -3.937732696533203,
      "step": 3095
    },
    {
      "epoch": 0.6715043864399437,
      "grad_norm": 18.625,
      "learning_rate": 1.3626833544702216e-05,
      "logits/chosen": -1.5788264274597168,
      "logits/rejected": -1.6180784702301025,
      "logps/chosen": -268.30389404296875,
      "logps/rejected": -292.61590576171875,
      "loss": 0.4762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.5804061889648438,
      "rewards/margins": 1.0845040082931519,
      "rewards/rejected": -3.664910078048706,
      "step": 3100
    },
    {
      "epoch": 0.6725874580309759,
      "grad_norm": 18.625,
      "learning_rate": 1.3623301570344942e-05,
      "logits/chosen": -1.6843624114990234,
      "logits/rejected": -1.6733795404434204,
      "logps/chosen": -278.88885498046875,
      "logps/rejected": -292.2353515625,
      "loss": 0.5072,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -3.0251002311706543,
      "rewards/margins": 0.9731191396713257,
      "rewards/rejected": -3.9982197284698486,
      "step": 3105
    },
    {
      "epoch": 0.673670529622008,
      "grad_norm": 17.375,
      "learning_rate": 1.3619763093205964e-05,
      "logits/chosen": -1.6619412899017334,
      "logits/rejected": -1.6443099975585938,
      "logps/chosen": -270.94647216796875,
      "logps/rejected": -288.2874450683594,
      "loss": 0.5032,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.7993290424346924,
      "rewards/margins": 0.9875049591064453,
      "rewards/rejected": -3.7868340015411377,
      "step": 3110
    },
    {
      "epoch": 0.6747536012130402,
      "grad_norm": 12.5625,
      "learning_rate": 1.3616218118330436e-05,
      "logits/chosen": -1.7370163202285767,
      "logits/rejected": -1.7524216175079346,
      "logps/chosen": -263.1135559082031,
      "logps/rejected": -283.4582824707031,
      "loss": 0.4354,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.6962873935699463,
      "rewards/margins": 1.3063684701919556,
      "rewards/rejected": -4.002655982971191,
      "step": 3115
    },
    {
      "epoch": 0.6758366728040723,
      "grad_norm": 23.25,
      "learning_rate": 1.3612666650772775e-05,
      "logits/chosen": -1.7538591623306274,
      "logits/rejected": -1.7163715362548828,
      "logps/chosen": -279.33514404296875,
      "logps/rejected": -297.55853271484375,
      "loss": 0.5598,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -3.100984573364258,
      "rewards/margins": 0.8832602500915527,
      "rewards/rejected": -3.9842448234558105,
      "step": 3120
    },
    {
      "epoch": 0.6769197443951045,
      "grad_norm": 19.375,
      "learning_rate": 1.360910869559666e-05,
      "logits/chosen": -1.6139914989471436,
      "logits/rejected": -1.6459827423095703,
      "logps/chosen": -299.31060791015625,
      "logps/rejected": -308.403076171875,
      "loss": 0.4798,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -2.8718509674072266,
      "rewards/margins": 1.0533956289291382,
      "rewards/rejected": -3.925246477127075,
      "step": 3125
    },
    {
      "epoch": 0.6780028159861367,
      "grad_norm": 23.25,
      "learning_rate": 1.3605544257875009e-05,
      "logits/chosen": -1.666884183883667,
      "logits/rejected": -1.7170692682266235,
      "logps/chosen": -289.7327880859375,
      "logps/rejected": -301.12420654296875,
      "loss": 0.4863,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.934844732284546,
      "rewards/margins": 1.096532940864563,
      "rewards/rejected": -4.03137731552124,
      "step": 3130
    },
    {
      "epoch": 0.6790858875771688,
      "grad_norm": 23.5,
      "learning_rate": 1.3601973342689997e-05,
      "logits/chosen": -1.7200626134872437,
      "logits/rejected": -1.8147064447402954,
      "logps/chosen": -296.7723083496094,
      "logps/rejected": -301.0378723144531,
      "loss": 0.5592,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -3.638347625732422,
      "rewards/margins": 0.770809531211853,
      "rewards/rejected": -4.4091572761535645,
      "step": 3135
    },
    {
      "epoch": 0.680168959168201,
      "grad_norm": 23.125,
      "learning_rate": 1.3598395955133025e-05,
      "logits/chosen": -1.8158143758773804,
      "logits/rejected": -1.7117950916290283,
      "logps/chosen": -292.61639404296875,
      "logps/rejected": -306.8173522949219,
      "loss": 0.5186,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -3.5127785205841064,
      "rewards/margins": 0.921409010887146,
      "rewards/rejected": -4.434187889099121,
      "step": 3140
    },
    {
      "epoch": 0.6812520307592332,
      "grad_norm": 19.875,
      "learning_rate": 1.3594812100304724e-05,
      "logits/chosen": -1.858246088027954,
      "logits/rejected": -1.8177906274795532,
      "logps/chosen": -283.4544982910156,
      "logps/rejected": -295.4494934082031,
      "loss": 0.486,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.1429123878479004,
      "rewards/margins": 1.0249651670455933,
      "rewards/rejected": -4.167878150939941,
      "step": 3145
    },
    {
      "epoch": 0.6823351023502654,
      "grad_norm": 15.4375,
      "learning_rate": 1.3591221783314947e-05,
      "logits/chosen": -1.597841501235962,
      "logits/rejected": -1.5828005075454712,
      "logps/chosen": -299.3563537597656,
      "logps/rejected": -319.66412353515625,
      "loss": 0.4603,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.157100200653076,
      "rewards/margins": 1.0790942907333374,
      "rewards/rejected": -4.236194610595703,
      "step": 3150
    },
    {
      "epoch": 0.6834181739412976,
      "grad_norm": 15.0625,
      "learning_rate": 1.3587625009282763e-05,
      "logits/chosen": -1.6413516998291016,
      "logits/rejected": -1.6810293197631836,
      "logps/chosen": -287.38763427734375,
      "logps/rejected": -300.91632080078125,
      "loss": 0.4275,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.8807613849639893,
      "rewards/margins": 1.2386974096298218,
      "rewards/rejected": -4.11945915222168,
      "step": 3155
    },
    {
      "epoch": 0.6845012455323297,
      "grad_norm": 20.75,
      "learning_rate": 1.3584021783336446e-05,
      "logits/chosen": -1.7222572565078735,
      "logits/rejected": -1.6841890811920166,
      "logps/chosen": -293.76434326171875,
      "logps/rejected": -313.1242980957031,
      "loss": 0.5314,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.3220112323760986,
      "rewards/margins": 1.0105412006378174,
      "rewards/rejected": -4.332552909851074,
      "step": 3160
    },
    {
      "epoch": 0.6855843171233619,
      "grad_norm": 17.875,
      "learning_rate": 1.3580412110613464e-05,
      "logits/chosen": -1.769883394241333,
      "logits/rejected": -1.6829360723495483,
      "logps/chosen": -293.59564208984375,
      "logps/rejected": -315.77484130859375,
      "loss": 0.497,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.371453046798706,
      "rewards/margins": 1.2191269397735596,
      "rewards/rejected": -4.590579986572266,
      "step": 3165
    },
    {
      "epoch": 0.686667388714394,
      "grad_norm": 16.375,
      "learning_rate": 1.3576795996260484e-05,
      "logits/chosen": -1.76157546043396,
      "logits/rejected": -1.5925976037979126,
      "logps/chosen": -300.16851806640625,
      "logps/rejected": -324.235107421875,
      "loss": 0.5074,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -3.3177177906036377,
      "rewards/margins": 0.9597232937812805,
      "rewards/rejected": -4.277440547943115,
      "step": 3170
    },
    {
      "epoch": 0.6877504603054262,
      "grad_norm": 19.75,
      "learning_rate": 1.3573173445433355e-05,
      "logits/chosen": -1.756280541419983,
      "logits/rejected": -1.7720552682876587,
      "logps/chosen": -271.7419128417969,
      "logps/rejected": -293.5558776855469,
      "loss": 0.4352,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.072471857070923,
      "rewards/margins": 1.1448071002960205,
      "rewards/rejected": -4.217279434204102,
      "step": 3175
    },
    {
      "epoch": 0.6888335318964584,
      "grad_norm": 18.125,
      "learning_rate": 1.35695444632971e-05,
      "logits/chosen": -1.7523727416992188,
      "logits/rejected": -1.7488880157470703,
      "logps/chosen": -303.565673828125,
      "logps/rejected": -317.2266845703125,
      "loss": 0.5697,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -3.812666416168213,
      "rewards/margins": 0.808798611164093,
      "rewards/rejected": -4.621464729309082,
      "step": 3180
    },
    {
      "epoch": 0.6899166034874905,
      "grad_norm": 12.875,
      "learning_rate": 1.3565909055025916e-05,
      "logits/chosen": -1.8907150030136108,
      "logits/rejected": -1.8867902755737305,
      "logps/chosen": -284.9692687988281,
      "logps/rejected": -305.47869873046875,
      "loss": 0.45,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.328519105911255,
      "rewards/margins": 1.3567142486572266,
      "rewards/rejected": -4.685233116149902,
      "step": 3185
    },
    {
      "epoch": 0.6909996750785227,
      "grad_norm": 17.375,
      "learning_rate": 1.3562267225803161e-05,
      "logits/chosen": -1.8762757778167725,
      "logits/rejected": -1.8438602685928345,
      "logps/chosen": -296.05670166015625,
      "logps/rejected": -315.05767822265625,
      "loss": 0.3987,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -3.3432750701904297,
      "rewards/margins": 1.4248406887054443,
      "rewards/rejected": -4.768115997314453,
      "step": 3190
    },
    {
      "epoch": 0.6920827466695548,
      "grad_norm": 16.125,
      "learning_rate": 1.3558618980821349e-05,
      "logits/chosen": -1.9078909158706665,
      "logits/rejected": -1.8133735656738281,
      "logps/chosen": -268.4021911621094,
      "logps/rejected": -296.6136779785156,
      "loss": 0.4344,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.876784324645996,
      "rewards/margins": 1.3989791870117188,
      "rewards/rejected": -4.275763511657715,
      "step": 3195
    },
    {
      "epoch": 0.693165818260587,
      "grad_norm": 18.375,
      "learning_rate": 1.3554964325282139e-05,
      "logits/chosen": -1.8411667346954346,
      "logits/rejected": -1.8738794326782227,
      "logps/chosen": -282.3146667480469,
      "logps/rejected": -284.8064880371094,
      "loss": 0.4971,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.1206047534942627,
      "rewards/margins": 0.9881079792976379,
      "rewards/rejected": -4.108712673187256,
      "step": 3200
    },
    {
      "epoch": 0.6942488898516191,
      "grad_norm": 18.0,
      "learning_rate": 1.3551303264396329e-05,
      "logits/chosen": -1.7021028995513916,
      "logits/rejected": -1.6700845956802368,
      "logps/chosen": -293.44976806640625,
      "logps/rejected": -321.5334167480469,
      "loss": 0.4384,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.007845401763916,
      "rewards/margins": 1.2936283349990845,
      "rewards/rejected": -4.301473617553711,
      "step": 3205
    },
    {
      "epoch": 0.6953319614426514,
      "grad_norm": 15.375,
      "learning_rate": 1.3547635803383858e-05,
      "logits/chosen": -1.8289982080459595,
      "logits/rejected": -1.7869033813476562,
      "logps/chosen": -274.14947509765625,
      "logps/rejected": -299.02099609375,
      "loss": 0.4295,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.957329273223877,
      "rewards/margins": 1.28240966796875,
      "rewards/rejected": -4.239739418029785,
      "step": 3210
    },
    {
      "epoch": 0.6964150330336836,
      "grad_norm": 19.0,
      "learning_rate": 1.354396194747378e-05,
      "logits/chosen": -1.8071991205215454,
      "logits/rejected": -1.813216209411621,
      "logps/chosen": -292.292236328125,
      "logps/rejected": -328.2326354980469,
      "loss": 0.4499,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.380006790161133,
      "rewards/margins": 1.3216259479522705,
      "rewards/rejected": -4.701632499694824,
      "step": 3215
    },
    {
      "epoch": 0.6974981046247157,
      "grad_norm": 19.375,
      "learning_rate": 1.3540281701904271e-05,
      "logits/chosen": -1.7845007181167603,
      "logits/rejected": -1.792708396911621,
      "logps/chosen": -300.61798095703125,
      "logps/rejected": -313.51708984375,
      "loss": 0.4029,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.07448148727417,
      "rewards/margins": 1.4542829990386963,
      "rewards/rejected": -4.528764724731445,
      "step": 3220
    },
    {
      "epoch": 0.6985811762157479,
      "grad_norm": 27.25,
      "learning_rate": 1.3536595071922623e-05,
      "logits/chosen": -1.8555316925048828,
      "logits/rejected": -1.8514986038208008,
      "logps/chosen": -294.2336120605469,
      "logps/rejected": -308.49249267578125,
      "loss": 0.5344,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -3.0994179248809814,
      "rewards/margins": 1.1183627843856812,
      "rewards/rejected": -4.217781066894531,
      "step": 3225
    },
    {
      "epoch": 0.69966424780678,
      "grad_norm": 23.0,
      "learning_rate": 1.3532902062785219e-05,
      "logits/chosen": -1.8511054515838623,
      "logits/rejected": -1.8325116634368896,
      "logps/chosen": -267.68670654296875,
      "logps/rejected": -301.8970031738281,
      "loss": 0.5368,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -3.156810998916626,
      "rewards/margins": 0.930034339427948,
      "rewards/rejected": -4.0868449211120605,
      "step": 3230
    },
    {
      "epoch": 0.7007473193978122,
      "grad_norm": 19.0,
      "learning_rate": 1.3529202679757548e-05,
      "logits/chosen": -1.577280879020691,
      "logits/rejected": -1.6519927978515625,
      "logps/chosen": -307.8363342285156,
      "logps/rejected": -317.8794250488281,
      "loss": 0.4862,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.3203721046447754,
      "rewards/margins": 1.1439987421035767,
      "rewards/rejected": -4.4643707275390625,
      "step": 3235
    },
    {
      "epoch": 0.7018303909888444,
      "grad_norm": 21.375,
      "learning_rate": 1.3525496928114184e-05,
      "logits/chosen": -1.5511760711669922,
      "logits/rejected": -1.619117021560669,
      "logps/chosen": -279.73883056640625,
      "logps/rejected": -293.8120422363281,
      "loss": 0.5198,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -2.518735885620117,
      "rewards/margins": 1.0477898120880127,
      "rewards/rejected": -3.56652569770813,
      "step": 3240
    },
    {
      "epoch": 0.7029134625798765,
      "grad_norm": 16.25,
      "learning_rate": 1.3521784813138774e-05,
      "logits/chosen": -1.5253551006317139,
      "logits/rejected": -1.560084342956543,
      "logps/chosen": -306.88189697265625,
      "logps/rejected": -305.75433349609375,
      "loss": 0.5663,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.184373140335083,
      "rewards/margins": 0.9832528233528137,
      "rewards/rejected": -4.16762638092041,
      "step": 3245
    },
    {
      "epoch": 0.7039965341709087,
      "grad_norm": 17.125,
      "learning_rate": 1.3518066340124051e-05,
      "logits/chosen": -1.547919750213623,
      "logits/rejected": -1.4938850402832031,
      "logps/chosen": -270.63507080078125,
      "logps/rejected": -283.8141174316406,
      "loss": 0.5075,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.991126537322998,
      "rewards/margins": 1.047492504119873,
      "rewards/rejected": -4.038619041442871,
      "step": 3250
    },
    {
      "epoch": 0.7050796057619408,
      "grad_norm": 14.375,
      "learning_rate": 1.3514341514371802e-05,
      "logits/chosen": -1.3911221027374268,
      "logits/rejected": -1.4301831722259521,
      "logps/chosen": -275.8356018066406,
      "logps/rejected": -296.88665771484375,
      "loss": 0.3999,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.317918539047241,
      "rewards/margins": 1.4206255674362183,
      "rewards/rejected": -3.73854398727417,
      "step": 3255
    },
    {
      "epoch": 0.706162677352973,
      "grad_norm": 18.625,
      "learning_rate": 1.351061034119288e-05,
      "logits/chosen": -1.2845951318740845,
      "logits/rejected": -1.3077584505081177,
      "logps/chosen": -272.28192138671875,
      "logps/rejected": -282.55731201171875,
      "loss": 0.4902,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.526578426361084,
      "rewards/margins": 1.1320271492004395,
      "rewards/rejected": -3.6586060523986816,
      "step": 3260
    },
    {
      "epoch": 0.7072457489440052,
      "grad_norm": 20.875,
      "learning_rate": 1.3506872825907181e-05,
      "logits/chosen": -1.2669156789779663,
      "logits/rejected": -1.3596198558807373,
      "logps/chosen": -287.04107666015625,
      "logps/rejected": -293.93743896484375,
      "loss": 0.4763,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -2.2733230590820312,
      "rewards/margins": 1.109950304031372,
      "rewards/rejected": -3.3832733631134033,
      "step": 3265
    },
    {
      "epoch": 0.7083288205350373,
      "grad_norm": 18.25,
      "learning_rate": 1.3503128973843647e-05,
      "logits/chosen": -1.2359472513198853,
      "logits/rejected": -1.1969469785690308,
      "logps/chosen": -271.0248718261719,
      "logps/rejected": -288.9308776855469,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.259467363357544,
      "rewards/margins": 0.9194499850273132,
      "rewards/rejected": -3.178917407989502,
      "step": 3270
    },
    {
      "epoch": 0.7094118921260696,
      "grad_norm": 12.75,
      "learning_rate": 1.3499378790340256e-05,
      "logits/chosen": -1.1757657527923584,
      "logits/rejected": -1.1464110612869263,
      "logps/chosen": -278.9063415527344,
      "logps/rejected": -278.8744201660156,
      "loss": 0.4757,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.3715176582336426,
      "rewards/margins": 0.9684489369392395,
      "rewards/rejected": -3.339966297149658,
      "step": 3275
    },
    {
      "epoch": 0.7104949637171017,
      "grad_norm": 16.875,
      "learning_rate": 1.349562228074401e-05,
      "logits/chosen": -1.2565408945083618,
      "logits/rejected": -1.2474156618118286,
      "logps/chosen": -265.56768798828125,
      "logps/rejected": -300.9644470214844,
      "loss": 0.4297,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.4098987579345703,
      "rewards/margins": 1.3138706684112549,
      "rewards/rejected": -3.723769426345825,
      "step": 3280
    },
    {
      "epoch": 0.7115780353081339,
      "grad_norm": 21.875,
      "learning_rate": 1.3491859450410935e-05,
      "logits/chosen": -1.2534343004226685,
      "logits/rejected": -1.2283895015716553,
      "logps/chosen": -266.3620910644531,
      "logps/rejected": -291.4456481933594,
      "loss": 0.5309,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.421872615814209,
      "rewards/margins": 0.9091731905937195,
      "rewards/rejected": -3.331045627593994,
      "step": 3285
    },
    {
      "epoch": 0.7126611068991661,
      "grad_norm": 16.25,
      "learning_rate": 1.3488090304706067e-05,
      "logits/chosen": -1.3723299503326416,
      "logits/rejected": -1.3328922986984253,
      "logps/chosen": -252.447509765625,
      "logps/rejected": -283.0818786621094,
      "loss": 0.4491,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -2.3664638996124268,
      "rewards/margins": 1.059944748878479,
      "rewards/rejected": -3.4264087677001953,
      "step": 3290
    },
    {
      "epoch": 0.7137441784901982,
      "grad_norm": 20.75,
      "learning_rate": 1.3484314849003445e-05,
      "logits/chosen": -1.3343050479888916,
      "logits/rejected": -1.2926274538040161,
      "logps/chosen": -260.72528076171875,
      "logps/rejected": -276.6120910644531,
      "loss": 0.5192,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.185847043991089,
      "rewards/margins": 1.0672460794448853,
      "rewards/rejected": -3.2530932426452637,
      "step": 3295
    },
    {
      "epoch": 0.7148272500812304,
      "grad_norm": 24.625,
      "learning_rate": 1.3480533088686106e-05,
      "logits/chosen": -1.4119930267333984,
      "logits/rejected": -1.398984670639038,
      "logps/chosen": -282.39190673828125,
      "logps/rejected": -290.83807373046875,
      "loss": 0.4963,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.414757013320923,
      "rewards/margins": 1.1917500495910645,
      "rewards/rejected": -3.606506824493408,
      "step": 3300
    },
    {
      "epoch": 0.7159103216722625,
      "grad_norm": 17.125,
      "learning_rate": 1.3476745029146077e-05,
      "logits/chosen": -1.3494926691055298,
      "logits/rejected": -1.2867289781570435,
      "logps/chosen": -269.52691650390625,
      "logps/rejected": -298.0158996582031,
      "loss": 0.4496,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.256678819656372,
      "rewards/margins": 1.118733286857605,
      "rewards/rejected": -3.3754119873046875,
      "step": 3305
    },
    {
      "epoch": 0.7169933932632947,
      "grad_norm": 16.75,
      "learning_rate": 1.3472950675784367e-05,
      "logits/chosen": -1.3311628103256226,
      "logits/rejected": -1.3131972551345825,
      "logps/chosen": -273.0638732910156,
      "logps/rejected": -295.007080078125,
      "loss": 0.4482,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.367083787918091,
      "rewards/margins": 1.2177393436431885,
      "rewards/rejected": -3.5848231315612793,
      "step": 3310
    },
    {
      "epoch": 0.7180764648543269,
      "grad_norm": 15.4375,
      "learning_rate": 1.3469150034010956e-05,
      "logits/chosen": -1.3494422435760498,
      "logits/rejected": -1.4348849058151245,
      "logps/chosen": -277.5322265625,
      "logps/rejected": -276.2645263671875,
      "loss": 0.5148,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.2093148231506348,
      "rewards/margins": 1.033313512802124,
      "rewards/rejected": -3.242628574371338,
      "step": 3315
    },
    {
      "epoch": 0.719159536445359,
      "grad_norm": 18.875,
      "learning_rate": 1.346534310924479e-05,
      "logits/chosen": -1.4507720470428467,
      "logits/rejected": -1.4727184772491455,
      "logps/chosen": -275.52142333984375,
      "logps/rejected": -298.8975830078125,
      "loss": 0.4717,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -2.4549190998077393,
      "rewards/margins": 1.1111751794815063,
      "rewards/rejected": -3.5660946369171143,
      "step": 3320
    },
    {
      "epoch": 0.7202426080363912,
      "grad_norm": 25.125,
      "learning_rate": 1.3461529906913776e-05,
      "logits/chosen": -1.4495089054107666,
      "logits/rejected": -1.4527026414871216,
      "logps/chosen": -273.1053771972656,
      "logps/rejected": -281.27886962890625,
      "loss": 0.5217,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.5454461574554443,
      "rewards/margins": 0.933520495891571,
      "rewards/rejected": -3.4789669513702393,
      "step": 3325
    },
    {
      "epoch": 0.7213256796274233,
      "grad_norm": 21.0,
      "learning_rate": 1.3457710432454772e-05,
      "logits/chosen": -1.336303472518921,
      "logits/rejected": -1.4112645387649536,
      "logps/chosen": -271.32757568359375,
      "logps/rejected": -288.3018493652344,
      "loss": 0.4806,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.154604911804199,
      "rewards/margins": 1.2783485651016235,
      "rewards/rejected": -3.4329535961151123,
      "step": 3330
    },
    {
      "epoch": 0.7224087512184555,
      "grad_norm": 17.25,
      "learning_rate": 1.3453884691313575e-05,
      "logits/chosen": -1.4643669128417969,
      "logits/rejected": -1.4404494762420654,
      "logps/chosen": -275.59381103515625,
      "logps/rejected": -285.11138916015625,
      "loss": 0.4621,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.7195732593536377,
      "rewards/margins": 1.117379903793335,
      "rewards/rejected": -3.8369529247283936,
      "step": 3335
    },
    {
      "epoch": 0.7234918228094878,
      "grad_norm": 23.0,
      "learning_rate": 1.3450052688944917e-05,
      "logits/chosen": -1.4134495258331299,
      "logits/rejected": -1.4855375289916992,
      "logps/chosen": -267.19659423828125,
      "logps/rejected": -278.04461669921875,
      "loss": 0.5096,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.4923808574676514,
      "rewards/margins": 0.9115761518478394,
      "rewards/rejected": -3.403956890106201,
      "step": 3340
    },
    {
      "epoch": 0.7245748944005199,
      "grad_norm": 19.875,
      "learning_rate": 1.3446214430812465e-05,
      "logits/chosen": -1.4020386934280396,
      "logits/rejected": -1.40433669090271,
      "logps/chosen": -273.0818786621094,
      "logps/rejected": -292.439697265625,
      "loss": 0.5237,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -2.7383174896240234,
      "rewards/margins": 0.9028131365776062,
      "rewards/rejected": -3.6411304473876953,
      "step": 3345
    },
    {
      "epoch": 0.7256579659915521,
      "grad_norm": 23.0,
      "learning_rate": 1.3442369922388793e-05,
      "logits/chosen": -1.5854641199111938,
      "logits/rejected": -1.6177116632461548,
      "logps/chosen": -253.09091186523438,
      "logps/rejected": -266.9271545410156,
      "loss": 0.5151,
      "rewards/accuracies": 0.6833333373069763,
      "rewards/chosen": -2.5089831352233887,
      "rewards/margins": 0.9213716387748718,
      "rewards/rejected": -3.430354595184326,
      "step": 3350
    },
    {
      "epoch": 0.7267410375825842,
      "grad_norm": 21.25,
      "learning_rate": 1.3438519169155399e-05,
      "logits/chosen": -1.517026424407959,
      "logits/rejected": -1.5053691864013672,
      "logps/chosen": -276.08087158203125,
      "logps/rejected": -299.46405029296875,
      "loss": 0.443,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.8336410522460938,
      "rewards/margins": 1.2681024074554443,
      "rewards/rejected": -4.101743221282959,
      "step": 3355
    },
    {
      "epoch": 0.7278241091736164,
      "grad_norm": 16.375,
      "learning_rate": 1.3434662176602676e-05,
      "logits/chosen": -1.5140360593795776,
      "logits/rejected": -1.490747094154358,
      "logps/chosen": -272.10577392578125,
      "logps/rejected": -297.1375427246094,
      "loss": 0.4723,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.3269095420837402,
      "rewards/margins": 1.1507306098937988,
      "rewards/rejected": -3.4776406288146973,
      "step": 3360
    },
    {
      "epoch": 0.7289071807646486,
      "grad_norm": 17.375,
      "learning_rate": 1.3430798950229917e-05,
      "logits/chosen": -1.424533486366272,
      "logits/rejected": -1.4897822141647339,
      "logps/chosen": -283.59808349609375,
      "logps/rejected": -300.7807312011719,
      "loss": 0.3868,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -2.5261645317077637,
      "rewards/margins": 1.3929020166397095,
      "rewards/rejected": -3.9190661907196045,
      "step": 3365
    },
    {
      "epoch": 0.7299902523556807,
      "grad_norm": 18.875,
      "learning_rate": 1.3426929495545304e-05,
      "logits/chosen": -1.6210237741470337,
      "logits/rejected": -1.6175239086151123,
      "logps/chosen": -276.166015625,
      "logps/rejected": -291.44879150390625,
      "loss": 0.4728,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.568711757659912,
      "rewards/margins": 1.1303600072860718,
      "rewards/rejected": -3.6990714073181152,
      "step": 3370
    },
    {
      "epoch": 0.7310733239467129,
      "grad_norm": 23.5,
      "learning_rate": 1.3423053818065894e-05,
      "logits/chosen": -1.492607831954956,
      "logits/rejected": -1.567421555519104,
      "logps/chosen": -267.6380310058594,
      "logps/rejected": -282.9537048339844,
      "loss": 0.5121,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.463433027267456,
      "rewards/margins": 1.052497148513794,
      "rewards/rejected": -3.51593017578125,
      "step": 3375
    },
    {
      "epoch": 0.732156395537745,
      "grad_norm": 22.0,
      "learning_rate": 1.3419171923317623e-05,
      "logits/chosen": -1.7171297073364258,
      "logits/rejected": -1.720123529434204,
      "logps/chosen": -275.4627990722656,
      "logps/rejected": -299.8365478515625,
      "loss": 0.507,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -2.778189182281494,
      "rewards/margins": 1.0516960620880127,
      "rewards/rejected": -3.8298850059509277,
      "step": 3380
    },
    {
      "epoch": 0.7332394671287772,
      "grad_norm": 19.125,
      "learning_rate": 1.3415283816835288e-05,
      "logits/chosen": -1.5528080463409424,
      "logits/rejected": -1.5730011463165283,
      "logps/chosen": -294.75140380859375,
      "logps/rejected": -311.14691162109375,
      "loss": 0.4575,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.6347787380218506,
      "rewards/margins": 1.1512839794158936,
      "rewards/rejected": -3.7860629558563232,
      "step": 3385
    },
    {
      "epoch": 0.7343225387198093,
      "grad_norm": 17.75,
      "learning_rate": 1.3411389504162543e-05,
      "logits/chosen": -1.7104995250701904,
      "logits/rejected": -1.651643991470337,
      "logps/chosen": -273.9181823730469,
      "logps/rejected": -293.0843200683594,
      "loss": 0.4413,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.73968505859375,
      "rewards/margins": 1.1628239154815674,
      "rewards/rejected": -3.9025089740753174,
      "step": 3390
    },
    {
      "epoch": 0.7354056103108415,
      "grad_norm": 17.125,
      "learning_rate": 1.3407488990851893e-05,
      "logits/chosen": -1.6715272665023804,
      "logits/rejected": -1.717589020729065,
      "logps/chosen": -258.8108825683594,
      "logps/rejected": -284.8998718261719,
      "loss": 0.4423,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.9563515186309814,
      "rewards/margins": 1.203331470489502,
      "rewards/rejected": -4.159682273864746,
      "step": 3395
    },
    {
      "epoch": 0.7364886819018737,
      "grad_norm": 19.375,
      "learning_rate": 1.3403582282464682e-05,
      "logits/chosen": -1.728467345237732,
      "logits/rejected": -1.9136276245117188,
      "logps/chosen": -289.57177734375,
      "logps/rejected": -294.9412841796875,
      "loss": 0.4209,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.7402455806732178,
      "rewards/margins": 1.3276346921920776,
      "rewards/rejected": -4.067880630493164,
      "step": 3400
    },
    {
      "epoch": 0.7375717534929059,
      "grad_norm": 18.0,
      "learning_rate": 1.3399669384571083e-05,
      "logits/chosen": -1.779894471168518,
      "logits/rejected": -1.7459115982055664,
      "logps/chosen": -266.14544677734375,
      "logps/rejected": -289.6311340332031,
      "loss": 0.3943,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -2.757960081100464,
      "rewards/margins": 1.53951096534729,
      "rewards/rejected": -4.297471046447754,
      "step": 3405
    },
    {
      "epoch": 0.7386548250839381,
      "grad_norm": 16.25,
      "learning_rate": 1.33957503027501e-05,
      "logits/chosen": -1.7459900379180908,
      "logits/rejected": -1.802411437034607,
      "logps/chosen": -275.5085144042969,
      "logps/rejected": -296.7806091308594,
      "loss": 0.4735,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.7858152389526367,
      "rewards/margins": 1.4410932064056396,
      "rewards/rejected": -4.2269086837768555,
      "step": 3410
    },
    {
      "epoch": 0.7397378966749703,
      "grad_norm": 18.5,
      "learning_rate": 1.3391825042589555e-05,
      "logits/chosen": -1.8736886978149414,
      "logits/rejected": -1.869179368019104,
      "logps/chosen": -280.97412109375,
      "logps/rejected": -296.51971435546875,
      "loss": 0.4376,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -3.08148193359375,
      "rewards/margins": 1.3498388528823853,
      "rewards/rejected": -4.431321144104004,
      "step": 3415
    },
    {
      "epoch": 0.7408209682660024,
      "grad_norm": 22.125,
      "learning_rate": 1.3387893609686074e-05,
      "logits/chosen": -1.8272216320037842,
      "logits/rejected": -1.7365877628326416,
      "logps/chosen": -287.20843505859375,
      "logps/rejected": -321.4733581542969,
      "loss": 0.5466,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.901035785675049,
      "rewards/margins": 1.080980658531189,
      "rewards/rejected": -3.9820163249969482,
      "step": 3420
    },
    {
      "epoch": 0.7419040398570346,
      "grad_norm": 25.375,
      "learning_rate": 1.3383956009645083e-05,
      "logits/chosen": -1.727051019668579,
      "logits/rejected": -1.814075231552124,
      "logps/chosen": -283.93524169921875,
      "logps/rejected": -296.6197509765625,
      "loss": 0.4816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0486109256744385,
      "rewards/margins": 1.0468285083770752,
      "rewards/rejected": -4.095439910888672,
      "step": 3425
    },
    {
      "epoch": 0.7429871114480667,
      "grad_norm": 22.25,
      "learning_rate": 1.3380012248080809e-05,
      "logits/chosen": -1.6406549215316772,
      "logits/rejected": -1.66471266746521,
      "logps/chosen": -287.6441345214844,
      "logps/rejected": -306.8715515136719,
      "loss": 0.4553,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.695866107940674,
      "rewards/margins": 1.2768323421478271,
      "rewards/rejected": -3.972698211669922,
      "step": 3430
    },
    {
      "epoch": 0.7440701830390989,
      "grad_norm": 17.5,
      "learning_rate": 1.337606233061626e-05,
      "logits/chosen": -1.8878610134124756,
      "logits/rejected": -1.8849235773086548,
      "logps/chosen": -291.8800354003906,
      "logps/rejected": -307.29180908203125,
      "loss": 0.4624,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.3878986835479736,
      "rewards/margins": 1.2627637386322021,
      "rewards/rejected": -4.650662422180176,
      "step": 3435
    },
    {
      "epoch": 0.745153254630131,
      "grad_norm": 18.0,
      "learning_rate": 1.3372106262883211e-05,
      "logits/chosen": -1.7884323596954346,
      "logits/rejected": -1.8084605932235718,
      "logps/chosen": -263.5498046875,
      "logps/rejected": -284.25823974609375,
      "loss": 0.4438,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.528132200241089,
      "rewards/margins": 1.3309342861175537,
      "rewards/rejected": -3.8590664863586426,
      "step": 3440
    },
    {
      "epoch": 0.7462363262211632,
      "grad_norm": 24.875,
      "learning_rate": 1.3368144050522227e-05,
      "logits/chosen": -1.7341125011444092,
      "logits/rejected": -1.7062925100326538,
      "logps/chosen": -287.02325439453125,
      "logps/rejected": -308.9339294433594,
      "loss": 0.4341,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.840766668319702,
      "rewards/margins": 1.3096189498901367,
      "rewards/rejected": -4.150385856628418,
      "step": 3445
    },
    {
      "epoch": 0.7473193978121954,
      "grad_norm": 23.5,
      "learning_rate": 1.3364175699182618e-05,
      "logits/chosen": -1.7899993658065796,
      "logits/rejected": -1.785332441329956,
      "logps/chosen": -291.35699462890625,
      "logps/rejected": -294.88916015625,
      "loss": 0.5225,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.0122478008270264,
      "rewards/margins": 1.1640716791152954,
      "rewards/rejected": -4.176319599151611,
      "step": 3450
    },
    {
      "epoch": 0.7484024694032275,
      "grad_norm": 18.25,
      "learning_rate": 1.3360201214522448e-05,
      "logits/chosen": -1.8092153072357178,
      "logits/rejected": -1.7595407962799072,
      "logps/chosen": -274.4051818847656,
      "logps/rejected": -305.4346923828125,
      "loss": 0.3754,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -3.045910596847534,
      "rewards/margins": 1.5779527425765991,
      "rewards/rejected": -4.623863220214844,
      "step": 3455
    },
    {
      "epoch": 0.7494855409942597,
      "grad_norm": 19.875,
      "learning_rate": 1.3356220602208532e-05,
      "logits/chosen": -1.9300734996795654,
      "logits/rejected": -1.9100725650787354,
      "logps/chosen": -282.6371154785156,
      "logps/rejected": -317.68072509765625,
      "loss": 0.4672,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0900607109069824,
      "rewards/margins": 1.5732734203338623,
      "rewards/rejected": -4.663334369659424,
      "step": 3460
    },
    {
      "epoch": 0.7505686125852918,
      "grad_norm": 19.5,
      "learning_rate": 1.3352233867916418e-05,
      "logits/chosen": -1.976985216140747,
      "logits/rejected": -1.9864909648895264,
      "logps/chosen": -279.2084655761719,
      "logps/rejected": -304.01507568359375,
      "loss": 0.4567,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.954252004623413,
      "rewards/margins": 1.2656322717666626,
      "rewards/rejected": -4.219883918762207,
      "step": 3465
    },
    {
      "epoch": 0.7516516841763241,
      "grad_norm": 17.625,
      "learning_rate": 1.3348241017330388e-05,
      "logits/chosen": -1.9884544610977173,
      "logits/rejected": -2.0775160789489746,
      "logps/chosen": -274.45745849609375,
      "logps/rejected": -287.1611633300781,
      "loss": 0.5222,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.2252540588378906,
      "rewards/margins": 1.2366509437561035,
      "rewards/rejected": -4.461905002593994,
      "step": 3470
    },
    {
      "epoch": 0.7527347557673563,
      "grad_norm": 16.125,
      "learning_rate": 1.3344242056143436e-05,
      "logits/chosen": -1.813916802406311,
      "logits/rejected": -1.8129955530166626,
      "logps/chosen": -268.3038330078125,
      "logps/rejected": -292.2157897949219,
      "loss": 0.4442,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.823944330215454,
      "rewards/margins": 1.2459533214569092,
      "rewards/rejected": -4.069897651672363,
      "step": 3475
    },
    {
      "epoch": 0.7538178273583884,
      "grad_norm": 22.125,
      "learning_rate": 1.3340236990057274e-05,
      "logits/chosen": -1.7621848583221436,
      "logits/rejected": -1.7493988275527954,
      "logps/chosen": -289.6591796875,
      "logps/rejected": -296.228515625,
      "loss": 0.4506,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.854482650756836,
      "rewards/margins": 1.2760932445526123,
      "rewards/rejected": -4.130576133728027,
      "step": 3480
    },
    {
      "epoch": 0.7549008989494206,
      "grad_norm": 24.0,
      "learning_rate": 1.3336225824782316e-05,
      "logits/chosen": -1.9536378383636475,
      "logits/rejected": -1.9263073205947876,
      "logps/chosen": -283.899169921875,
      "logps/rejected": -290.97796630859375,
      "loss": 0.485,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.2091331481933594,
      "rewards/margins": 1.0568368434906006,
      "rewards/rejected": -4.265970230102539,
      "step": 3485
    },
    {
      "epoch": 0.7559839705404527,
      "grad_norm": 18.25,
      "learning_rate": 1.3332208566037678e-05,
      "logits/chosen": -1.9542220830917358,
      "logits/rejected": -1.9852927923202515,
      "logps/chosen": -277.2191467285156,
      "logps/rejected": -300.70281982421875,
      "loss": 0.5026,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.593010902404785,
      "rewards/margins": 1.3363484144210815,
      "rewards/rejected": -3.9293599128723145,
      "step": 3490
    },
    {
      "epoch": 0.7570670421314849,
      "grad_norm": 15.1875,
      "learning_rate": 1.3328185219551151e-05,
      "logits/chosen": -1.8671798706054688,
      "logits/rejected": -1.8913676738739014,
      "logps/chosen": -273.6955261230469,
      "logps/rejected": -313.89752197265625,
      "loss": 0.4431,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.683859348297119,
      "rewards/margins": 1.394958734512329,
      "rewards/rejected": -4.078818321228027,
      "step": 3495
    },
    {
      "epoch": 0.7581501137225171,
      "grad_norm": 25.5,
      "learning_rate": 1.3324155791059222e-05,
      "logits/chosen": -2.061408042907715,
      "logits/rejected": -2.0384230613708496,
      "logps/chosen": -287.5957946777344,
      "logps/rejected": -312.600341796875,
      "loss": 0.5221,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.108133316040039,
      "rewards/margins": 1.1483697891235352,
      "rewards/rejected": -4.256503105163574,
      "step": 3500
    },
    {
      "epoch": 0.7581501137225171,
      "eval_logits/chosen": -2.0964341163635254,
      "eval_logits/rejected": -2.028290271759033,
      "eval_logps/chosen": -270.171142578125,
      "eval_logps/rejected": -295.4356994628906,
      "eval_loss": 0.5122702717781067,
      "eval_rewards/accuracies": 0.7300000190734863,
      "eval_rewards/chosen": -2.7984936237335205,
      "eval_rewards/margins": 1.090435266494751,
      "eval_rewards/rejected": -3.8889288902282715,
      "eval_runtime": 14.4435,
      "eval_samples_per_second": 13.847,
      "eval_steps_per_second": 13.847,
      "step": 3500
    },
    {
      "epoch": 0.7592331853135492,
      "grad_norm": 11.0625,
      "learning_rate": 1.3320120286307039e-05,
      "logits/chosen": -2.079071283340454,
      "logits/rejected": -2.0824217796325684,
      "logps/chosen": -279.38763427734375,
      "logps/rejected": -300.81597900390625,
      "loss": 0.4293,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.888094663619995,
      "rewards/margins": 1.4132301807403564,
      "rewards/rejected": -4.301324844360352,
      "step": 3505
    },
    {
      "epoch": 0.7603162569045814,
      "grad_norm": 16.625,
      "learning_rate": 1.3316078711048417e-05,
      "logits/chosen": -1.8966010808944702,
      "logits/rejected": -1.9481102228164673,
      "logps/chosen": -298.1983337402344,
      "logps/rejected": -331.0272521972656,
      "loss": 0.4142,
      "rewards/accuracies": 0.7499999403953552,
      "rewards/chosen": -2.9509928226470947,
      "rewards/margins": 1.6246496438980103,
      "rewards/rejected": -4.5756425857543945,
      "step": 3510
    },
    {
      "epoch": 0.7613993284956135,
      "grad_norm": 24.375,
      "learning_rate": 1.3312031071045826e-05,
      "logits/chosen": -2.092648983001709,
      "logits/rejected": -2.0156655311584473,
      "logps/chosen": -275.22967529296875,
      "logps/rejected": -318.15423583984375,
      "loss": 0.4285,
      "rewards/accuracies": 0.7666667699813843,
      "rewards/chosen": -2.8433563709259033,
      "rewards/margins": 1.4213975667953491,
      "rewards/rejected": -4.264753818511963,
      "step": 3515
    },
    {
      "epoch": 0.7624824000866457,
      "grad_norm": 23.875,
      "learning_rate": 1.3307977372070383e-05,
      "logits/chosen": -2.191359519958496,
      "logits/rejected": -2.1382975578308105,
      "logps/chosen": -274.97149658203125,
      "logps/rejected": -296.5928955078125,
      "loss": 0.4575,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.186749219894409,
      "rewards/margins": 1.360784649848938,
      "rewards/rejected": -4.547533988952637,
      "step": 3520
    },
    {
      "epoch": 0.7635654716776779,
      "grad_norm": 18.625,
      "learning_rate": 1.3303917619901845e-05,
      "logits/chosen": -2.2301688194274902,
      "logits/rejected": -2.1558291912078857,
      "logps/chosen": -274.7452697753906,
      "logps/rejected": -309.96368408203125,
      "loss": 0.4622,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.933026075363159,
      "rewards/margins": 1.4668344259262085,
      "rewards/rejected": -4.39985990524292,
      "step": 3525
    },
    {
      "epoch": 0.76464854326871,
      "grad_norm": 22.375,
      "learning_rate": 1.3299851820328593e-05,
      "logits/chosen": -2.1142191886901855,
      "logits/rejected": -2.1467320919036865,
      "logps/chosen": -288.11260986328125,
      "logps/rejected": -312.09405517578125,
      "loss": 0.5285,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -3.6252646446228027,
      "rewards/margins": 1.1833083629608154,
      "rewards/rejected": -4.808572292327881,
      "step": 3530
    },
    {
      "epoch": 0.7657316148597423,
      "grad_norm": 16.5,
      "learning_rate": 1.3295779979147642e-05,
      "logits/chosen": -2.159667730331421,
      "logits/rejected": -2.2131080627441406,
      "logps/chosen": -303.5115966796875,
      "logps/rejected": -323.41961669921875,
      "loss": 0.4595,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.4854798316955566,
      "rewards/margins": 1.433417558670044,
      "rewards/rejected": -4.9188971519470215,
      "step": 3535
    },
    {
      "epoch": 0.7668146864507744,
      "grad_norm": 21.0,
      "learning_rate": 1.3291702102164612e-05,
      "logits/chosen": -2.172394275665283,
      "logits/rejected": -2.10688853263855,
      "logps/chosen": -291.53863525390625,
      "logps/rejected": -317.1921081542969,
      "loss": 0.448,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.4661953449249268,
      "rewards/margins": 1.3563175201416016,
      "rewards/rejected": -4.822512626647949,
      "step": 3540
    },
    {
      "epoch": 0.7678977580418066,
      "grad_norm": 22.375,
      "learning_rate": 1.3287618195193736e-05,
      "logits/chosen": -2.1741371154785156,
      "logits/rejected": -2.184985876083374,
      "logps/chosen": -270.27667236328125,
      "logps/rejected": -290.4717102050781,
      "loss": 0.4887,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.966670274734497,
      "rewards/margins": 1.331038475036621,
      "rewards/rejected": -4.297708034515381,
      "step": 3545
    },
    {
      "epoch": 0.7689808296328388,
      "grad_norm": 18.375,
      "learning_rate": 1.3283528264057833e-05,
      "logits/chosen": -2.0309529304504395,
      "logits/rejected": -2.0515811443328857,
      "logps/chosen": -292.18292236328125,
      "logps/rejected": -320.15435791015625,
      "loss": 0.439,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.1495814323425293,
      "rewards/margins": 1.3646154403686523,
      "rewards/rejected": -4.514196872711182,
      "step": 3550
    },
    {
      "epoch": 0.7700639012238709,
      "grad_norm": 16.0,
      "learning_rate": 1.3279432314588324e-05,
      "logits/chosen": -2.0860400199890137,
      "logits/rejected": -1.9682528972625732,
      "logps/chosen": -286.1904296875,
      "logps/rejected": -313.30126953125,
      "loss": 0.4259,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -3.313199520111084,
      "rewards/margins": 1.3760862350463867,
      "rewards/rejected": -4.689285755157471,
      "step": 3555
    },
    {
      "epoch": 0.7711469728149031,
      "grad_norm": 12.0,
      "learning_rate": 1.3275330352625205e-05,
      "logits/chosen": -2.132369041442871,
      "logits/rejected": -2.105441093444824,
      "logps/chosen": -269.3824157714844,
      "logps/rejected": -288.4572448730469,
      "loss": 0.4678,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.857123613357544,
      "rewards/margins": 1.3384020328521729,
      "rewards/rejected": -4.195526123046875,
      "step": 3560
    },
    {
      "epoch": 0.7722300444059352,
      "grad_norm": 21.125,
      "learning_rate": 1.3271222384017041e-05,
      "logits/chosen": -2.1574277877807617,
      "logits/rejected": -2.0359930992126465,
      "logps/chosen": -278.2089538574219,
      "logps/rejected": -309.2167663574219,
      "loss": 0.5597,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.3983638286590576,
      "rewards/margins": 0.9614431262016296,
      "rewards/rejected": -4.359806537628174,
      "step": 3565
    },
    {
      "epoch": 0.7733131159969674,
      "grad_norm": 18.625,
      "learning_rate": 1.3267108414620973e-05,
      "logits/chosen": -2.0517497062683105,
      "logits/rejected": -1.9830671548843384,
      "logps/chosen": -261.43145751953125,
      "logps/rejected": -300.81549072265625,
      "loss": 0.4063,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -2.3865232467651367,
      "rewards/margins": 1.428425908088684,
      "rewards/rejected": -3.8149490356445312,
      "step": 3570
    },
    {
      "epoch": 0.7743961875879996,
      "grad_norm": 24.625,
      "learning_rate": 1.3262988450302685e-05,
      "logits/chosen": -2.1084840297698975,
      "logits/rejected": -2.0384654998779297,
      "logps/chosen": -280.1694641113281,
      "logps/rejected": -310.32421875,
      "loss": 0.4289,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -2.5434858798980713,
      "rewards/margins": 1.3837164640426636,
      "rewards/rejected": -3.9272027015686035,
      "step": 3575
    },
    {
      "epoch": 0.7754792591790317,
      "grad_norm": 20.5,
      "learning_rate": 1.3258862496936412e-05,
      "logits/chosen": -2.2250709533691406,
      "logits/rejected": -2.0886082649230957,
      "logps/chosen": -273.227783203125,
      "logps/rejected": -303.0842590332031,
      "loss": 0.5032,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -2.895843267440796,
      "rewards/margins": 1.1852529048919678,
      "rewards/rejected": -4.081096172332764,
      "step": 3580
    },
    {
      "epoch": 0.7765623307700639,
      "grad_norm": 26.875,
      "learning_rate": 1.3254730560404935e-05,
      "logits/chosen": -2.326592445373535,
      "logits/rejected": -2.356827974319458,
      "logps/chosen": -280.0945739746094,
      "logps/rejected": -291.5137939453125,
      "loss": 0.5095,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.2928173542022705,
      "rewards/margins": 1.1421830654144287,
      "rewards/rejected": -4.435000419616699,
      "step": 3585
    },
    {
      "epoch": 0.777645402361096,
      "grad_norm": 20.125,
      "learning_rate": 1.325059264659956e-05,
      "logits/chosen": -2.047400951385498,
      "logits/rejected": -2.057119846343994,
      "logps/chosen": -292.58380126953125,
      "logps/rejected": -317.23382568359375,
      "loss": 0.4339,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.9949052333831787,
      "rewards/margins": 1.3290255069732666,
      "rewards/rejected": -4.323930263519287,
      "step": 3590
    },
    {
      "epoch": 0.7787284739521282,
      "grad_norm": 18.125,
      "learning_rate": 1.324644876142011e-05,
      "logits/chosen": -2.011967897415161,
      "logits/rejected": -1.9735959768295288,
      "logps/chosen": -285.25860595703125,
      "logps/rejected": -299.94061279296875,
      "loss": 0.4417,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -2.6324057579040527,
      "rewards/margins": 1.262708306312561,
      "rewards/rejected": -3.895113706588745,
      "step": 3595
    },
    {
      "epoch": 0.7798115455431605,
      "grad_norm": 20.875,
      "learning_rate": 1.3242298910774938e-05,
      "logits/chosen": -2.0955021381378174,
      "logits/rejected": -2.082380533218384,
      "logps/chosen": -291.531982421875,
      "logps/rejected": -301.78338623046875,
      "loss": 0.5037,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.058342933654785,
      "rewards/margins": 1.094801902770996,
      "rewards/rejected": -4.153144836425781,
      "step": 3600
    },
    {
      "epoch": 0.7808946171341926,
      "grad_norm": 21.75,
      "learning_rate": 1.3238143100580884e-05,
      "logits/chosen": -2.011601209640503,
      "logits/rejected": -2.0011677742004395,
      "logps/chosen": -282.0636901855469,
      "logps/rejected": -295.21038818359375,
      "loss": 0.4885,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.873103141784668,
      "rewards/margins": 1.158366322517395,
      "rewards/rejected": -4.031469821929932,
      "step": 3605
    },
    {
      "epoch": 0.7819776887252248,
      "grad_norm": 20.0,
      "learning_rate": 1.3233981336763299e-05,
      "logits/chosen": -1.9357506036758423,
      "logits/rejected": -1.8856064081192017,
      "logps/chosen": -275.1585693359375,
      "logps/rejected": -304.46087646484375,
      "loss": 0.4431,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -2.8434090614318848,
      "rewards/margins": 1.318665862083435,
      "rewards/rejected": -4.162075042724609,
      "step": 3610
    },
    {
      "epoch": 0.7830607603162569,
      "grad_norm": 15.1875,
      "learning_rate": 1.3229813625256014e-05,
      "logits/chosen": -2.0404982566833496,
      "logits/rejected": -2.0793778896331787,
      "logps/chosen": -292.4850158691406,
      "logps/rejected": -303.65740966796875,
      "loss": 0.4759,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.2972686290740967,
      "rewards/margins": 1.080260992050171,
      "rewards/rejected": -4.377529144287109,
      "step": 3615
    },
    {
      "epoch": 0.7841438319072891,
      "grad_norm": 21.375,
      "learning_rate": 1.3225639972001347e-05,
      "logits/chosen": -1.9076764583587646,
      "logits/rejected": -1.91448974609375,
      "logps/chosen": -289.5611877441406,
      "logps/rejected": -309.5157470703125,
      "loss": 0.5355,
      "rewards/accuracies": 0.7416667342185974,
      "rewards/chosen": -3.416983127593994,
      "rewards/margins": 1.1274241209030151,
      "rewards/rejected": -4.544407844543457,
      "step": 3620
    },
    {
      "epoch": 0.7852269034983212,
      "grad_norm": 20.625,
      "learning_rate": 1.3221460382950083e-05,
      "logits/chosen": -1.8729215860366821,
      "logits/rejected": -1.8392505645751953,
      "logps/chosen": -263.37152099609375,
      "logps/rejected": -296.8345642089844,
      "loss": 0.5059,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.612476110458374,
      "rewards/margins": 0.976012110710144,
      "rewards/rejected": -3.5884883403778076,
      "step": 3625
    },
    {
      "epoch": 0.7863099750893534,
      "grad_norm": 17.5,
      "learning_rate": 1.3217274864061473e-05,
      "logits/chosen": -1.7535299062728882,
      "logits/rejected": -1.8268436193466187,
      "logps/chosen": -287.9184875488281,
      "logps/rejected": -302.761962890625,
      "loss": 0.4545,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.00799560546875,
      "rewards/margins": 1.2970749139785767,
      "rewards/rejected": -4.305070400238037,
      "step": 3630
    },
    {
      "epoch": 0.7873930466803856,
      "grad_norm": 24.75,
      "learning_rate": 1.3213083421303218e-05,
      "logits/chosen": -1.7947819232940674,
      "logits/rejected": -1.7921946048736572,
      "logps/chosen": -267.5419616699219,
      "logps/rejected": -300.38616943359375,
      "loss": 0.4593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.250330686569214,
      "rewards/margins": 1.3595349788665771,
      "rewards/rejected": -3.609865665435791,
      "step": 3635
    },
    {
      "epoch": 0.7884761182714177,
      "grad_norm": 21.875,
      "learning_rate": 1.320888606065147e-05,
      "logits/chosen": -1.8269840478897095,
      "logits/rejected": -1.8164193630218506,
      "logps/chosen": -288.146484375,
      "logps/rejected": -300.3746032714844,
      "loss": 0.4596,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -3.0033295154571533,
      "rewards/margins": 1.2434450387954712,
      "rewards/rejected": -4.246774196624756,
      "step": 3640
    },
    {
      "epoch": 0.7895591898624499,
      "grad_norm": 16.0,
      "learning_rate": 1.320468278809082e-05,
      "logits/chosen": -1.9271929264068604,
      "logits/rejected": -1.8883157968521118,
      "logps/chosen": -254.2425537109375,
      "logps/rejected": -269.4250183105469,
      "loss": 0.4793,
      "rewards/accuracies": 0.7166667580604553,
      "rewards/chosen": -2.3458569049835205,
      "rewards/margins": 1.1751854419708252,
      "rewards/rejected": -3.5210423469543457,
      "step": 3645
    },
    {
      "epoch": 0.790642261453482,
      "grad_norm": 17.5,
      "learning_rate": 1.3200473609614282e-05,
      "logits/chosen": -1.8070513010025024,
      "logits/rejected": -1.7613712549209595,
      "logps/chosen": -272.3204650878906,
      "logps/rejected": -297.28753662109375,
      "loss": 0.4801,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.7602031230926514,
      "rewards/margins": 1.2372914552688599,
      "rewards/rejected": -3.9974944591522217,
      "step": 3650
    },
    {
      "epoch": 0.7917253330445142,
      "grad_norm": 22.0,
      "learning_rate": 1.3196258531223295e-05,
      "logits/chosen": -1.7161462306976318,
      "logits/rejected": -1.7076704502105713,
      "logps/chosen": -280.87200927734375,
      "logps/rejected": -307.0533752441406,
      "loss": 0.4804,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.624406337738037,
      "rewards/margins": 1.2485517263412476,
      "rewards/rejected": -3.872957706451416,
      "step": 3655
    },
    {
      "epoch": 0.7928084046355464,
      "grad_norm": 19.625,
      "learning_rate": 1.3192037558927714e-05,
      "logits/chosen": -1.670824408531189,
      "logits/rejected": -1.6626828908920288,
      "logps/chosen": -283.6042785644531,
      "logps/rejected": -306.0826110839844,
      "loss": 0.498,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.7098066806793213,
      "rewards/margins": 1.1786224842071533,
      "rewards/rejected": -3.8884284496307373,
      "step": 3660
    },
    {
      "epoch": 0.7938914762265786,
      "grad_norm": 16.125,
      "learning_rate": 1.3187810698745786e-05,
      "logits/chosen": -1.7894821166992188,
      "logits/rejected": -1.6527645587921143,
      "logps/chosen": -289.31585693359375,
      "logps/rejected": -323.32568359375,
      "loss": 0.4284,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -2.7134361267089844,
      "rewards/margins": 1.303762435913086,
      "rewards/rejected": -4.017198085784912,
      "step": 3665
    },
    {
      "epoch": 0.7949745478176108,
      "grad_norm": 16.125,
      "learning_rate": 1.3183577956704163e-05,
      "logits/chosen": -1.804645299911499,
      "logits/rejected": -1.8579552173614502,
      "logps/chosen": -285.9300231933594,
      "logps/rejected": -309.5963134765625,
      "loss": 0.4718,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.0524425506591797,
      "rewards/margins": 1.3175276517868042,
      "rewards/rejected": -4.369969844818115,
      "step": 3670
    },
    {
      "epoch": 0.796057619408643,
      "grad_norm": 24.25,
      "learning_rate": 1.3179339338837881e-05,
      "logits/chosen": -1.6900421380996704,
      "logits/rejected": -1.6686763763427734,
      "logps/chosen": -286.5816955566406,
      "logps/rejected": -300.953369140625,
      "loss": 0.5258,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -2.896825075149536,
      "rewards/margins": 1.0744065046310425,
      "rewards/rejected": -3.9712319374084473,
      "step": 3675
    },
    {
      "epoch": 0.7971406909996751,
      "grad_norm": 24.75,
      "learning_rate": 1.3175094851190352e-05,
      "logits/chosen": -1.7603079080581665,
      "logits/rejected": -1.7814266681671143,
      "logps/chosen": -287.53131103515625,
      "logps/rejected": -299.49322509765625,
      "loss": 0.5509,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.142821788787842,
      "rewards/margins": 0.8651250004768372,
      "rewards/rejected": -4.007946968078613,
      "step": 3680
    },
    {
      "epoch": 0.7982237625907073,
      "grad_norm": 16.125,
      "learning_rate": 1.3170844499813354e-05,
      "logits/chosen": -1.7642505168914795,
      "logits/rejected": -1.7544740438461304,
      "logps/chosen": -290.1250915527344,
      "logps/rejected": -316.7225341796875,
      "loss": 0.4003,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -2.816340446472168,
      "rewards/margins": 1.286742091178894,
      "rewards/rejected": -4.103082656860352,
      "step": 3685
    },
    {
      "epoch": 0.7993068341817394,
      "grad_norm": 15.6875,
      "learning_rate": 1.3166588290767037e-05,
      "logits/chosen": -1.945425271987915,
      "logits/rejected": -1.896842360496521,
      "logps/chosen": -269.65460205078125,
      "logps/rejected": -294.05291748046875,
      "loss": 0.4114,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -3.080601930618286,
      "rewards/margins": 1.317673921585083,
      "rewards/rejected": -4.398275375366211,
      "step": 3690
    },
    {
      "epoch": 0.8003899057727716,
      "grad_norm": 20.875,
      "learning_rate": 1.3162326230119887e-05,
      "logits/chosen": -1.841583490371704,
      "logits/rejected": -1.8742716312408447,
      "logps/chosen": -273.8441467285156,
      "logps/rejected": -288.3057556152344,
      "loss": 0.5118,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.8615777492523193,
      "rewards/margins": 1.1646783351898193,
      "rewards/rejected": -4.026256084442139,
      "step": 3695
    },
    {
      "epoch": 0.8014729773638037,
      "grad_norm": 22.5,
      "learning_rate": 1.3158058323948749e-05,
      "logits/chosen": -1.7098842859268188,
      "logits/rejected": -1.6466352939605713,
      "logps/chosen": -284.0284423828125,
      "logps/rejected": -300.66278076171875,
      "loss": 0.552,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -3.0215373039245605,
      "rewards/margins": 0.8271936178207397,
      "rewards/rejected": -3.8487308025360107,
      "step": 3700
    },
    {
      "epoch": 0.8025560489548359,
      "grad_norm": 25.625,
      "learning_rate": 1.3153784578338791e-05,
      "logits/chosen": -1.8450019359588623,
      "logits/rejected": -1.771222472190857,
      "logps/chosen": -270.4474792480469,
      "logps/rejected": -295.9034729003906,
      "loss": 0.4282,
      "rewards/accuracies": 0.7499999403953552,
      "rewards/chosen": -2.7865865230560303,
      "rewards/margins": 1.4661986827850342,
      "rewards/rejected": -4.252785682678223,
      "step": 3705
    },
    {
      "epoch": 0.8036391205458681,
      "grad_norm": 18.375,
      "learning_rate": 1.3149504999383511e-05,
      "logits/chosen": -1.7795867919921875,
      "logits/rejected": -1.799926996231079,
      "logps/chosen": -296.8211669921875,
      "logps/rejected": -301.42083740234375,
      "loss": 0.4533,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -3.0923075675964355,
      "rewards/margins": 1.2160956859588623,
      "rewards/rejected": -4.308403968811035,
      "step": 3710
    },
    {
      "epoch": 0.8047221921369002,
      "grad_norm": 21.75,
      "learning_rate": 1.3145219593184725e-05,
      "logits/chosen": -1.8396066427230835,
      "logits/rejected": -1.6836191415786743,
      "logps/chosen": -265.06683349609375,
      "logps/rejected": -314.2569580078125,
      "loss": 0.4037,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.9715139865875244,
      "rewards/margins": 1.4952093362808228,
      "rewards/rejected": -4.4667229652404785,
      "step": 3715
    },
    {
      "epoch": 0.8058052637279324,
      "grad_norm": 21.5,
      "learning_rate": 1.3140928365852557e-05,
      "logits/chosen": -1.8305270671844482,
      "logits/rejected": -1.7606685161590576,
      "logps/chosen": -285.8798522949219,
      "logps/rejected": -298.60601806640625,
      "loss": 0.5799,
      "rewards/accuracies": 0.6500000357627869,
      "rewards/chosen": -2.940795421600342,
      "rewards/margins": 0.8434941172599792,
      "rewards/rejected": -3.784289598464966,
      "step": 3720
    },
    {
      "epoch": 0.8068883353189645,
      "grad_norm": 23.625,
      "learning_rate": 1.313663132350543e-05,
      "logits/chosen": -1.7584108114242554,
      "logits/rejected": -1.7315380573272705,
      "logps/chosen": -281.89947509765625,
      "logps/rejected": -305.22357177734375,
      "loss": 0.5056,
      "rewards/accuracies": 0.7166666984558105,
      "rewards/chosen": -3.0225882530212402,
      "rewards/margins": 1.1932451725006104,
      "rewards/rejected": -4.21583366394043,
      "step": 3725
    },
    {
      "epoch": 0.8079714069099968,
      "grad_norm": 24.5,
      "learning_rate": 1.3132328472270057e-05,
      "logits/chosen": -1.800616979598999,
      "logits/rejected": -1.7144044637680054,
      "logps/chosen": -271.3162536621094,
      "logps/rejected": -288.1465148925781,
      "loss": 0.5099,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.8021256923675537,
      "rewards/margins": 1.2015607357025146,
      "rewards/rejected": -4.00368595123291,
      "step": 3730
    },
    {
      "epoch": 0.809054478501029,
      "grad_norm": 22.5,
      "learning_rate": 1.3128019818281436e-05,
      "logits/chosen": -1.57765793800354,
      "logits/rejected": -1.6909081935882568,
      "logps/chosen": -278.2663879394531,
      "logps/rejected": -300.29486083984375,
      "loss": 0.4195,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -2.940337657928467,
      "rewards/margins": 1.3967081308364868,
      "rewards/rejected": -4.337046146392822,
      "step": 3735
    },
    {
      "epoch": 0.8101375500920611,
      "grad_norm": 15.3125,
      "learning_rate": 1.3123705367682835e-05,
      "logits/chosen": -1.5019893646240234,
      "logits/rejected": -1.5783658027648926,
      "logps/chosen": -299.95849609375,
      "logps/rejected": -306.6734924316406,
      "loss": 0.4615,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.922496795654297,
      "rewards/margins": 1.1513136625289917,
      "rewards/rejected": -4.07381010055542,
      "step": 3740
    },
    {
      "epoch": 0.8112206216830933,
      "grad_norm": 15.6875,
      "learning_rate": 1.3119385126625792e-05,
      "logits/chosen": -1.4579459428787231,
      "logits/rejected": -1.456383466720581,
      "logps/chosen": -270.6466369628906,
      "logps/rejected": -295.97222900390625,
      "loss": 0.3869,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -2.2886223793029785,
      "rewards/margins": 1.5476120710372925,
      "rewards/rejected": -3.8362345695495605,
      "step": 3745
    },
    {
      "epoch": 0.8123036932741254,
      "grad_norm": 16.125,
      "learning_rate": 1.3115059101270099e-05,
      "logits/chosen": -1.5519174337387085,
      "logits/rejected": -1.456413984298706,
      "logps/chosen": -269.2893981933594,
      "logps/rejected": -294.25030517578125,
      "loss": 0.4377,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.2507901191711426,
      "rewards/margins": 1.5035457611083984,
      "rewards/rejected": -3.75433611869812,
      "step": 3750
    },
    {
      "epoch": 0.8133867648651576,
      "grad_norm": 20.0,
      "learning_rate": 1.3110727297783787e-05,
      "logits/chosen": -1.4651787281036377,
      "logits/rejected": -1.4194777011871338,
      "logps/chosen": -270.90362548828125,
      "logps/rejected": -308.4999694824219,
      "loss": 0.4825,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.484057664871216,
      "rewards/margins": 1.2844593524932861,
      "rewards/rejected": -3.768517017364502,
      "step": 3755
    },
    {
      "epoch": 0.8144698364561898,
      "grad_norm": 13.1875,
      "learning_rate": 1.3106389722343142e-05,
      "logits/chosen": -1.5066286325454712,
      "logits/rejected": -1.5508878231048584,
      "logps/chosen": -275.99029541015625,
      "logps/rejected": -286.07391357421875,
      "loss": 0.4512,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -2.5534474849700928,
      "rewards/margins": 1.3505537509918213,
      "rewards/rejected": -3.904001235961914,
      "step": 3760
    },
    {
      "epoch": 0.8155529080472219,
      "grad_norm": 15.3125,
      "learning_rate": 1.3102046381132665e-05,
      "logits/chosen": -1.4775007963180542,
      "logits/rejected": -1.482452154159546,
      "logps/chosen": -286.2796936035156,
      "logps/rejected": -314.97625732421875,
      "loss": 0.4028,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.881981372833252,
      "rewards/margins": 1.3198678493499756,
      "rewards/rejected": -4.201849460601807,
      "step": 3765
    },
    {
      "epoch": 0.8166359796382541,
      "grad_norm": 21.0,
      "learning_rate": 1.3097697280345081e-05,
      "logits/chosen": -1.5666784048080444,
      "logits/rejected": -1.6286227703094482,
      "logps/chosen": -280.946533203125,
      "logps/rejected": -301.7801818847656,
      "loss": 0.4981,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.1144702434539795,
      "rewards/margins": 1.0940163135528564,
      "rewards/rejected": -4.208486557006836,
      "step": 3770
    },
    {
      "epoch": 0.8177190512292862,
      "grad_norm": 19.375,
      "learning_rate": 1.3093342426181338e-05,
      "logits/chosen": -1.5650714635849,
      "logits/rejected": -1.544999122619629,
      "logps/chosen": -278.24365234375,
      "logps/rejected": -294.66387939453125,
      "loss": 0.4128,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -2.98185396194458,
      "rewards/margins": 1.4315941333770752,
      "rewards/rejected": -4.413448333740234,
      "step": 3775
    },
    {
      "epoch": 0.8188021228203184,
      "grad_norm": 21.25,
      "learning_rate": 1.308898182485057e-05,
      "logits/chosen": -1.5942405462265015,
      "logits/rejected": -1.5314041376113892,
      "logps/chosen": -281.29705810546875,
      "logps/rejected": -303.44232177734375,
      "loss": 0.5265,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -3.3402457237243652,
      "rewards/margins": 0.8787516355514526,
      "rewards/rejected": -4.218997478485107,
      "step": 3780
    },
    {
      "epoch": 0.8198851944113505,
      "grad_norm": 14.0,
      "learning_rate": 1.3084615482570114e-05,
      "logits/chosen": -1.5037797689437866,
      "logits/rejected": -1.5433540344238281,
      "logps/chosen": -287.6862487792969,
      "logps/rejected": -303.5072326660156,
      "loss": 0.4781,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -2.530308723449707,
      "rewards/margins": 1.3783375024795532,
      "rewards/rejected": -3.9086456298828125,
      "step": 3785
    },
    {
      "epoch": 0.8209682660023827,
      "grad_norm": 13.25,
      "learning_rate": 1.3080243405565496e-05,
      "logits/chosen": -1.7300026416778564,
      "logits/rejected": -1.6821842193603516,
      "logps/chosen": -282.72821044921875,
      "logps/rejected": -312.9586486816406,
      "loss": 0.47,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.4068989753723145,
      "rewards/margins": 1.2348637580871582,
      "rewards/rejected": -4.641762733459473,
      "step": 3790
    },
    {
      "epoch": 0.822051337593415,
      "grad_norm": 23.5,
      "learning_rate": 1.3075865600070414e-05,
      "logits/chosen": -1.6592485904693604,
      "logits/rejected": -1.644054651260376,
      "logps/chosen": -309.1129150390625,
      "logps/rejected": -331.02874755859375,
      "loss": 0.4145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2853684425354004,
      "rewards/margins": 1.4981034994125366,
      "rewards/rejected": -4.783471584320068,
      "step": 3795
    },
    {
      "epoch": 0.8231344091844471,
      "grad_norm": 19.375,
      "learning_rate": 1.3071482072326731e-05,
      "logits/chosen": -1.761217474937439,
      "logits/rejected": -1.6815162897109985,
      "logps/chosen": -273.0993957519531,
      "logps/rejected": -307.59271240234375,
      "loss": 0.4331,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.092329502105713,
      "rewards/margins": 1.5219217538833618,
      "rewards/rejected": -4.614251136779785,
      "step": 3800
    },
    {
      "epoch": 0.8242174807754793,
      "grad_norm": 22.0,
      "learning_rate": 1.3067092828584477e-05,
      "logits/chosen": -1.6965110301971436,
      "logits/rejected": -1.7744449377059937,
      "logps/chosen": -273.7412109375,
      "logps/rejected": -299.361083984375,
      "loss": 0.3833,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.40649151802063,
      "rewards/margins": 1.6201807260513306,
      "rewards/rejected": -4.026671886444092,
      "step": 3805
    },
    {
      "epoch": 0.8253005523665115,
      "grad_norm": 17.125,
      "learning_rate": 1.3062697875101823e-05,
      "logits/chosen": -1.7635695934295654,
      "logits/rejected": -1.7610403299331665,
      "logps/chosen": -287.591552734375,
      "logps/rejected": -326.49798583984375,
      "loss": 0.4393,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.339456558227539,
      "rewards/margins": 1.497575044631958,
      "rewards/rejected": -4.837031364440918,
      "step": 3810
    },
    {
      "epoch": 0.8263836239575436,
      "grad_norm": 11.1875,
      "learning_rate": 1.3058297218145086e-05,
      "logits/chosen": -1.7007697820663452,
      "logits/rejected": -1.7216050624847412,
      "logps/chosen": -254.4221649169922,
      "logps/rejected": -284.3150329589844,
      "loss": 0.3843,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -2.6428372859954834,
      "rewards/margins": 1.4580879211425781,
      "rewards/rejected": -4.100925922393799,
      "step": 3815
    },
    {
      "epoch": 0.8274666955485758,
      "grad_norm": 13.3125,
      "learning_rate": 1.3053890863988712e-05,
      "logits/chosen": -1.6312252283096313,
      "logits/rejected": -1.6798884868621826,
      "logps/chosen": -291.1435241699219,
      "logps/rejected": -305.0799865722656,
      "loss": 0.4425,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.233151912689209,
      "rewards/margins": 1.4268620014190674,
      "rewards/rejected": -4.660013675689697,
      "step": 3820
    },
    {
      "epoch": 0.8285497671396079,
      "grad_norm": 22.625,
      "learning_rate": 1.3049478818915274e-05,
      "logits/chosen": -1.6705020666122437,
      "logits/rejected": -1.6263341903686523,
      "logps/chosen": -305.22698974609375,
      "logps/rejected": -332.61199951171875,
      "loss": 0.5143,
      "rewards/accuracies": 0.7083333730697632,
      "rewards/chosen": -3.644498109817505,
      "rewards/margins": 1.0895346403121948,
      "rewards/rejected": -4.734033107757568,
      "step": 3825
    },
    {
      "epoch": 0.8296328387306401,
      "grad_norm": 19.0,
      "learning_rate": 1.3045061089215458e-05,
      "logits/chosen": -1.701055884361267,
      "logits/rejected": -1.6327035427093506,
      "logps/chosen": -288.7308349609375,
      "logps/rejected": -321.81878662109375,
      "loss": 0.3905,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.2797279357910156,
      "rewards/margins": 1.561493158340454,
      "rewards/rejected": -4.841221809387207,
      "step": 3830
    },
    {
      "epoch": 0.8307159103216722,
      "grad_norm": 24.5,
      "learning_rate": 1.3040637681188053e-05,
      "logits/chosen": -1.7614498138427734,
      "logits/rejected": -1.6734787225723267,
      "logps/chosen": -291.728759765625,
      "logps/rejected": -305.41729736328125,
      "loss": 0.5238,
      "rewards/accuracies": 0.6916666626930237,
      "rewards/chosen": -3.1405816078186035,
      "rewards/margins": 1.024078607559204,
      "rewards/rejected": -4.164660453796387,
      "step": 3835
    },
    {
      "epoch": 0.8317989819127044,
      "grad_norm": 22.375,
      "learning_rate": 1.3036208601139944e-05,
      "logits/chosen": -1.6743911504745483,
      "logits/rejected": -1.7061469554901123,
      "logps/chosen": -286.3898620605469,
      "logps/rejected": -326.75897216796875,
      "loss": 0.4761,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.0324153900146484,
      "rewards/margins": 1.6341893672943115,
      "rewards/rejected": -4.666604995727539,
      "step": 3840
    },
    {
      "epoch": 0.8328820535037366,
      "grad_norm": 20.25,
      "learning_rate": 1.3031773855386106e-05,
      "logits/chosen": -1.8183238506317139,
      "logits/rejected": -1.750054121017456,
      "logps/chosen": -294.6192626953125,
      "logps/rejected": -322.7720642089844,
      "loss": 0.4427,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.5847771167755127,
      "rewards/margins": 1.3745603561401367,
      "rewards/rejected": -4.959338188171387,
      "step": 3845
    },
    {
      "epoch": 0.8339651250947687,
      "grad_norm": 19.5,
      "learning_rate": 1.3027333450249589e-05,
      "logits/chosen": -1.7171401977539062,
      "logits/rejected": -1.7613847255706787,
      "logps/chosen": -288.0212707519531,
      "logps/rejected": -301.60760498046875,
      "loss": 0.5333,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.3027865886688232,
      "rewards/margins": 1.022318959236145,
      "rewards/rejected": -4.325105667114258,
      "step": 3850
    },
    {
      "epoch": 0.8350481966858009,
      "grad_norm": 15.8125,
      "learning_rate": 1.3022887392061517e-05,
      "logits/chosen": -1.5975449085235596,
      "logits/rejected": -1.593679666519165,
      "logps/chosen": -277.0356140136719,
      "logps/rejected": -298.6958923339844,
      "loss": 0.4003,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.8223094940185547,
      "rewards/margins": 1.3940777778625488,
      "rewards/rejected": -4.216387748718262,
      "step": 3855
    },
    {
      "epoch": 0.8361312682768332,
      "grad_norm": 19.5,
      "learning_rate": 1.3018435687161067e-05,
      "logits/chosen": -1.7284901142120361,
      "logits/rejected": -1.7042710781097412,
      "logps/chosen": -271.894775390625,
      "logps/rejected": -304.96063232421875,
      "loss": 0.4743,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -2.9613537788391113,
      "rewards/margins": 1.3395583629608154,
      "rewards/rejected": -4.300912380218506,
      "step": 3860
    },
    {
      "epoch": 0.8372143398678653,
      "grad_norm": 19.875,
      "learning_rate": 1.301397834189547e-05,
      "logits/chosen": -1.7351185083389282,
      "logits/rejected": -1.754831075668335,
      "logps/chosen": -291.2852478027344,
      "logps/rejected": -308.6584777832031,
      "loss": 0.427,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.948509931564331,
      "rewards/margins": 1.3647925853729248,
      "rewards/rejected": -4.313302516937256,
      "step": 3865
    },
    {
      "epoch": 0.8382974114588975,
      "grad_norm": 21.875,
      "learning_rate": 1.3009515362620006e-05,
      "logits/chosen": -1.8238605260849,
      "logits/rejected": -1.7771332263946533,
      "logps/chosen": -299.0949401855469,
      "logps/rejected": -324.9104919433594,
      "loss": 0.4652,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.1855885982513428,
      "rewards/margins": 1.4027221202850342,
      "rewards/rejected": -4.588311195373535,
      "step": 3870
    },
    {
      "epoch": 0.8393804830499296,
      "grad_norm": 21.375,
      "learning_rate": 1.3005046755697977e-05,
      "logits/chosen": -1.801393747329712,
      "logits/rejected": -1.8948405981063843,
      "logps/chosen": -288.3680114746094,
      "logps/rejected": -292.5406799316406,
      "loss": 0.46,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.7821788787841797,
      "rewards/margins": 1.3986035585403442,
      "rewards/rejected": -4.180782318115234,
      "step": 3875
    },
    {
      "epoch": 0.8404635546409618,
      "grad_norm": 14.1875,
      "learning_rate": 1.3000572527500714e-05,
      "logits/chosen": -1.759232759475708,
      "logits/rejected": -1.6815086603164673,
      "logps/chosen": -271.7196350097656,
      "logps/rejected": -309.98321533203125,
      "loss": 0.4102,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.585984230041504,
      "rewards/margins": 1.4460833072662354,
      "rewards/rejected": -4.032067775726318,
      "step": 3880
    },
    {
      "epoch": 0.8415466262319939,
      "grad_norm": 20.75,
      "learning_rate": 1.2996092684407565e-05,
      "logits/chosen": -1.8145349025726318,
      "logits/rejected": -1.7436721324920654,
      "logps/chosen": -267.15435791015625,
      "logps/rejected": -297.32293701171875,
      "loss": 0.457,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.962905168533325,
      "rewards/margins": 1.2825720310211182,
      "rewards/rejected": -4.245477199554443,
      "step": 3885
    },
    {
      "epoch": 0.8426296978230261,
      "grad_norm": 22.25,
      "learning_rate": 1.299160723280588e-05,
      "logits/chosen": -1.6349022388458252,
      "logits/rejected": -1.6959747076034546,
      "logps/chosen": -273.8650817871094,
      "logps/rejected": -299.7972717285156,
      "loss": 0.3966,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.6532816886901855,
      "rewards/margins": 1.5337765216827393,
      "rewards/rejected": -4.187058448791504,
      "step": 3890
    },
    {
      "epoch": 0.8437127694140583,
      "grad_norm": 22.375,
      "learning_rate": 1.2987116179091005e-05,
      "logits/chosen": -1.7174657583236694,
      "logits/rejected": -1.6874405145645142,
      "logps/chosen": -276.0401306152344,
      "logps/rejected": -303.6053161621094,
      "loss": 0.4039,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -3.1231765747070312,
      "rewards/margins": 1.5495916604995728,
      "rewards/rejected": -4.672768592834473,
      "step": 3895
    },
    {
      "epoch": 0.8447958410050904,
      "grad_norm": 20.375,
      "learning_rate": 1.2982619529666279e-05,
      "logits/chosen": -1.8088724613189697,
      "logits/rejected": -1.8155714273452759,
      "logps/chosen": -268.1481018066406,
      "logps/rejected": -309.6062927246094,
      "loss": 0.4238,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.8756632804870605,
      "rewards/margins": 1.6329158544540405,
      "rewards/rejected": -4.508578777313232,
      "step": 3900
    },
    {
      "epoch": 0.8458789125961226,
      "grad_norm": 19.125,
      "learning_rate": 1.2978117290943012e-05,
      "logits/chosen": -1.7787158489227295,
      "logits/rejected": -1.7777328491210938,
      "logps/chosen": -281.10003662109375,
      "logps/rejected": -303.04437255859375,
      "loss": 0.3484,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.0567879676818848,
      "rewards/margins": 1.740696668624878,
      "rewards/rejected": -4.797484874725342,
      "step": 3905
    },
    {
      "epoch": 0.8469619841871547,
      "grad_norm": 15.6875,
      "learning_rate": 1.2973609469340489e-05,
      "logits/chosen": -1.8904567956924438,
      "logits/rejected": -1.9208710193634033,
      "logps/chosen": -286.8519592285156,
      "logps/rejected": -313.3398132324219,
      "loss": 0.4151,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -3.1845273971557617,
      "rewards/margins": 1.5163817405700684,
      "rewards/rejected": -4.700909614562988,
      "step": 3910
    },
    {
      "epoch": 0.8480450557781869,
      "grad_norm": 17.5,
      "learning_rate": 1.2969096071285954e-05,
      "logits/chosen": -1.8430448770523071,
      "logits/rejected": -1.815605878829956,
      "logps/chosen": -280.177001953125,
      "logps/rejected": -310.3946838378906,
      "loss": 0.4279,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -2.5302090644836426,
      "rewards/margins": 1.54547917842865,
      "rewards/rejected": -4.075688362121582,
      "step": 3915
    },
    {
      "epoch": 0.849128127369219,
      "grad_norm": 21.75,
      "learning_rate": 1.29645771032146e-05,
      "logits/chosen": -1.9107954502105713,
      "logits/rejected": -1.8694734573364258,
      "logps/chosen": -265.66827392578125,
      "logps/rejected": -294.3473205566406,
      "loss": 0.4577,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.0053439140319824,
      "rewards/margins": 1.4059840440750122,
      "rewards/rejected": -4.411328315734863,
      "step": 3920
    },
    {
      "epoch": 0.8502111989602513,
      "grad_norm": 17.0,
      "learning_rate": 1.296005257156956e-05,
      "logits/chosen": -1.7647573947906494,
      "logits/rejected": -1.9031846523284912,
      "logps/chosen": -284.50152587890625,
      "logps/rejected": -294.2745666503906,
      "loss": 0.4266,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.895958185195923,
      "rewards/margins": 1.380183219909668,
      "rewards/rejected": -4.27614164352417,
      "step": 3925
    },
    {
      "epoch": 0.8512942705512835,
      "grad_norm": 16.75,
      "learning_rate": 1.2955522482801905e-05,
      "logits/chosen": -1.892054557800293,
      "logits/rejected": -1.910447120666504,
      "logps/chosen": -275.14971923828125,
      "logps/rejected": -294.9456481933594,
      "loss": 0.4,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.1882214546203613,
      "rewards/margins": 1.414692759513855,
      "rewards/rejected": -4.602914333343506,
      "step": 3930
    },
    {
      "epoch": 0.8523773421423156,
      "grad_norm": 13.1875,
      "learning_rate": 1.2950986843370626e-05,
      "logits/chosen": -1.8852369785308838,
      "logits/rejected": -1.841072678565979,
      "logps/chosen": -278.11029052734375,
      "logps/rejected": -307.42059326171875,
      "loss": 0.4537,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.0534040927886963,
      "rewards/margins": 1.3781583309173584,
      "rewards/rejected": -4.4315619468688965,
      "step": 3935
    },
    {
      "epoch": 0.8534604137333478,
      "grad_norm": 18.875,
      "learning_rate": 1.2946445659742626e-05,
      "logits/chosen": -1.7894608974456787,
      "logits/rejected": -1.878432273864746,
      "logps/chosen": -291.19171142578125,
      "logps/rejected": -295.3894958496094,
      "loss": 0.4176,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -2.9212231636047363,
      "rewards/margins": 1.4170219898223877,
      "rewards/rejected": -4.338245391845703,
      "step": 3940
    },
    {
      "epoch": 0.85454348532438,
      "grad_norm": 23.0,
      "learning_rate": 1.2941898938392722e-05,
      "logits/chosen": -1.7067220211029053,
      "logits/rejected": -1.800050973892212,
      "logps/chosen": -310.11785888671875,
      "logps/rejected": -315.46014404296875,
      "loss": 0.4002,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -3.1145589351654053,
      "rewards/margins": 1.4382915496826172,
      "rewards/rejected": -4.552850246429443,
      "step": 3945
    },
    {
      "epoch": 0.8556265569154121,
      "grad_norm": 19.5,
      "learning_rate": 1.2937346685803612e-05,
      "logits/chosen": -1.7949994802474976,
      "logits/rejected": -1.8257646560668945,
      "logps/chosen": -282.5219421386719,
      "logps/rejected": -304.46881103515625,
      "loss": 0.4279,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -3.1618130207061768,
      "rewards/margins": 1.5573699474334717,
      "rewards/rejected": -4.719182968139648,
      "step": 3950
    },
    {
      "epoch": 0.8567096285064443,
      "grad_norm": 23.625,
      "learning_rate": 1.2932788908465895e-05,
      "logits/chosen": -1.7710742950439453,
      "logits/rejected": -1.7416614294052124,
      "logps/chosen": -277.455810546875,
      "logps/rejected": -306.8828125,
      "loss": 0.498,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.0135350227355957,
      "rewards/margins": 1.239375352859497,
      "rewards/rejected": -4.252910137176514,
      "step": 3955
    },
    {
      "epoch": 0.8577927000974764,
      "grad_norm": 25.375,
      "learning_rate": 1.2928225612878036e-05,
      "logits/chosen": -1.7910830974578857,
      "logits/rejected": -1.7370498180389404,
      "logps/chosen": -298.0028991699219,
      "logps/rejected": -327.0425720214844,
      "loss": 0.4444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.3877310752868652,
      "rewards/margins": 1.4595117568969727,
      "rewards/rejected": -4.847242832183838,
      "step": 3960
    },
    {
      "epoch": 0.8588757716885086,
      "grad_norm": 17.5,
      "learning_rate": 1.2923656805546377e-05,
      "logits/chosen": -1.799102544784546,
      "logits/rejected": -1.8553287982940674,
      "logps/chosen": -283.2781677246094,
      "logps/rejected": -299.0710754394531,
      "loss": 0.4628,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -2.958505630493164,
      "rewards/margins": 1.2008638381958008,
      "rewards/rejected": -4.159369468688965,
      "step": 3965
    },
    {
      "epoch": 0.8599588432795408,
      "grad_norm": 17.375,
      "learning_rate": 1.291908249298511e-05,
      "logits/chosen": -1.7266849279403687,
      "logits/rejected": -1.720679521560669,
      "logps/chosen": -267.91729736328125,
      "logps/rejected": -291.7174377441406,
      "loss": 0.4228,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.6325936317443848,
      "rewards/margins": 1.5607788562774658,
      "rewards/rejected": -4.1933722496032715,
      "step": 3970
    },
    {
      "epoch": 0.8610419148705729,
      "grad_norm": 23.125,
      "learning_rate": 1.2914502681716285e-05,
      "logits/chosen": -1.9181123971939087,
      "logits/rejected": -1.9337173700332642,
      "logps/chosen": -265.8172302246094,
      "logps/rejected": -295.81341552734375,
      "loss": 0.4133,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.800363063812256,
      "rewards/margins": 1.589342713356018,
      "rewards/rejected": -4.389705657958984,
      "step": 3975
    },
    {
      "epoch": 0.8621249864616051,
      "grad_norm": 18.0,
      "learning_rate": 1.2909917378269784e-05,
      "logits/chosen": -1.8409006595611572,
      "logits/rejected": -1.7629457712173462,
      "logps/chosen": -297.04058837890625,
      "logps/rejected": -325.4743347167969,
      "loss": 0.4706,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.3464512825012207,
      "rewards/margins": 1.334627389907837,
      "rewards/rejected": -4.681078910827637,
      "step": 3980
    },
    {
      "epoch": 0.8632080580526372,
      "grad_norm": 17.875,
      "learning_rate": 1.2905326589183324e-05,
      "logits/chosen": -1.854386329650879,
      "logits/rejected": -1.8846368789672852,
      "logps/chosen": -278.9513854980469,
      "logps/rejected": -293.97320556640625,
      "loss": 0.428,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.048725128173828,
      "rewards/margins": 1.2981363534927368,
      "rewards/rejected": -4.346861362457275,
      "step": 3985
    },
    {
      "epoch": 0.8642911296436695,
      "grad_norm": 16.75,
      "learning_rate": 1.2900730321002446e-05,
      "logits/chosen": -1.9660873413085938,
      "logits/rejected": -2.046487808227539,
      "logps/chosen": -271.1876525878906,
      "logps/rejected": -293.8605041503906,
      "loss": 0.3925,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.0603127479553223,
      "rewards/margins": 1.4821852445602417,
      "rewards/rejected": -4.5424981117248535,
      "step": 3990
    },
    {
      "epoch": 0.8653742012347017,
      "grad_norm": 16.75,
      "learning_rate": 1.2896128580280498e-05,
      "logits/chosen": -1.9985088109970093,
      "logits/rejected": -1.8972314596176147,
      "logps/chosen": -279.0354919433594,
      "logps/rejected": -302.2833557128906,
      "loss": 0.4582,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.177802085876465,
      "rewards/margins": 1.3672856092453003,
      "rewards/rejected": -4.545087814331055,
      "step": 3995
    },
    {
      "epoch": 0.8664572728257338,
      "grad_norm": 20.125,
      "learning_rate": 1.289152137357863e-05,
      "logits/chosen": -1.732007622718811,
      "logits/rejected": -1.703741431236267,
      "logps/chosen": -295.22760009765625,
      "logps/rejected": -330.69439697265625,
      "loss": 0.4532,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -2.784867286682129,
      "rewards/margins": 1.3936280012130737,
      "rewards/rejected": -4.178495407104492,
      "step": 4000
    },
    {
      "epoch": 0.8664572728257338,
      "eval_logits/chosen": -2.071317434310913,
      "eval_logits/rejected": -2.012477159500122,
      "eval_logps/chosen": -272.1488037109375,
      "eval_logps/rejected": -299.9363098144531,
      "eval_loss": 0.5017386078834534,
      "eval_rewards/accuracies": 0.7149999737739563,
      "eval_rewards/chosen": -2.8973772525787354,
      "eval_rewards/margins": 1.2165803909301758,
      "eval_rewards/rejected": -4.11395788192749,
      "eval_runtime": 14.4125,
      "eval_samples_per_second": 13.877,
      "eval_steps_per_second": 13.877,
      "step": 4000
    },
    {
      "epoch": 0.867540344416766,
      "grad_norm": 21.875,
      "learning_rate": 1.2886908707465792e-05,
      "logits/chosen": -2.149946928024292,
      "logits/rejected": -2.131511688232422,
      "logps/chosen": -283.2459716796875,
      "logps/rejected": -323.13104248046875,
      "loss": 0.3841,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.301548719406128,
      "rewards/margins": 1.6605708599090576,
      "rewards/rejected": -4.9621195793151855,
      "step": 4005
    },
    {
      "epoch": 0.8686234160077981,
      "grad_norm": 24.5,
      "learning_rate": 1.2882290588518712e-05,
      "logits/chosen": -2.1234331130981445,
      "logits/rejected": -2.084108829498291,
      "logps/chosen": -285.7658386230469,
      "logps/rejected": -324.86553955078125,
      "loss": 0.4579,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.695234775543213,
      "rewards/margins": 1.5725113153457642,
      "rewards/rejected": -5.267745494842529,
      "step": 4010
    },
    {
      "epoch": 0.8697064875988303,
      "grad_norm": 19.875,
      "learning_rate": 1.2877667023321897e-05,
      "logits/chosen": -2.0407419204711914,
      "logits/rejected": -2.015709400177002,
      "logps/chosen": -279.9422607421875,
      "logps/rejected": -311.7301330566406,
      "loss": 0.4161,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.3266055583953857,
      "rewards/margins": 1.5765469074249268,
      "rewards/rejected": -4.903152942657471,
      "step": 4015
    },
    {
      "epoch": 0.8707895591898624,
      "grad_norm": 20.5,
      "learning_rate": 1.2873038018467613e-05,
      "logits/chosen": -2.0711147785186768,
      "logits/rejected": -2.0756094455718994,
      "logps/chosen": -284.4831237792969,
      "logps/rejected": -309.5926513671875,
      "loss": 0.4789,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.6655399799346924,
      "rewards/margins": 1.3818371295928955,
      "rewards/rejected": -5.047377109527588,
      "step": 4020
    },
    {
      "epoch": 0.8718726307808946,
      "grad_norm": 26.625,
      "learning_rate": 1.2868403580555888e-05,
      "logits/chosen": -1.9412555694580078,
      "logits/rejected": -1.9154335260391235,
      "logps/chosen": -277.01580810546875,
      "logps/rejected": -305.2056884765625,
      "loss": 0.4964,
      "rewards/accuracies": 0.7250000834465027,
      "rewards/chosen": -2.8203697204589844,
      "rewards/margins": 1.4264953136444092,
      "rewards/rejected": -4.246865272521973,
      "step": 4025
    },
    {
      "epoch": 0.8729557023719268,
      "grad_norm": 21.5,
      "learning_rate": 1.2863763716194495e-05,
      "logits/chosen": -1.9198758602142334,
      "logits/rejected": -1.8608754873275757,
      "logps/chosen": -271.8978576660156,
      "logps/rejected": -297.3638916015625,
      "loss": 0.4366,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -2.8565475940704346,
      "rewards/margins": 1.4782770872116089,
      "rewards/rejected": -4.334824085235596,
      "step": 4030
    },
    {
      "epoch": 0.8740387739629589,
      "grad_norm": 20.75,
      "learning_rate": 1.2859118431998942e-05,
      "logits/chosen": -1.925126314163208,
      "logits/rejected": -1.9051227569580078,
      "logps/chosen": -276.4223327636719,
      "logps/rejected": -292.7510681152344,
      "loss": 0.5042,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.227837085723877,
      "rewards/margins": 1.1613997220993042,
      "rewards/rejected": -4.389237403869629,
      "step": 4035
    },
    {
      "epoch": 0.8751218455539911,
      "grad_norm": 18.875,
      "learning_rate": 1.2854467734592468e-05,
      "logits/chosen": -1.9683687686920166,
      "logits/rejected": -1.9263534545898438,
      "logps/chosen": -272.24176025390625,
      "logps/rejected": -302.0603942871094,
      "loss": 0.4001,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.9441254138946533,
      "rewards/margins": 1.5865839719772339,
      "rewards/rejected": -4.530709266662598,
      "step": 4040
    },
    {
      "epoch": 0.8762049171450232,
      "grad_norm": 22.25,
      "learning_rate": 1.2849811630606024e-05,
      "logits/chosen": -1.816166639328003,
      "logits/rejected": -1.7756894826889038,
      "logps/chosen": -277.3994445800781,
      "logps/rejected": -302.8851013183594,
      "loss": 0.4697,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.1469337940216064,
      "rewards/margins": 1.284557580947876,
      "rewards/rejected": -4.431491374969482,
      "step": 4045
    },
    {
      "epoch": 0.8772879887360554,
      "grad_norm": 19.125,
      "learning_rate": 1.2845150126678277e-05,
      "logits/chosen": -1.8421332836151123,
      "logits/rejected": -1.897733449935913,
      "logps/chosen": -315.3729248046875,
      "logps/rejected": -333.8653869628906,
      "loss": 0.466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.7569289207458496,
      "rewards/margins": 1.2837069034576416,
      "rewards/rejected": -5.040635585784912,
      "step": 4050
    },
    {
      "epoch": 0.8783710603270877,
      "grad_norm": 14.375,
      "learning_rate": 1.284048322945559e-05,
      "logits/chosen": -1.8562179803848267,
      "logits/rejected": -1.7429332733154297,
      "logps/chosen": -275.7851257324219,
      "logps/rejected": -320.33929443359375,
      "loss": 0.4043,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.804938793182373,
      "rewards/margins": 1.4091761112213135,
      "rewards/rejected": -4.214115142822266,
      "step": 4055
    },
    {
      "epoch": 0.8794541319181198,
      "grad_norm": 21.25,
      "learning_rate": 1.2835810945592013e-05,
      "logits/chosen": -1.896909475326538,
      "logits/rejected": -1.81813645362854,
      "logps/chosen": -286.8065185546875,
      "logps/rejected": -315.74151611328125,
      "loss": 0.389,
      "rewards/accuracies": 0.8166667819023132,
      "rewards/chosen": -2.8337457180023193,
      "rewards/margins": 1.5066168308258057,
      "rewards/rejected": -4.340362548828125,
      "step": 4060
    },
    {
      "epoch": 0.880537203509152,
      "grad_norm": 21.5,
      "learning_rate": 1.2831133281749283e-05,
      "logits/chosen": -1.8072153329849243,
      "logits/rejected": -1.7612260580062866,
      "logps/chosen": -282.85736083984375,
      "logps/rejected": -305.3613586425781,
      "loss": 0.4697,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.8138058185577393,
      "rewards/margins": 1.266601800918579,
      "rewards/rejected": -4.08040714263916,
      "step": 4065
    },
    {
      "epoch": 0.8816202751001841,
      "grad_norm": 20.375,
      "learning_rate": 1.28264502445968e-05,
      "logits/chosen": -1.9924131631851196,
      "logits/rejected": -1.973728895187378,
      "logps/chosen": -269.0211486816406,
      "logps/rejected": -293.8561096191406,
      "loss": 0.4288,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -2.830986261367798,
      "rewards/margins": 1.4554232358932495,
      "rewards/rejected": -4.286409854888916,
      "step": 4070
    },
    {
      "epoch": 0.8827033466912163,
      "grad_norm": 19.0,
      "learning_rate": 1.2821761840811629e-05,
      "logits/chosen": -1.9693931341171265,
      "logits/rejected": -1.8455235958099365,
      "logps/chosen": -270.1879577636719,
      "logps/rejected": -304.4454650878906,
      "loss": 0.436,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -2.4611945152282715,
      "rewards/margins": 1.6331920623779297,
      "rewards/rejected": -4.094387054443359,
      "step": 4075
    },
    {
      "epoch": 0.8837864182822485,
      "grad_norm": 23.75,
      "learning_rate": 1.2817068077078488e-05,
      "logits/chosen": -2.012524127960205,
      "logits/rejected": -2.0349159240722656,
      "logps/chosen": -273.3743591308594,
      "logps/rejected": -283.64764404296875,
      "loss": 0.5115,
      "rewards/accuracies": 0.6916667222976685,
      "rewards/chosen": -3.0330026149749756,
      "rewards/margins": 1.2960546016693115,
      "rewards/rejected": -4.329057216644287,
      "step": 4080
    },
    {
      "epoch": 0.8848694898732806,
      "grad_norm": 19.375,
      "learning_rate": 1.2812368960089735e-05,
      "logits/chosen": -2.00530743598938,
      "logits/rejected": -1.8491363525390625,
      "logps/chosen": -272.2079772949219,
      "logps/rejected": -305.77655029296875,
      "loss": 0.4444,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -3.1543984413146973,
      "rewards/margins": 1.2529605627059937,
      "rewards/rejected": -4.407358646392822,
      "step": 4085
    },
    {
      "epoch": 0.8859525614643128,
      "grad_norm": 18.375,
      "learning_rate": 1.2807664496545365e-05,
      "logits/chosen": -1.8669811487197876,
      "logits/rejected": -1.803598165512085,
      "logps/chosen": -286.9441833496094,
      "logps/rejected": -315.275390625,
      "loss": 0.4407,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -2.966085910797119,
      "rewards/margins": 1.32362961769104,
      "rewards/rejected": -4.289715766906738,
      "step": 4090
    },
    {
      "epoch": 0.8870356330553449,
      "grad_norm": 22.25,
      "learning_rate": 1.280295469315299e-05,
      "logits/chosen": -1.8898519277572632,
      "logits/rejected": -1.9232845306396484,
      "logps/chosen": -285.23504638671875,
      "logps/rejected": -308.4598083496094,
      "loss": 0.4069,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -2.9484469890594482,
      "rewards/margins": 1.4908998012542725,
      "rewards/rejected": -4.439347267150879,
      "step": 4095
    },
    {
      "epoch": 0.8881187046463771,
      "grad_norm": 17.5,
      "learning_rate": 1.2798239556627836e-05,
      "logits/chosen": -1.8234646320343018,
      "logits/rejected": -1.7539596557617188,
      "logps/chosen": -294.24444580078125,
      "logps/rejected": -322.9757080078125,
      "loss": 0.4653,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -3.233243227005005,
      "rewards/margins": 1.3089319467544556,
      "rewards/rejected": -4.542174816131592,
      "step": 4100
    },
    {
      "epoch": 0.8892017762374093,
      "grad_norm": 16.125,
      "learning_rate": 1.2793519093692738e-05,
      "logits/chosen": -1.931632399559021,
      "logits/rejected": -1.898834466934204,
      "logps/chosen": -277.3853759765625,
      "logps/rejected": -304.3611145019531,
      "loss": 0.4675,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.2234764099121094,
      "rewards/margins": 1.2400844097137451,
      "rewards/rejected": -4.463561058044434,
      "step": 4105
    },
    {
      "epoch": 0.8902848478284414,
      "grad_norm": 14.4375,
      "learning_rate": 1.2788793311078123e-05,
      "logits/chosen": -1.86288583278656,
      "logits/rejected": -1.9241819381713867,
      "logps/chosen": -269.94244384765625,
      "logps/rejected": -300.50531005859375,
      "loss": 0.4555,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.1307594776153564,
      "rewards/margins": 1.4783756732940674,
      "rewards/rejected": -4.609135627746582,
      "step": 4110
    },
    {
      "epoch": 0.8913679194194736,
      "grad_norm": 18.875,
      "learning_rate": 1.2784062215522e-05,
      "logits/chosen": -1.9990489482879639,
      "logits/rejected": -1.9828760623931885,
      "logps/chosen": -270.9298095703125,
      "logps/rejected": -301.2403564453125,
      "loss": 0.4417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.842463254928589,
      "rewards/margins": 1.3722401857376099,
      "rewards/rejected": -4.214703559875488,
      "step": 4115
    },
    {
      "epoch": 0.8924509910105058,
      "grad_norm": 16.375,
      "learning_rate": 1.2779325813769958e-05,
      "logits/chosen": -2.008328437805176,
      "logits/rejected": -1.9913524389266968,
      "logps/chosen": -255.91708374023438,
      "logps/rejected": -291.2000732421875,
      "loss": 0.4167,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.969266414642334,
      "rewards/margins": 1.5726497173309326,
      "rewards/rejected": -4.541916847229004,
      "step": 4120
    },
    {
      "epoch": 0.893534062601538,
      "grad_norm": 21.0,
      "learning_rate": 1.2774584112575147e-05,
      "logits/chosen": -2.040353298187256,
      "logits/rejected": -2.072216510772705,
      "logps/chosen": -292.21575927734375,
      "logps/rejected": -305.7843322753906,
      "loss": 0.4046,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.425417423248291,
      "rewards/margins": 1.5803929567337036,
      "rewards/rejected": -5.005810737609863,
      "step": 4125
    },
    {
      "epoch": 0.8946171341925702,
      "grad_norm": 19.5,
      "learning_rate": 1.2769837118698276e-05,
      "logits/chosen": -2.0999791622161865,
      "logits/rejected": -2.063030481338501,
      "logps/chosen": -273.4391174316406,
      "logps/rejected": -302.14288330078125,
      "loss": 0.4175,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -3.0210540294647217,
      "rewards/margins": 1.5110820531845093,
      "rewards/rejected": -4.532135963439941,
      "step": 4130
    },
    {
      "epoch": 0.8957002057836023,
      "grad_norm": 17.375,
      "learning_rate": 1.2765084838907601e-05,
      "logits/chosen": -2.0234947204589844,
      "logits/rejected": -2.0332577228546143,
      "logps/chosen": -285.88421630859375,
      "logps/rejected": -307.43585205078125,
      "loss": 0.3865,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.1881814002990723,
      "rewards/margins": 1.532335877418518,
      "rewards/rejected": -4.720517635345459,
      "step": 4135
    },
    {
      "epoch": 0.8967832773746345,
      "grad_norm": 19.0,
      "learning_rate": 1.276032727997891e-05,
      "logits/chosen": -2.008080005645752,
      "logits/rejected": -1.994519829750061,
      "logps/chosen": -292.1842956542969,
      "logps/rejected": -317.37091064453125,
      "loss": 0.4083,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -3.2472336292266846,
      "rewards/margins": 1.5230680704116821,
      "rewards/rejected": -4.770301342010498,
      "step": 4140
    },
    {
      "epoch": 0.8978663489656666,
      "grad_norm": 20.75,
      "learning_rate": 1.2755564448695523e-05,
      "logits/chosen": -2.003757953643799,
      "logits/rejected": -2.054577350616455,
      "logps/chosen": -286.55706787109375,
      "logps/rejected": -321.091552734375,
      "loss": 0.3999,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.4291229248046875,
      "rewards/margins": 1.5342775583267212,
      "rewards/rejected": -4.963400363922119,
      "step": 4145
    },
    {
      "epoch": 0.8989494205566988,
      "grad_norm": 24.75,
      "learning_rate": 1.2750796351848271e-05,
      "logits/chosen": -2.035946846008301,
      "logits/rejected": -1.9635694026947021,
      "logps/chosen": -303.5261535644531,
      "logps/rejected": -328.3379821777344,
      "loss": 0.4778,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.6917786598205566,
      "rewards/margins": 1.131446123123169,
      "rewards/rejected": -4.823224067687988,
      "step": 4150
    },
    {
      "epoch": 0.900032492147731,
      "grad_norm": 16.375,
      "learning_rate": 1.2746022996235505e-05,
      "logits/chosen": -1.9845504760742188,
      "logits/rejected": -1.9520604610443115,
      "logps/chosen": -294.1664123535156,
      "logps/rejected": -320.0483093261719,
      "loss": 0.4222,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.420017719268799,
      "rewards/margins": 1.4004091024398804,
      "rewards/rejected": -4.820426940917969,
      "step": 4155
    },
    {
      "epoch": 0.9011155637387631,
      "grad_norm": 17.0,
      "learning_rate": 1.274124438866306e-05,
      "logits/chosen": -2.0969715118408203,
      "logits/rejected": -2.0559451580047607,
      "logps/chosen": -277.4725646972656,
      "logps/rejected": -316.75970458984375,
      "loss": 0.4465,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.23138689994812,
      "rewards/margins": 1.5095622539520264,
      "rewards/rejected": -4.740949630737305,
      "step": 4160
    },
    {
      "epoch": 0.9021986353297953,
      "grad_norm": 24.875,
      "learning_rate": 1.2736460535944263e-05,
      "logits/chosen": -2.0396835803985596,
      "logits/rejected": -1.939814567565918,
      "logps/chosen": -302.29754638671875,
      "logps/rejected": -331.4088439941406,
      "loss": 0.4762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2070400714874268,
      "rewards/margins": 1.3411197662353516,
      "rewards/rejected": -4.548160076141357,
      "step": 4165
    },
    {
      "epoch": 0.9032817069208274,
      "grad_norm": 14.1875,
      "learning_rate": 1.2731671444899927e-05,
      "logits/chosen": -2.0070114135742188,
      "logits/rejected": -1.891145944595337,
      "logps/chosen": -283.0845642089844,
      "logps/rejected": -331.0159606933594,
      "loss": 0.4675,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -2.975923538208008,
      "rewards/margins": 1.3548475503921509,
      "rewards/rejected": -4.330770969390869,
      "step": 4170
    },
    {
      "epoch": 0.9043647785118596,
      "grad_norm": 25.625,
      "learning_rate": 1.2726877122358324e-05,
      "logits/chosen": -1.9726791381835938,
      "logits/rejected": -2.027496576309204,
      "logps/chosen": -277.3419494628906,
      "logps/rejected": -305.97833251953125,
      "loss": 0.418,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -2.7869873046875,
      "rewards/margins": 1.5459022521972656,
      "rewards/rejected": -4.332889556884766,
      "step": 4175
    },
    {
      "epoch": 0.9054478501028917,
      "grad_norm": 16.375,
      "learning_rate": 1.2722077575155194e-05,
      "logits/chosen": -2.052417516708374,
      "logits/rejected": -2.023043632507324,
      "logps/chosen": -283.40966796875,
      "logps/rejected": -306.1465759277344,
      "loss": 0.4205,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -3.293595552444458,
      "rewards/margins": 1.387333869934082,
      "rewards/rejected": -4.680929183959961,
      "step": 4180
    },
    {
      "epoch": 0.906530921693924,
      "grad_norm": 13.9375,
      "learning_rate": 1.2717272810133722e-05,
      "logits/chosen": -1.9860385656356812,
      "logits/rejected": -1.9368934631347656,
      "logps/chosen": -276.0509338378906,
      "logps/rejected": -301.14129638671875,
      "loss": 0.4475,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -3.1351566314697266,
      "rewards/margins": 1.4080991744995117,
      "rewards/rejected": -4.543255805969238,
      "step": 4185
    },
    {
      "epoch": 0.9076139932849562,
      "grad_norm": 22.5,
      "learning_rate": 1.2712462834144524e-05,
      "logits/chosen": -1.9979041814804077,
      "logits/rejected": -1.9438331127166748,
      "logps/chosen": -274.7174987792969,
      "logps/rejected": -306.0865783691406,
      "loss": 0.4912,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.072740077972412,
      "rewards/margins": 1.2378599643707275,
      "rewards/rejected": -4.3105998039245605,
      "step": 4190
    },
    {
      "epoch": 0.9086970648759883,
      "grad_norm": 19.625,
      "learning_rate": 1.2707647654045664e-05,
      "logits/chosen": -2.0620858669281006,
      "logits/rejected": -2.072779655456543,
      "logps/chosen": -271.7284851074219,
      "logps/rejected": -297.888427734375,
      "loss": 0.3807,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -3.292973041534424,
      "rewards/margins": 1.444884181022644,
      "rewards/rejected": -4.737856864929199,
      "step": 4195
    },
    {
      "epoch": 0.9097801364670205,
      "grad_norm": 16.375,
      "learning_rate": 1.270282727670261e-05,
      "logits/chosen": -1.9259350299835205,
      "logits/rejected": -1.8748836517333984,
      "logps/chosen": -275.5623474121094,
      "logps/rejected": -291.5859069824219,
      "loss": 0.4595,
      "rewards/accuracies": 0.7750000953674316,
      "rewards/chosen": -3.0871541500091553,
      "rewards/margins": 1.1853797435760498,
      "rewards/rejected": -4.272534370422363,
      "step": 4200
    },
    {
      "epoch": 0.9108632080580527,
      "grad_norm": 18.125,
      "learning_rate": 1.2698001708988248e-05,
      "logits/chosen": -1.8496227264404297,
      "logits/rejected": -1.8917138576507568,
      "logps/chosen": -280.29058837890625,
      "logps/rejected": -288.7463073730469,
      "loss": 0.4535,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.818840742111206,
      "rewards/margins": 1.2950290441513062,
      "rewards/rejected": -4.113869667053223,
      "step": 4205
    },
    {
      "epoch": 0.9119462796490848,
      "grad_norm": 17.625,
      "learning_rate": 1.2693170957782863e-05,
      "logits/chosen": -1.8794364929199219,
      "logits/rejected": -1.880179762840271,
      "logps/chosen": -287.0778503417969,
      "logps/rejected": -311.46270751953125,
      "loss": 0.4481,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.5950119495391846,
      "rewards/margins": 1.304438591003418,
      "rewards/rejected": -4.899450778961182,
      "step": 4210
    },
    {
      "epoch": 0.913029351240117,
      "grad_norm": 22.25,
      "learning_rate": 1.2688335029974132e-05,
      "logits/chosen": -1.7352149486541748,
      "logits/rejected": -1.8247225284576416,
      "logps/chosen": -303.44677734375,
      "logps/rejected": -304.80511474609375,
      "loss": 0.4421,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.364905595779419,
      "rewards/margins": 1.2637782096862793,
      "rewards/rejected": -4.628684043884277,
      "step": 4215
    },
    {
      "epoch": 0.9141124228311491,
      "grad_norm": 20.5,
      "learning_rate": 1.2683493932457108e-05,
      "logits/chosen": -1.8192014694213867,
      "logits/rejected": -1.856056809425354,
      "logps/chosen": -291.13067626953125,
      "logps/rejected": -316.5272521972656,
      "loss": 0.4533,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.7245724201202393,
      "rewards/margins": 1.6044288873672485,
      "rewards/rejected": -4.329001426696777,
      "step": 4220
    },
    {
      "epoch": 0.9151954944221813,
      "grad_norm": 21.75,
      "learning_rate": 1.2678647672134218e-05,
      "logits/chosen": -1.907440423965454,
      "logits/rejected": -1.9208176136016846,
      "logps/chosen": -276.5250549316406,
      "logps/rejected": -299.58612060546875,
      "loss": 0.4333,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -3.183234453201294,
      "rewards/margins": 1.2188074588775635,
      "rewards/rejected": -4.402041912078857,
      "step": 4225
    },
    {
      "epoch": 0.9162785660132134,
      "grad_norm": 17.0,
      "learning_rate": 1.2673796255915253e-05,
      "logits/chosen": -1.7539215087890625,
      "logits/rejected": -1.7876793146133423,
      "logps/chosen": -279.96209716796875,
      "logps/rejected": -305.355712890625,
      "loss": 0.4696,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -2.213440418243408,
      "rewards/margins": 1.4732253551483154,
      "rewards/rejected": -3.6866660118103027,
      "step": 4230
    },
    {
      "epoch": 0.9173616376042456,
      "grad_norm": 17.125,
      "learning_rate": 1.2668939690717352e-05,
      "logits/chosen": -1.8239339590072632,
      "logits/rejected": -1.8290221691131592,
      "logps/chosen": -275.2393493652344,
      "logps/rejected": -300.98455810546875,
      "loss": 0.4286,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -2.6334824562072754,
      "rewards/margins": 1.4992592334747314,
      "rewards/rejected": -4.132741451263428,
      "step": 4235
    },
    {
      "epoch": 0.9184447091952778,
      "grad_norm": 17.25,
      "learning_rate": 1.2664077983464995e-05,
      "logits/chosen": -1.9896084070205688,
      "logits/rejected": -1.9654076099395752,
      "logps/chosen": -276.98883056640625,
      "logps/rejected": -297.9440002441406,
      "loss": 0.4276,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.142335891723633,
      "rewards/margins": 1.330697774887085,
      "rewards/rejected": -4.473033428192139,
      "step": 4240
    },
    {
      "epoch": 0.9195277807863099,
      "grad_norm": 23.5,
      "learning_rate": 1.2659211141089994e-05,
      "logits/chosen": -1.9007136821746826,
      "logits/rejected": -1.8937476873397827,
      "logps/chosen": -286.45623779296875,
      "logps/rejected": -305.83770751953125,
      "loss": 0.434,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.312440872192383,
      "rewards/margins": 1.3460516929626465,
      "rewards/rejected": -4.658492565155029,
      "step": 4245
    },
    {
      "epoch": 0.9206108523773422,
      "grad_norm": 23.25,
      "learning_rate": 1.2654339170531483e-05,
      "logits/chosen": -1.8357454538345337,
      "logits/rejected": -1.7318551540374756,
      "logps/chosen": -286.1278076171875,
      "logps/rejected": -313.4300842285156,
      "loss": 0.4519,
      "rewards/accuracies": 0.7500001192092896,
      "rewards/chosen": -3.015471935272217,
      "rewards/margins": 1.5121138095855713,
      "rewards/rejected": -4.527585506439209,
      "step": 4250
    },
    {
      "epoch": 0.9216939239683744,
      "grad_norm": 17.25,
      "learning_rate": 1.264946207873591e-05,
      "logits/chosen": -1.9062484502792358,
      "logits/rejected": -1.874924898147583,
      "logps/chosen": -283.246337890625,
      "logps/rejected": -308.71417236328125,
      "loss": 0.4068,
      "rewards/accuracies": 0.7916667461395264,
      "rewards/chosen": -3.3838322162628174,
      "rewards/margins": 1.6652873754501343,
      "rewards/rejected": -5.04911994934082,
      "step": 4255
    },
    {
      "epoch": 0.9227769955594065,
      "grad_norm": 20.875,
      "learning_rate": 1.2644579872657019e-05,
      "logits/chosen": -1.9110510349273682,
      "logits/rejected": -1.8604990243911743,
      "logps/chosen": -274.90679931640625,
      "logps/rejected": -310.1817626953125,
      "loss": 0.4271,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -2.84309720993042,
      "rewards/margins": 1.561825156211853,
      "rewards/rejected": -4.404922962188721,
      "step": 4260
    },
    {
      "epoch": 0.9238600671504387,
      "grad_norm": 17.625,
      "learning_rate": 1.2639692559255853e-05,
      "logits/chosen": -1.7286818027496338,
      "logits/rejected": -1.6704909801483154,
      "logps/chosen": -292.8489074707031,
      "logps/rejected": -299.25836181640625,
      "loss": 0.3753,
      "rewards/accuracies": 0.8166667819023132,
      "rewards/chosen": -2.9667670726776123,
      "rewards/margins": 1.4511921405792236,
      "rewards/rejected": -4.417959213256836,
      "step": 4265
    },
    {
      "epoch": 0.9249431387414708,
      "grad_norm": 23.625,
      "learning_rate": 1.2634800145500733e-05,
      "logits/chosen": -1.7654412984848022,
      "logits/rejected": -1.7578105926513672,
      "logps/chosen": -281.64019775390625,
      "logps/rejected": -310.63336181640625,
      "loss": 0.3834,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -2.647219181060791,
      "rewards/margins": 1.5096896886825562,
      "rewards/rejected": -4.156908988952637,
      "step": 4270
    },
    {
      "epoch": 0.926026210332503,
      "grad_norm": 23.5,
      "learning_rate": 1.2629902638367251e-05,
      "logits/chosen": -1.8224233388900757,
      "logits/rejected": -1.7618077993392944,
      "logps/chosen": -267.97705078125,
      "logps/rejected": -291.2639465332031,
      "loss": 0.4647,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -2.642000913619995,
      "rewards/margins": 1.2700517177581787,
      "rewards/rejected": -3.912052631378174,
      "step": 4275
    },
    {
      "epoch": 0.9271092819235351,
      "grad_norm": 14.4375,
      "learning_rate": 1.2625000044838264e-05,
      "logits/chosen": -1.7778581380844116,
      "logits/rejected": -1.8295128345489502,
      "logps/chosen": -274.02972412109375,
      "logps/rejected": -293.424560546875,
      "loss": 0.4507,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -2.9099016189575195,
      "rewards/margins": 1.2449612617492676,
      "rewards/rejected": -4.154862880706787,
      "step": 4280
    },
    {
      "epoch": 0.9281923535145673,
      "grad_norm": 25.375,
      "learning_rate": 1.2620092371903877e-05,
      "logits/chosen": -1.8788909912109375,
      "logits/rejected": -1.8980627059936523,
      "logps/chosen": -293.701904296875,
      "logps/rejected": -319.30596923828125,
      "loss": 0.4398,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -3.42494535446167,
      "rewards/margins": 1.347374439239502,
      "rewards/rejected": -4.772319793701172,
      "step": 4285
    },
    {
      "epoch": 0.9292754251055995,
      "grad_norm": 26.5,
      "learning_rate": 1.2615179626561447e-05,
      "logits/chosen": -1.9830983877182007,
      "logits/rejected": -1.9549452066421509,
      "logps/chosen": -276.596435546875,
      "logps/rejected": -292.1119689941406,
      "loss": 0.4391,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.237757444381714,
      "rewards/margins": 1.3736765384674072,
      "rewards/rejected": -4.611433982849121,
      "step": 4290
    },
    {
      "epoch": 0.9303584966966316,
      "grad_norm": 15.4375,
      "learning_rate": 1.261026181581555e-05,
      "logits/chosen": -1.795090913772583,
      "logits/rejected": -1.7781927585601807,
      "logps/chosen": -303.05841064453125,
      "logps/rejected": -326.42852783203125,
      "loss": 0.3849,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -3.5008151531219482,
      "rewards/margins": 1.4723474979400635,
      "rewards/rejected": -4.97316312789917,
      "step": 4295
    },
    {
      "epoch": 0.9314415682876638,
      "grad_norm": 22.25,
      "learning_rate": 1.2605338946677995e-05,
      "logits/chosen": -1.7968305349349976,
      "logits/rejected": -1.8013712167739868,
      "logps/chosen": -288.4836730957031,
      "logps/rejected": -304.2983703613281,
      "loss": 0.3986,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -3.115896463394165,
      "rewards/margins": 1.5888035297393799,
      "rewards/rejected": -4.704699516296387,
      "step": 4300
    },
    {
      "epoch": 0.9325246398786959,
      "grad_norm": 18.125,
      "learning_rate": 1.2600411026167796e-05,
      "logits/chosen": -1.8411359786987305,
      "logits/rejected": -1.834167718887329,
      "logps/chosen": -286.12396240234375,
      "logps/rejected": -310.96917724609375,
      "loss": 0.3979,
      "rewards/accuracies": 0.783333420753479,
      "rewards/chosen": -3.376178026199341,
      "rewards/margins": 1.465836763381958,
      "rewards/rejected": -4.842014312744141,
      "step": 4305
    },
    {
      "epoch": 0.9336077114697281,
      "grad_norm": 19.125,
      "learning_rate": 1.2595478061311174e-05,
      "logits/chosen": -1.877187967300415,
      "logits/rejected": -1.9001489877700806,
      "logps/chosen": -289.76507568359375,
      "logps/rejected": -335.51104736328125,
      "loss": 0.4021,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.4850521087646484,
      "rewards/margins": 1.6077468395233154,
      "rewards/rejected": -5.092799186706543,
      "step": 4310
    },
    {
      "epoch": 0.9346907830607604,
      "grad_norm": 17.5,
      "learning_rate": 1.2590540059141536e-05,
      "logits/chosen": -1.965557336807251,
      "logits/rejected": -1.932112455368042,
      "logps/chosen": -290.0404968261719,
      "logps/rejected": -310.1722106933594,
      "loss": 0.384,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.517747402191162,
      "rewards/margins": 1.4657468795776367,
      "rewards/rejected": -4.983494758605957,
      "step": 4315
    },
    {
      "epoch": 0.9357738546517925,
      "grad_norm": 19.25,
      "learning_rate": 1.2585597026699483e-05,
      "logits/chosen": -1.9430230855941772,
      "logits/rejected": -1.973260521888733,
      "logps/chosen": -298.2364807128906,
      "logps/rejected": -328.20440673828125,
      "loss": 0.4414,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.713106155395508,
      "rewards/margins": 1.56121826171875,
      "rewards/rejected": -5.274324893951416,
      "step": 4320
    },
    {
      "epoch": 0.9368569262428247,
      "grad_norm": 15.0,
      "learning_rate": 1.2580648971032778e-05,
      "logits/chosen": -2.005139112472534,
      "logits/rejected": -2.0785164833068848,
      "logps/chosen": -293.57476806640625,
      "logps/rejected": -306.00750732421875,
      "loss": 0.3976,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.3842132091522217,
      "rewards/margins": 1.46743643283844,
      "rewards/rejected": -4.851650238037109,
      "step": 4325
    },
    {
      "epoch": 0.9379399978338568,
      "grad_norm": 20.75,
      "learning_rate": 1.2575695899196342e-05,
      "logits/chosen": -2.047623872756958,
      "logits/rejected": -1.9148248434066772,
      "logps/chosen": -282.4787292480469,
      "logps/rejected": -317.4384460449219,
      "loss": 0.4136,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.4659981727600098,
      "rewards/margins": 1.4328906536102295,
      "rewards/rejected": -4.89888858795166,
      "step": 4330
    },
    {
      "epoch": 0.939023069424889,
      "grad_norm": 18.125,
      "learning_rate": 1.2570737818252265e-05,
      "logits/chosen": -2.0994105339050293,
      "logits/rejected": -1.9717590808868408,
      "logps/chosen": -284.41485595703125,
      "logps/rejected": -330.59942626953125,
      "loss": 0.3776,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -3.6625866889953613,
      "rewards/margins": 1.6259784698486328,
      "rewards/rejected": -5.288564682006836,
      "step": 4335
    },
    {
      "epoch": 0.9401061410159212,
      "grad_norm": 23.5,
      "learning_rate": 1.2565774735269762e-05,
      "logits/chosen": -2.1505863666534424,
      "logits/rejected": -2.038300037384033,
      "logps/chosen": -286.8604431152344,
      "logps/rejected": -319.9667053222656,
      "loss": 0.434,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -3.979247570037842,
      "rewards/margins": 1.3571064472198486,
      "rewards/rejected": -5.3363542556762695,
      "step": 4340
    },
    {
      "epoch": 0.9411892126069533,
      "grad_norm": 23.5,
      "learning_rate": 1.2560806657325189e-05,
      "logits/chosen": -2.2512474060058594,
      "logits/rejected": -2.3147835731506348,
      "logps/chosen": -312.2272033691406,
      "logps/rejected": -344.6650695800781,
      "loss": 0.4214,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -4.431089878082275,
      "rewards/margins": 1.8158756494522095,
      "rewards/rejected": -6.2469658851623535,
      "step": 4345
    },
    {
      "epoch": 0.9422722841979855,
      "grad_norm": 21.125,
      "learning_rate": 1.2555833591502023e-05,
      "logits/chosen": -2.1348037719726562,
      "logits/rejected": -2.1238794326782227,
      "logps/chosen": -303.6044921875,
      "logps/rejected": -329.6804504394531,
      "loss": 0.4286,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.631897449493408,
      "rewards/margins": 1.6570831537246704,
      "rewards/rejected": -5.288980007171631,
      "step": 4350
    },
    {
      "epoch": 0.9433553557890176,
      "grad_norm": 28.0,
      "learning_rate": 1.2550855544890851e-05,
      "logits/chosen": -2.1997640132904053,
      "logits/rejected": -2.249734401702881,
      "logps/chosen": -299.6760559082031,
      "logps/rejected": -327.26702880859375,
      "loss": 0.4339,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.9754462242126465,
      "rewards/margins": 1.6126365661621094,
      "rewards/rejected": -5.588083744049072,
      "step": 4355
    },
    {
      "epoch": 0.9444384273800498,
      "grad_norm": 24.0,
      "learning_rate": 1.2545872524589357e-05,
      "logits/chosen": -2.2237014770507812,
      "logits/rejected": -2.2281947135925293,
      "logps/chosen": -300.56805419921875,
      "logps/rejected": -333.5813903808594,
      "loss": 0.3933,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -3.9496688842773438,
      "rewards/margins": 1.834753394126892,
      "rewards/rejected": -5.784421920776367,
      "step": 4360
    },
    {
      "epoch": 0.945521498971082,
      "grad_norm": 18.125,
      "learning_rate": 1.2540884537702326e-05,
      "logits/chosen": -2.1609694957733154,
      "logits/rejected": -2.0786843299865723,
      "logps/chosen": -303.354248046875,
      "logps/rejected": -330.8381652832031,
      "loss": 0.4498,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -4.178247451782227,
      "rewards/margins": 1.4972851276397705,
      "rewards/rejected": -5.675532341003418,
      "step": 4365
    },
    {
      "epoch": 0.9466045705621141,
      "grad_norm": 20.625,
      "learning_rate": 1.253589159134162e-05,
      "logits/chosen": -2.0912623405456543,
      "logits/rejected": -2.0561788082122803,
      "logps/chosen": -302.03936767578125,
      "logps/rejected": -327.85675048828125,
      "loss": 0.4346,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -4.033164978027344,
      "rewards/margins": 1.5433348417282104,
      "rewards/rejected": -5.576500415802002,
      "step": 4370
    },
    {
      "epoch": 0.9476876421531463,
      "grad_norm": 20.0,
      "learning_rate": 1.2530893692626171e-05,
      "logits/chosen": -1.9643694162368774,
      "logits/rejected": -2.0400376319885254,
      "logps/chosen": -314.73980712890625,
      "logps/rejected": -336.81414794921875,
      "loss": 0.3925,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -3.8945412635803223,
      "rewards/margins": 1.7303463220596313,
      "rewards/rejected": -5.624887466430664,
      "step": 4375
    },
    {
      "epoch": 0.9487707137441785,
      "grad_norm": 21.875,
      "learning_rate": 1.252589084868197e-05,
      "logits/chosen": -1.8945846557617188,
      "logits/rejected": -1.8830559253692627,
      "logps/chosen": -306.71148681640625,
      "logps/rejected": -323.7619934082031,
      "loss": 0.4588,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.3058974742889404,
      "rewards/margins": 1.5960019826889038,
      "rewards/rejected": -4.901899337768555,
      "step": 4380
    },
    {
      "epoch": 0.9498537853352107,
      "grad_norm": 15.875,
      "learning_rate": 1.2520883066642066e-05,
      "logits/chosen": -2.034250259399414,
      "logits/rejected": -1.995631456375122,
      "logps/chosen": -295.8401184082031,
      "logps/rejected": -321.7897033691406,
      "loss": 0.3796,
      "rewards/accuracies": 0.8083332777023315,
      "rewards/chosen": -3.854076385498047,
      "rewards/margins": 1.6218507289886475,
      "rewards/rejected": -5.475926876068115,
      "step": 4385
    },
    {
      "epoch": 0.9509368569262429,
      "grad_norm": 17.375,
      "learning_rate": 1.2515870353646545e-05,
      "logits/chosen": -1.7146854400634766,
      "logits/rejected": -1.7995634078979492,
      "logps/chosen": -308.05572509765625,
      "logps/rejected": -322.86676025390625,
      "loss": 0.4719,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": -3.586578369140625,
      "rewards/margins": 1.4610439538955688,
      "rewards/rejected": -5.047622203826904,
      "step": 4390
    },
    {
      "epoch": 0.952019928517275,
      "grad_norm": 13.875,
      "learning_rate": 1.2510852716842517e-05,
      "logits/chosen": -1.8424065113067627,
      "logits/rejected": -1.9278942346572876,
      "logps/chosen": -307.2522277832031,
      "logps/rejected": -317.3575744628906,
      "loss": 0.4026,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -3.865396022796631,
      "rewards/margins": 1.6160008907318115,
      "rewards/rejected": -5.481396675109863,
      "step": 4395
    },
    {
      "epoch": 0.9531030001083072,
      "grad_norm": 17.875,
      "learning_rate": 1.2505830163384124e-05,
      "logits/chosen": -1.867214560508728,
      "logits/rejected": -1.8189752101898193,
      "logps/chosen": -289.7314147949219,
      "logps/rejected": -306.80987548828125,
      "loss": 0.4904,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -3.6121017932891846,
      "rewards/margins": 1.2523047924041748,
      "rewards/rejected": -4.864406585693359,
      "step": 4400
    },
    {
      "epoch": 0.9541860716993393,
      "grad_norm": 20.625,
      "learning_rate": 1.2500802700432511e-05,
      "logits/chosen": -1.7876293659210205,
      "logits/rejected": -1.733522653579712,
      "logps/chosen": -318.0638427734375,
      "logps/rejected": -335.43255615234375,
      "loss": 0.431,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -4.136447906494141,
      "rewards/margins": 1.2122222185134888,
      "rewards/rejected": -5.34867000579834,
      "step": 4405
    },
    {
      "epoch": 0.9552691432903715,
      "grad_norm": 23.5,
      "learning_rate": 1.2495770335155824e-05,
      "logits/chosen": -1.704803705215454,
      "logits/rejected": -1.7611122131347656,
      "logps/chosen": -305.67034912109375,
      "logps/rejected": -326.1611022949219,
      "loss": 0.4251,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.5680015087127686,
      "rewards/margins": 1.4625334739685059,
      "rewards/rejected": -5.030534744262695,
      "step": 4410
    },
    {
      "epoch": 0.9563522148814037,
      "grad_norm": 20.75,
      "learning_rate": 1.24907330747292e-05,
      "logits/chosen": -1.7903238534927368,
      "logits/rejected": -1.8884944915771484,
      "logps/chosen": -297.063720703125,
      "logps/rejected": -318.3424072265625,
      "loss": 0.4442,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": -3.6880569458007812,
      "rewards/margins": 1.6012111902236938,
      "rewards/rejected": -5.289267539978027,
      "step": 4415
    },
    {
      "epoch": 0.9574352864724358,
      "grad_norm": 16.5,
      "learning_rate": 1.2485690926334758e-05,
      "logits/chosen": -1.7976763248443604,
      "logits/rejected": -1.7764915227890015,
      "logps/chosen": -288.3514404296875,
      "logps/rejected": -322.0364074707031,
      "loss": 0.4219,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.8085086345672607,
      "rewards/margins": 1.39483642578125,
      "rewards/rejected": -5.203344821929932,
      "step": 4420
    },
    {
      "epoch": 0.958518358063468,
      "grad_norm": 21.75,
      "learning_rate": 1.2480643897161576e-05,
      "logits/chosen": -1.7876018285751343,
      "logits/rejected": -1.7205699682235718,
      "logps/chosen": -291.01153564453125,
      "logps/rejected": -305.11676025390625,
      "loss": 0.4283,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.5645041465759277,
      "rewards/margins": 1.4771734476089478,
      "rewards/rejected": -5.041677474975586,
      "step": 4425
    },
    {
      "epoch": 0.9596014296545001,
      "grad_norm": 22.875,
      "learning_rate": 1.2475591994405703e-05,
      "logits/chosen": -1.7491251230239868,
      "logits/rejected": -1.7680256366729736,
      "logps/chosen": -314.5827331542969,
      "logps/rejected": -350.83770751953125,
      "loss": 0.4378,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -4.265383243560791,
      "rewards/margins": 1.4678879976272583,
      "rewards/rejected": -5.733271598815918,
      "step": 4430
    },
    {
      "epoch": 0.9606845012455323,
      "grad_norm": 19.5,
      "learning_rate": 1.2470535225270128e-05,
      "logits/chosen": -1.836722731590271,
      "logits/rejected": -1.8227802515029907,
      "logps/chosen": -297.80194091796875,
      "logps/rejected": -329.453125,
      "loss": 0.3779,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.7071003913879395,
      "rewards/margins": 1.7589807510375977,
      "rewards/rejected": -5.466080665588379,
      "step": 4435
    },
    {
      "epoch": 0.9617675728365644,
      "grad_norm": 19.625,
      "learning_rate": 1.2465473596964788e-05,
      "logits/chosen": -1.7820699214935303,
      "logits/rejected": -1.7256380319595337,
      "logps/chosen": -290.74774169921875,
      "logps/rejected": -325.802734375,
      "loss": 0.4539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.7921643257141113,
      "rewards/margins": 1.4956412315368652,
      "rewards/rejected": -5.287806034088135,
      "step": 4440
    },
    {
      "epoch": 0.9628506444275967,
      "grad_norm": 15.875,
      "learning_rate": 1.246040711670654e-05,
      "logits/chosen": -1.85440194606781,
      "logits/rejected": -1.7085994482040405,
      "logps/chosen": -285.66595458984375,
      "logps/rejected": -313.5930480957031,
      "loss": 0.4351,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.528695583343506,
      "rewards/margins": 1.4883188009262085,
      "rewards/rejected": -5.017014980316162,
      "step": 4445
    },
    {
      "epoch": 0.9639337160186289,
      "grad_norm": 24.375,
      "learning_rate": 1.2455335791719156e-05,
      "logits/chosen": -1.6343214511871338,
      "logits/rejected": -1.634418249130249,
      "logps/chosen": -288.0504455566406,
      "logps/rejected": -309.4483337402344,
      "loss": 0.4155,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.4913761615753174,
      "rewards/margins": 1.432180643081665,
      "rewards/rejected": -4.923557281494141,
      "step": 4450
    },
    {
      "epoch": 0.965016787609661,
      "grad_norm": 22.0,
      "learning_rate": 1.245025962923333e-05,
      "logits/chosen": -1.7188469171524048,
      "logits/rejected": -1.7188736200332642,
      "logps/chosen": -291.404296875,
      "logps/rejected": -313.15240478515625,
      "loss": 0.43,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -3.4026572704315186,
      "rewards/margins": 1.3810718059539795,
      "rewards/rejected": -4.783729553222656,
      "step": 4455
    },
    {
      "epoch": 0.9660998592006932,
      "grad_norm": 19.375,
      "learning_rate": 1.2445178636486637e-05,
      "logits/chosen": -1.7655484676361084,
      "logits/rejected": -1.7470662593841553,
      "logps/chosen": -281.40313720703125,
      "logps/rejected": -309.8265686035156,
      "loss": 0.4334,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.303086042404175,
      "rewards/margins": 1.3754727840423584,
      "rewards/rejected": -4.678559303283691,
      "step": 4460
    },
    {
      "epoch": 0.9671829307917253,
      "grad_norm": 13.6875,
      "learning_rate": 1.2440092820723551e-05,
      "logits/chosen": -1.7381947040557861,
      "logits/rejected": -1.692983627319336,
      "logps/chosen": -306.2000732421875,
      "logps/rejected": -330.0061950683594,
      "loss": 0.3546,
      "rewards/accuracies": 0.8166667819023132,
      "rewards/chosen": -3.6655526161193848,
      "rewards/margins": 1.7023589611053467,
      "rewards/rejected": -5.367911338806152,
      "step": 4465
    },
    {
      "epoch": 0.9682660023827575,
      "grad_norm": 22.0,
      "learning_rate": 1.2435002189195414e-05,
      "logits/chosen": -1.7145410776138306,
      "logits/rejected": -1.830822229385376,
      "logps/chosen": -310.37908935546875,
      "logps/rejected": -330.3597106933594,
      "loss": 0.4712,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.5644192695617676,
      "rewards/margins": 1.3943729400634766,
      "rewards/rejected": -4.958792686462402,
      "step": 4470
    },
    {
      "epoch": 0.9693490739737897,
      "grad_norm": 20.0,
      "learning_rate": 1.2429906749160442e-05,
      "logits/chosen": -1.8726692199707031,
      "logits/rejected": -1.876338243484497,
      "logps/chosen": -303.0646667480469,
      "logps/rejected": -328.1528625488281,
      "loss": 0.4169,
      "rewards/accuracies": 0.7833333015441895,
      "rewards/chosen": -3.761448383331299,
      "rewards/margins": 1.4167615175247192,
      "rewards/rejected": -5.17820930480957,
      "step": 4475
    },
    {
      "epoch": 0.9704321455648218,
      "grad_norm": 20.375,
      "learning_rate": 1.2424806507883701e-05,
      "logits/chosen": -2.1249096393585205,
      "logits/rejected": -2.1484930515289307,
      "logps/chosen": -290.45709228515625,
      "logps/rejected": -308.7458801269531,
      "loss": 0.4179,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.047412395477295,
      "rewards/margins": 1.3628572225570679,
      "rewards/rejected": -5.4102702140808105,
      "step": 4480
    },
    {
      "epoch": 0.971515217155854,
      "grad_norm": 21.375,
      "learning_rate": 1.2419701472637105e-05,
      "logits/chosen": -2.0796608924865723,
      "logits/rejected": -2.067497968673706,
      "logps/chosen": -283.21136474609375,
      "logps/rejected": -299.2059020996094,
      "loss": 0.4214,
      "rewards/accuracies": 0.7916666269302368,
      "rewards/chosen": -3.384122371673584,
      "rewards/margins": 1.5914827585220337,
      "rewards/rejected": -4.975605010986328,
      "step": 4485
    },
    {
      "epoch": 0.9725982887468861,
      "grad_norm": 27.125,
      "learning_rate": 1.2414591650699402e-05,
      "logits/chosen": -2.078761100769043,
      "logits/rejected": -2.07090425491333,
      "logps/chosen": -308.92047119140625,
      "logps/rejected": -327.54974365234375,
      "loss": 0.4188,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -3.9623184204101562,
      "rewards/margins": 1.5997635126113892,
      "rewards/rejected": -5.562081813812256,
      "step": 4490
    },
    {
      "epoch": 0.9736813603379183,
      "grad_norm": 23.25,
      "learning_rate": 1.2409477049356168e-05,
      "logits/chosen": -2.0848641395568848,
      "logits/rejected": -1.9606831073760986,
      "logps/chosen": -301.14288330078125,
      "logps/rejected": -329.7367858886719,
      "loss": 0.4322,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.178199291229248,
      "rewards/margins": 1.512603998184204,
      "rewards/rejected": -5.690803527832031,
      "step": 4495
    },
    {
      "epoch": 0.9747644319289505,
      "grad_norm": 24.625,
      "learning_rate": 1.2404357675899791e-05,
      "logits/chosen": -2.0006446838378906,
      "logits/rejected": -1.89458429813385,
      "logps/chosen": -302.00811767578125,
      "logps/rejected": -338.0984802246094,
      "loss": 0.403,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.9492759704589844,
      "rewards/margins": 1.506424069404602,
      "rewards/rejected": -5.455700874328613,
      "step": 4500
    },
    {
      "epoch": 0.9747644319289505,
      "eval_logits/chosen": -2.134739637374878,
      "eval_logits/rejected": -2.071303367614746,
      "eval_logps/chosen": -283.5088195800781,
      "eval_logps/rejected": -311.9000244140625,
      "eval_loss": 0.4741727411746979,
      "eval_rewards/accuracies": 0.7450000047683716,
      "eval_rewards/chosen": -3.4653778076171875,
      "eval_rewards/margins": 1.2467679977416992,
      "eval_rewards/rejected": -4.712145805358887,
      "eval_runtime": 14.4303,
      "eval_samples_per_second": 13.86,
      "eval_steps_per_second": 13.86,
      "step": 4500
    },
    {
      "epoch": 0.9758475035199826,
      "grad_norm": 23.375,
      "learning_rate": 1.239923353762946e-05,
      "logits/chosen": -1.9626178741455078,
      "logits/rejected": -1.9427320957183838,
      "logps/chosen": -291.6824035644531,
      "logps/rejected": -307.244384765625,
      "loss": 0.4905,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": -3.968203067779541,
      "rewards/margins": 1.3258140087127686,
      "rewards/rejected": -5.2940168380737305,
      "step": 4505
    },
    {
      "epoch": 0.9769305751110149,
      "grad_norm": 18.5,
      "learning_rate": 1.2394104641851165e-05,
      "logits/chosen": -2.058183193206787,
      "logits/rejected": -2.003455400466919,
      "logps/chosen": -303.1637268066406,
      "logps/rejected": -332.5771484375,
      "loss": 0.4202,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -3.6823554039001465,
      "rewards/margins": 1.6321752071380615,
      "rewards/rejected": -5.314530372619629,
      "step": 4510
    },
    {
      "epoch": 0.978013646702047,
      "grad_norm": 19.0,
      "learning_rate": 1.2388970995877674e-05,
      "logits/chosen": -1.9874699115753174,
      "logits/rejected": -1.8863065242767334,
      "logps/chosen": -288.4817810058594,
      "logps/rejected": -318.53997802734375,
      "loss": 0.4197,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -3.4658050537109375,
      "rewards/margins": 1.4161043167114258,
      "rewards/rejected": -4.881909370422363,
      "step": 4515
    },
    {
      "epoch": 0.9790967182930792,
      "grad_norm": 12.75,
      "learning_rate": 1.2383832607028527e-05,
      "logits/chosen": -2.103811025619507,
      "logits/rejected": -2.085202693939209,
      "logps/chosen": -294.6976623535156,
      "logps/rejected": -321.5780944824219,
      "loss": 0.4076,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -3.6962108612060547,
      "rewards/margins": 1.6404234170913696,
      "rewards/rejected": -5.336634159088135,
      "step": 4520
    },
    {
      "epoch": 0.9801797898841114,
      "grad_norm": 17.75,
      "learning_rate": 1.2378689482630033e-05,
      "logits/chosen": -1.968409538269043,
      "logits/rejected": -2.006053924560547,
      "logps/chosen": -297.82318115234375,
      "logps/rejected": -317.1937255859375,
      "loss": 0.4164,
      "rewards/accuracies": 0.7666667699813843,
      "rewards/chosen": -3.690446376800537,
      "rewards/margins": 1.4271847009658813,
      "rewards/rejected": -5.117630958557129,
      "step": 4525
    },
    {
      "epoch": 0.9812628614751435,
      "grad_norm": 19.125,
      "learning_rate": 1.2373541630015246e-05,
      "logits/chosen": -1.9405126571655273,
      "logits/rejected": -1.9451783895492554,
      "logps/chosen": -289.56488037109375,
      "logps/rejected": -323.157470703125,
      "loss": 0.4007,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -3.068802833557129,
      "rewards/margins": 1.397515892982483,
      "rewards/rejected": -4.466318607330322,
      "step": 4530
    },
    {
      "epoch": 0.9823459330661757,
      "grad_norm": 15.625,
      "learning_rate": 1.2368389056523964e-05,
      "logits/chosen": -2.0836970806121826,
      "logits/rejected": -2.099428653717041,
      "logps/chosen": -284.32659912109375,
      "logps/rejected": -293.25823974609375,
      "loss": 0.4568,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -3.740750789642334,
      "rewards/margins": 1.2272307872772217,
      "rewards/rejected": -4.967981815338135,
      "step": 4535
    },
    {
      "epoch": 0.9834290046572078,
      "grad_norm": 16.375,
      "learning_rate": 1.2363231769502714e-05,
      "logits/chosen": -1.9299418926239014,
      "logits/rejected": -1.9368374347686768,
      "logps/chosen": -287.8735046386719,
      "logps/rejected": -313.3299560546875,
      "loss": 0.3862,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -3.532322406768799,
      "rewards/margins": 1.6261135339736938,
      "rewards/rejected": -5.158435821533203,
      "step": 4540
    },
    {
      "epoch": 0.98451207624824,
      "grad_norm": 21.375,
      "learning_rate": 1.2358069776304747e-05,
      "logits/chosen": -1.9066404104232788,
      "logits/rejected": -1.921120047569275,
      "logps/chosen": -292.7264709472656,
      "logps/rejected": -313.10516357421875,
      "loss": 0.4185,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -3.2595584392547607,
      "rewards/margins": 1.5674712657928467,
      "rewards/rejected": -4.827030181884766,
      "step": 4545
    },
    {
      "epoch": 0.9855951478392722,
      "grad_norm": 20.875,
      "learning_rate": 1.2352903084290024e-05,
      "logits/chosen": -1.860327124595642,
      "logits/rejected": -1.900777816772461,
      "logps/chosen": -296.99359130859375,
      "logps/rejected": -319.8380126953125,
      "loss": 0.4353,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.277463912963867,
      "rewards/margins": 1.5836925506591797,
      "rewards/rejected": -4.861156463623047,
      "step": 4550
    },
    {
      "epoch": 0.9866782194303043,
      "grad_norm": 25.25,
      "learning_rate": 1.2347731700825201e-05,
      "logits/chosen": -1.9222733974456787,
      "logits/rejected": -1.9224770069122314,
      "logps/chosen": -294.31890869140625,
      "logps/rejected": -307.3768005371094,
      "loss": 0.4026,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -3.575410842895508,
      "rewards/margins": 1.4278597831726074,
      "rewards/rejected": -5.003270626068115,
      "step": 4555
    },
    {
      "epoch": 0.9877612910213365,
      "grad_norm": 19.0,
      "learning_rate": 1.234255563328363e-05,
      "logits/chosen": -2.095578908920288,
      "logits/rejected": -2.0482945442199707,
      "logps/chosen": -296.23272705078125,
      "logps/rejected": -315.879150390625,
      "loss": 0.3718,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -3.592697858810425,
      "rewards/margins": 1.7312771081924438,
      "rewards/rejected": -5.323975563049316,
      "step": 4560
    },
    {
      "epoch": 0.9888443626123686,
      "grad_norm": 23.125,
      "learning_rate": 1.2337374889045335e-05,
      "logits/chosen": -2.000298023223877,
      "logits/rejected": -1.9342836141586304,
      "logps/chosen": -302.443603515625,
      "logps/rejected": -322.69659423828125,
      "loss": 0.407,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -3.492798328399658,
      "rewards/margins": 1.4687284231185913,
      "rewards/rejected": -4.961527347564697,
      "step": 4565
    },
    {
      "epoch": 0.9899274342034008,
      "grad_norm": 24.375,
      "learning_rate": 1.233218947549701e-05,
      "logits/chosen": -2.112919330596924,
      "logits/rejected": -2.0939137935638428,
      "logps/chosen": -274.5894470214844,
      "logps/rejected": -301.48822021484375,
      "loss": 0.4127,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -3.2632975578308105,
      "rewards/margins": 1.506353497505188,
      "rewards/rejected": -4.769650936126709,
      "step": 4570
    },
    {
      "epoch": 0.9910105057944331,
      "grad_norm": 18.875,
      "learning_rate": 1.2326999400032009e-05,
      "logits/chosen": -2.0002357959747314,
      "logits/rejected": -2.0410971641540527,
      "logps/chosen": -299.67230224609375,
      "logps/rejected": -312.1649475097656,
      "loss": 0.4353,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -3.5534865856170654,
      "rewards/margins": 1.4083473682403564,
      "rewards/rejected": -4.961833953857422,
      "step": 4575
    },
    {
      "epoch": 0.9920935773854652,
      "grad_norm": 19.875,
      "learning_rate": 1.2321804670050333e-05,
      "logits/chosen": -2.07084321975708,
      "logits/rejected": -2.0763463973999023,
      "logps/chosen": -292.61407470703125,
      "logps/rejected": -322.1815490722656,
      "loss": 0.3204,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.889727830886841,
      "rewards/margins": 1.7843759059906006,
      "rewards/rejected": -5.674103736877441,
      "step": 4580
    },
    {
      "epoch": 0.9931766489764974,
      "grad_norm": 23.375,
      "learning_rate": 1.2316605292958612e-05,
      "logits/chosen": -2.1553454399108887,
      "logits/rejected": -2.0104129314422607,
      "logps/chosen": -294.48358154296875,
      "logps/rejected": -327.43658447265625,
      "loss": 0.441,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -4.132212162017822,
      "rewards/margins": 1.4965295791625977,
      "rewards/rejected": -5.62874174118042,
      "step": 4585
    },
    {
      "epoch": 0.9942597205675295,
      "grad_norm": 18.625,
      "learning_rate": 1.2311401276170113e-05,
      "logits/chosen": -1.982320785522461,
      "logits/rejected": -2.0591468811035156,
      "logps/chosen": -310.7012634277344,
      "logps/rejected": -322.70758056640625,
      "loss": 0.4456,
      "rewards/accuracies": 0.7583333849906921,
      "rewards/chosen": -3.821185350418091,
      "rewards/margins": 1.3451917171478271,
      "rewards/rejected": -5.166377067565918,
      "step": 4590
    },
    {
      "epoch": 0.9953427921585617,
      "grad_norm": 25.75,
      "learning_rate": 1.2306192627104709e-05,
      "logits/chosen": -2.0248942375183105,
      "logits/rejected": -2.03106951713562,
      "logps/chosen": -294.9580078125,
      "logps/rejected": -319.4584655761719,
      "loss": 0.4044,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.892077684402466,
      "rewards/margins": 1.6711841821670532,
      "rewards/rejected": -5.56326150894165,
      "step": 4595
    },
    {
      "epoch": 0.9964258637495939,
      "grad_norm": 21.375,
      "learning_rate": 1.2300979353188882e-05,
      "logits/chosen": -2.047666549682617,
      "logits/rejected": -2.0200984477996826,
      "logps/chosen": -293.6147766113281,
      "logps/rejected": -328.77520751953125,
      "loss": 0.4655,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -4.046487331390381,
      "rewards/margins": 1.4053709506988525,
      "rewards/rejected": -5.4518585205078125,
      "step": 4600
    },
    {
      "epoch": 0.997508935340626,
      "grad_norm": 20.125,
      "learning_rate": 1.229576146185571e-05,
      "logits/chosen": -1.9519813060760498,
      "logits/rejected": -1.881188988685608,
      "logps/chosen": -306.1051025390625,
      "logps/rejected": -335.84454345703125,
      "loss": 0.4188,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": -4.0544962882995605,
      "rewards/margins": 1.525693655014038,
      "rewards/rejected": -5.580190658569336,
      "step": 4605
    },
    {
      "epoch": 0.9985920069316582,
      "grad_norm": 15.4375,
      "learning_rate": 1.229053896054485e-05,
      "logits/chosen": -2.115124464035034,
      "logits/rejected": -2.1207714080810547,
      "logps/chosen": -285.43548583984375,
      "logps/rejected": -314.9853210449219,
      "loss": 0.3739,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -3.8249588012695312,
      "rewards/margins": 1.5160130262374878,
      "rewards/rejected": -5.340971946716309,
      "step": 4610
    },
    {
      "epoch": 0.9996750785226903,
      "grad_norm": 24.875,
      "learning_rate": 1.228531185670253e-05,
      "logits/chosen": -1.7500499486923218,
      "logits/rejected": -1.922803282737732,
      "logps/chosen": -341.9169006347656,
      "logps/rejected": -355.04620361328125,
      "loss": 0.4462,
      "rewards/accuracies": 0.7750000953674316,
      "rewards/chosen": -4.071297645568848,
      "rewards/margins": 1.643218755722046,
      "rewards/rejected": -5.714515686035156,
      "step": 4615
    },
    {
      "epoch": 0.9998916928408967,
      "eval_logits/chosen": -2.1375327110290527,
      "eval_logits/rejected": -2.068173885345459,
      "eval_logps/chosen": -288.2662048339844,
      "eval_logps/rejected": -319.2733459472656,
      "eval_loss": 0.4471246302127838,
      "eval_rewards/accuracies": 0.7699999809265137,
      "eval_rewards/chosen": -3.703246831893921,
      "eval_rewards/margins": 1.3775652647018433,
      "eval_rewards/rejected": -5.080811977386475,
      "eval_runtime": 14.7116,
      "eval_samples_per_second": 13.595,
      "eval_steps_per_second": 13.595,
      "step": 4616
    },
    {
      "epoch": 1.0006498429546193,
      "grad_norm": 17.625,
      "learning_rate": 1.2280080157781553e-05,
      "logits/chosen": -2.049691677093506,
      "logits/rejected": -2.047917366027832,
      "logps/chosen": -282.8787536621094,
      "logps/rejected": -305.3478088378906,
      "loss": 0.3111,
      "rewards/accuracies": 0.8611111044883728,
      "rewards/chosen": -3.7249059677124023,
      "rewards/margins": 1.7252955436706543,
      "rewards/rejected": -5.450201034545898,
      "step": 4620
    },
    {
      "epoch": 1.0017329145456515,
      "grad_norm": 17.625,
      "learning_rate": 1.2274843871241258e-05,
      "logits/chosen": -1.9997451305389404,
      "logits/rejected": -2.087052345275879,
      "logps/chosen": -306.888427734375,
      "logps/rejected": -328.37713623046875,
      "loss": 0.2667,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.051124095916748,
      "rewards/margins": 2.0648975372314453,
      "rewards/rejected": -6.116021156311035,
      "step": 4625
    },
    {
      "epoch": 1.0028159861366837,
      "grad_norm": 14.75,
      "learning_rate": 1.2269603004547534e-05,
      "logits/chosen": -2.1469826698303223,
      "logits/rejected": -2.2380526065826416,
      "logps/chosen": -299.1263732910156,
      "logps/rejected": -328.54058837890625,
      "loss": 0.2529,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.025455951690674,
      "rewards/margins": 2.0675742626190186,
      "rewards/rejected": -6.0930304527282715,
      "step": 4630
    },
    {
      "epoch": 1.0038990577277158,
      "grad_norm": 19.0,
      "learning_rate": 1.2264357565172796e-05,
      "logits/chosen": -2.064950466156006,
      "logits/rejected": -2.071648120880127,
      "logps/chosen": -299.9688415527344,
      "logps/rejected": -336.66644287109375,
      "loss": 0.2867,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -3.9760384559631348,
      "rewards/margins": 1.9765857458114624,
      "rewards/rejected": -5.952624320983887,
      "step": 4635
    },
    {
      "epoch": 1.004982129318748,
      "grad_norm": 28.875,
      "learning_rate": 1.2259107560595985e-05,
      "logits/chosen": -2.327000856399536,
      "logits/rejected": -2.210339069366455,
      "logps/chosen": -305.55712890625,
      "logps/rejected": -343.2110900878906,
      "loss": 0.3051,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -4.404684543609619,
      "rewards/margins": 1.9195072650909424,
      "rewards/rejected": -6.324192047119141,
      "step": 4640
    },
    {
      "epoch": 1.0060652009097801,
      "grad_norm": 19.625,
      "learning_rate": 1.2253852998302544e-05,
      "logits/chosen": -2.431898832321167,
      "logits/rejected": -2.4476609230041504,
      "logps/chosen": -312.0470886230469,
      "logps/rejected": -345.6715393066406,
      "loss": 0.3079,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.076813220977783,
      "rewards/margins": 2.0144553184509277,
      "rewards/rejected": -7.0912675857543945,
      "step": 4645
    },
    {
      "epoch": 1.0071482725008123,
      "grad_norm": 9.8125,
      "learning_rate": 1.2248593885784417e-05,
      "logits/chosen": -2.43218994140625,
      "logits/rejected": -2.5221848487854004,
      "logps/chosen": -326.4375,
      "logps/rejected": -349.18231201171875,
      "loss": 0.2174,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -4.534524440765381,
      "rewards/margins": 2.5779380798339844,
      "rewards/rejected": -7.112463474273682,
      "step": 4650
    },
    {
      "epoch": 1.0082313440918445,
      "grad_norm": 17.625,
      "learning_rate": 1.224333023054004e-05,
      "logits/chosen": -2.363892078399658,
      "logits/rejected": -2.386971950531006,
      "logps/chosen": -331.1203918457031,
      "logps/rejected": -373.43023681640625,
      "loss": 0.2517,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.864957332611084,
      "rewards/margins": 2.515854597091675,
      "rewards/rejected": -7.380812168121338,
      "step": 4655
    },
    {
      "epoch": 1.0093144156828766,
      "grad_norm": 20.375,
      "learning_rate": 1.2238062040074318e-05,
      "logits/chosen": -2.365919828414917,
      "logits/rejected": -2.3838014602661133,
      "logps/chosen": -314.1370544433594,
      "logps/rejected": -345.7989196777344,
      "loss": 0.2441,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -4.4119133949279785,
      "rewards/margins": 2.2802774906158447,
      "rewards/rejected": -6.692191123962402,
      "step": 4660
    },
    {
      "epoch": 1.0103974872739088,
      "grad_norm": 13.9375,
      "learning_rate": 1.2232789321898626e-05,
      "logits/chosen": -2.564701557159424,
      "logits/rejected": -2.52888560295105,
      "logps/chosen": -311.90716552734375,
      "logps/rejected": -352.4566345214844,
      "loss": 0.2373,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -4.566357612609863,
      "rewards/margins": 2.559063673019409,
      "rewards/rejected": -7.125421047210693,
      "step": 4665
    },
    {
      "epoch": 1.011480558864941,
      "grad_norm": 18.5,
      "learning_rate": 1.2227512083530798e-05,
      "logits/chosen": -2.548677682876587,
      "logits/rejected": -2.5734047889709473,
      "logps/chosen": -298.52313232421875,
      "logps/rejected": -334.0669860839844,
      "loss": 0.2905,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.377018928527832,
      "rewards/margins": 2.3062233924865723,
      "rewards/rejected": -6.6832427978515625,
      "step": 4670
    },
    {
      "epoch": 1.012563630455973,
      "grad_norm": 19.5,
      "learning_rate": 1.2222230332495106e-05,
      "logits/chosen": -2.598353385925293,
      "logits/rejected": -2.6722285747528076,
      "logps/chosen": -323.1211853027344,
      "logps/rejected": -366.73199462890625,
      "loss": 0.2665,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.212936878204346,
      "rewards/margins": 2.4278078079223633,
      "rewards/rejected": -7.640744686126709,
      "step": 4675
    },
    {
      "epoch": 1.0136467020470052,
      "grad_norm": 20.75,
      "learning_rate": 1.2216944076322265e-05,
      "logits/chosen": -2.5954606533050537,
      "logits/rejected": -2.609168529510498,
      "logps/chosen": -334.48736572265625,
      "logps/rejected": -387.25140380859375,
      "loss": 0.2707,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.322426795959473,
      "rewards/margins": 2.6118624210357666,
      "rewards/rejected": -7.93428897857666,
      "step": 4680
    },
    {
      "epoch": 1.0147297736380374,
      "grad_norm": 21.125,
      "learning_rate": 1.2211653322549406e-05,
      "logits/chosen": -2.77207612991333,
      "logits/rejected": -2.7835090160369873,
      "logps/chosen": -349.5242004394531,
      "logps/rejected": -385.205810546875,
      "loss": 0.29,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.519827842712402,
      "rewards/margins": 2.5634469985961914,
      "rewards/rejected": -8.083274841308594,
      "step": 4685
    },
    {
      "epoch": 1.0158128452290696,
      "grad_norm": 36.5,
      "learning_rate": 1.2206358078720073e-05,
      "logits/chosen": -2.928328037261963,
      "logits/rejected": -2.899935007095337,
      "logps/chosen": -326.0615539550781,
      "logps/rejected": -376.71826171875,
      "loss": 0.3112,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.607903957366943,
      "rewards/margins": 2.588974952697754,
      "rewards/rejected": -8.196878433227539,
      "step": 4690
    },
    {
      "epoch": 1.0168959168201017,
      "grad_norm": 17.5,
      "learning_rate": 1.2201058352384218e-05,
      "logits/chosen": -2.932490825653076,
      "logits/rejected": -2.824338912963867,
      "logps/chosen": -326.9004211425781,
      "logps/rejected": -380.7578125,
      "loss": 0.2853,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.692027568817139,
      "rewards/margins": 2.4894039630889893,
      "rewards/rejected": -8.181431770324707,
      "step": 4695
    },
    {
      "epoch": 1.0179789884111339,
      "grad_norm": 13.875,
      "learning_rate": 1.2195754151098177e-05,
      "logits/chosen": -2.852247953414917,
      "logits/rejected": -2.92454195022583,
      "logps/chosen": -320.3943786621094,
      "logps/rejected": -361.3133544921875,
      "loss": 0.2376,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.4626030921936035,
      "rewards/margins": 2.4660515785217285,
      "rewards/rejected": -7.92865514755249,
      "step": 4700
    },
    {
      "epoch": 1.019062060002166,
      "grad_norm": 19.5,
      "learning_rate": 1.2190445482424673e-05,
      "logits/chosen": -2.7675445079803467,
      "logits/rejected": -2.732334613800049,
      "logps/chosen": -343.4591064453125,
      "logps/rejected": -390.317626953125,
      "loss": 0.265,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.455132484436035,
      "rewards/margins": 2.593916893005371,
      "rewards/rejected": -8.049049377441406,
      "step": 4705
    },
    {
      "epoch": 1.0201451315931984,
      "grad_norm": 22.125,
      "learning_rate": 1.2185132353932789e-05,
      "logits/chosen": -2.8185269832611084,
      "logits/rejected": -2.8642497062683105,
      "logps/chosen": -337.4569396972656,
      "logps/rejected": -388.3210754394531,
      "loss": 0.2287,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.960319519042969,
      "rewards/margins": 2.4630255699157715,
      "rewards/rejected": -8.423345565795898,
      "step": 4710
    },
    {
      "epoch": 1.0212282031842306,
      "grad_norm": 25.0,
      "learning_rate": 1.2179814773197976e-05,
      "logits/chosen": -2.7233452796936035,
      "logits/rejected": -2.6659932136535645,
      "logps/chosen": -344.57293701171875,
      "logps/rejected": -384.4847717285156,
      "loss": 0.2989,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.389194488525391,
      "rewards/margins": 2.15324330329895,
      "rewards/rejected": -7.5424394607543945,
      "step": 4715
    },
    {
      "epoch": 1.0223112747752627,
      "grad_norm": 19.75,
      "learning_rate": 1.2174492747802033e-05,
      "logits/chosen": -2.908599376678467,
      "logits/rejected": -2.941403865814209,
      "logps/chosen": -332.5396423339844,
      "logps/rejected": -372.29449462890625,
      "loss": 0.2496,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.547204494476318,
      "rewards/margins": 2.5463266372680664,
      "rewards/rejected": -8.093530654907227,
      "step": 4720
    },
    {
      "epoch": 1.023394346366295,
      "grad_norm": 24.25,
      "learning_rate": 1.216916628533309e-05,
      "logits/chosen": -2.8911235332489014,
      "logits/rejected": -2.794656753540039,
      "logps/chosen": -327.9949645996094,
      "logps/rejected": -365.265869140625,
      "loss": 0.3225,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.721887111663818,
      "rewards/margins": 2.3196489810943604,
      "rewards/rejected": -8.041536331176758,
      "step": 4725
    },
    {
      "epoch": 1.024477417957327,
      "grad_norm": 16.125,
      "learning_rate": 1.2163835393385605e-05,
      "logits/chosen": -2.800959348678589,
      "logits/rejected": -2.716242551803589,
      "logps/chosen": -326.7389221191406,
      "logps/rejected": -375.86236572265625,
      "loss": 0.2604,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.187568187713623,
      "rewards/margins": 2.551945209503174,
      "rewards/rejected": -7.739514350891113,
      "step": 4730
    },
    {
      "epoch": 1.0255604895483592,
      "grad_norm": 22.125,
      "learning_rate": 1.2158500079560359e-05,
      "logits/chosen": -2.849689245223999,
      "logits/rejected": -2.744291305541992,
      "logps/chosen": -325.0724792480469,
      "logps/rejected": -392.33770751953125,
      "loss": 0.2727,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.190507411956787,
      "rewards/margins": 2.900144577026367,
      "rewards/rejected": -8.090652465820312,
      "step": 4735
    },
    {
      "epoch": 1.0266435611393914,
      "grad_norm": 17.625,
      "learning_rate": 1.2153160351464427e-05,
      "logits/chosen": -2.7810165882110596,
      "logits/rejected": -2.6795358657836914,
      "logps/chosen": -331.0771179199219,
      "logps/rejected": -381.8999328613281,
      "loss": 0.2292,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.216971397399902,
      "rewards/margins": 2.2899739742279053,
      "rewards/rejected": -7.506946563720703,
      "step": 4740
    },
    {
      "epoch": 1.0277266327304235,
      "grad_norm": 13.0625,
      "learning_rate": 1.214781621671118e-05,
      "logits/chosen": -3.039000988006592,
      "logits/rejected": -2.982637405395508,
      "logps/chosen": -290.6871337890625,
      "logps/rejected": -337.4967956542969,
      "loss": 0.274,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -4.799887657165527,
      "rewards/margins": 2.3727779388427734,
      "rewards/rejected": -7.172666072845459,
      "step": 4745
    },
    {
      "epoch": 1.0288097043214557,
      "grad_norm": 21.25,
      "learning_rate": 1.2142467682920282e-05,
      "logits/chosen": -2.7395713329315186,
      "logits/rejected": -2.8338658809661865,
      "logps/chosen": -355.52349853515625,
      "logps/rejected": -376.9250793457031,
      "loss": 0.3053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.336416244506836,
      "rewards/margins": 2.257211685180664,
      "rewards/rejected": -7.5936279296875,
      "step": 4750
    },
    {
      "epoch": 1.0298927759124878,
      "grad_norm": 21.5,
      "learning_rate": 1.2137114757717657e-05,
      "logits/chosen": -2.8874263763427734,
      "logits/rejected": -2.8041512966156006,
      "logps/chosen": -328.0692138671875,
      "logps/rejected": -382.6759338378906,
      "loss": 0.3151,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.717616081237793,
      "rewards/margins": 2.2968482971191406,
      "rewards/rejected": -8.01446533203125,
      "step": 4755
    },
    {
      "epoch": 1.03097584750352,
      "grad_norm": 19.125,
      "learning_rate": 1.2131757448735493e-05,
      "logits/chosen": -2.792522430419922,
      "logits/rejected": -2.8015143871307373,
      "logps/chosen": -331.2496032714844,
      "logps/rejected": -369.63897705078125,
      "loss": 0.2768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.244373321533203,
      "rewards/margins": 2.3144850730895996,
      "rewards/rejected": -7.558858394622803,
      "step": 4760
    },
    {
      "epoch": 1.0320589190945522,
      "grad_norm": 20.75,
      "learning_rate": 1.2126395763612236e-05,
      "logits/chosen": -2.9495491981506348,
      "logits/rejected": -2.9155220985412598,
      "logps/chosen": -314.27423095703125,
      "logps/rejected": -362.6643981933594,
      "loss": 0.2566,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.917237281799316,
      "rewards/margins": 2.3560492992401123,
      "rewards/rejected": -7.273285865783691,
      "step": 4765
    },
    {
      "epoch": 1.0331419906855843,
      "grad_norm": 23.375,
      "learning_rate": 1.2121029709992563e-05,
      "logits/chosen": -2.9709227085113525,
      "logits/rejected": -2.967010021209717,
      "logps/chosen": -317.26580810546875,
      "logps/rejected": -366.546630859375,
      "loss": 0.2752,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.994808197021484,
      "rewards/margins": 2.7048208713531494,
      "rewards/rejected": -7.699629306793213,
      "step": 4770
    },
    {
      "epoch": 1.0342250622766165,
      "grad_norm": 21.375,
      "learning_rate": 1.2115659295527379e-05,
      "logits/chosen": -2.8915140628814697,
      "logits/rejected": -2.761735439300537,
      "logps/chosen": -300.64007568359375,
      "logps/rejected": -365.3411560058594,
      "loss": 0.2529,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.962449073791504,
      "rewards/margins": 2.4753270149230957,
      "rewards/rejected": -7.4377760887146,
      "step": 4775
    },
    {
      "epoch": 1.0353081338676486,
      "grad_norm": 31.625,
      "learning_rate": 1.2110284527873815e-05,
      "logits/chosen": -2.895094633102417,
      "logits/rejected": -2.8008549213409424,
      "logps/chosen": -338.53521728515625,
      "logps/rejected": -385.4776916503906,
      "loss": 0.3569,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.960578918457031,
      "rewards/margins": 2.3560588359832764,
      "rewards/rejected": -8.316637992858887,
      "step": 4780
    },
    {
      "epoch": 1.0363912054586808,
      "grad_norm": 30.625,
      "learning_rate": 1.2104905414695203e-05,
      "logits/chosen": -2.9126710891723633,
      "logits/rejected": -2.999474048614502,
      "logps/chosen": -317.3303527832031,
      "logps/rejected": -345.7444152832031,
      "loss": 0.2947,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.893077373504639,
      "rewards/margins": 2.103637218475342,
      "rewards/rejected": -6.9967145919799805,
      "step": 4785
    },
    {
      "epoch": 1.037474277049713,
      "grad_norm": 29.25,
      "learning_rate": 1.2099521963661074e-05,
      "logits/chosen": -2.763031482696533,
      "logits/rejected": -2.830173969268799,
      "logps/chosen": -320.2712707519531,
      "logps/rejected": -346.184326171875,
      "loss": 0.3073,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.609015941619873,
      "rewards/margins": 2.2757370471954346,
      "rewards/rejected": -6.8847527503967285,
      "step": 4790
    },
    {
      "epoch": 1.0385573486407451,
      "grad_norm": 20.375,
      "learning_rate": 1.2094134182447134e-05,
      "logits/chosen": -2.6815178394317627,
      "logits/rejected": -2.7604079246520996,
      "logps/chosen": -313.26849365234375,
      "logps/rejected": -351.8580017089844,
      "loss": 0.3343,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.829355239868164,
      "rewards/margins": 2.253704071044922,
      "rewards/rejected": -7.083058834075928,
      "step": 4795
    },
    {
      "epoch": 1.0396404202317773,
      "grad_norm": 17.25,
      "learning_rate": 1.2088742078735277e-05,
      "logits/chosen": -2.6073873043060303,
      "logits/rejected": -2.539896249771118,
      "logps/chosen": -305.82012939453125,
      "logps/rejected": -358.07598876953125,
      "loss": 0.2576,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.4476799964904785,
      "rewards/margins": 2.27394437789917,
      "rewards/rejected": -6.72162389755249,
      "step": 4800
    },
    {
      "epoch": 1.0407234918228094,
      "grad_norm": 22.0,
      "learning_rate": 1.2083345660213552e-05,
      "logits/chosen": -2.652764320373535,
      "logits/rejected": -2.624030590057373,
      "logps/chosen": -311.93560791015625,
      "logps/rejected": -346.4599609375,
      "loss": 0.258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.489089012145996,
      "rewards/margins": 2.316237688064575,
      "rewards/rejected": -6.80532693862915,
      "step": 4805
    },
    {
      "epoch": 1.0418065634138416,
      "grad_norm": 16.5,
      "learning_rate": 1.2077944934576163e-05,
      "logits/chosen": -2.5048513412475586,
      "logits/rejected": -2.545429229736328,
      "logps/chosen": -314.44549560546875,
      "logps/rejected": -365.6279296875,
      "loss": 0.2541,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.152022361755371,
      "rewards/margins": 2.5465292930603027,
      "rewards/rejected": -6.698551177978516,
      "step": 4810
    },
    {
      "epoch": 1.0428896350048737,
      "grad_norm": 16.75,
      "learning_rate": 1.207253990952345e-05,
      "logits/chosen": -2.7654459476470947,
      "logits/rejected": -2.7288639545440674,
      "logps/chosen": -301.43359375,
      "logps/rejected": -350.3265075683594,
      "loss": 0.2554,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.195709228515625,
      "rewards/margins": 2.377047061920166,
      "rewards/rejected": -6.572756290435791,
      "step": 4815
    },
    {
      "epoch": 1.043972706595906,
      "grad_norm": 14.375,
      "learning_rate": 1.2067130592761893e-05,
      "logits/chosen": -2.572094440460205,
      "logits/rejected": -2.5534470081329346,
      "logps/chosen": -311.11602783203125,
      "logps/rejected": -354.9997863769531,
      "loss": 0.2759,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.208915710449219,
      "rewards/margins": 2.23966383934021,
      "rewards/rejected": -6.44857931137085,
      "step": 4820
    },
    {
      "epoch": 1.045055778186938,
      "grad_norm": 25.25,
      "learning_rate": 1.2061716992004078e-05,
      "logits/chosen": -2.8022468090057373,
      "logits/rejected": -2.777343511581421,
      "logps/chosen": -300.34149169921875,
      "logps/rejected": -352.2551574707031,
      "loss": 0.2592,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.854915618896484,
      "rewards/margins": 2.2466537952423096,
      "rewards/rejected": -7.101568698883057,
      "step": 4825
    },
    {
      "epoch": 1.0461388497779702,
      "grad_norm": 27.25,
      "learning_rate": 1.2056299114968713e-05,
      "logits/chosen": -2.7834525108337402,
      "logits/rejected": -2.7662806510925293,
      "logps/chosen": -327.2135314941406,
      "logps/rejected": -369.1499328613281,
      "loss": 0.3123,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.055089950561523,
      "rewards/margins": 2.4051856994628906,
      "rewards/rejected": -7.460275173187256,
      "step": 4830
    },
    {
      "epoch": 1.0472219213690024,
      "grad_norm": 19.75,
      "learning_rate": 1.2050876969380588e-05,
      "logits/chosen": -2.948120355606079,
      "logits/rejected": -2.89259934425354,
      "logps/chosen": -298.18792724609375,
      "logps/rejected": -355.685546875,
      "loss": 0.2278,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -4.6887712478637695,
      "rewards/margins": 2.5883374214172363,
      "rewards/rejected": -7.277109622955322,
      "step": 4835
    },
    {
      "epoch": 1.0483049929600348,
      "grad_norm": 17.125,
      "learning_rate": 1.2045450562970596e-05,
      "logits/chosen": -3.0414795875549316,
      "logits/rejected": -2.85253643989563,
      "logps/chosen": -311.2836608886719,
      "logps/rejected": -371.32867431640625,
      "loss": 0.3029,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.689549446105957,
      "rewards/margins": 2.4715869426727295,
      "rewards/rejected": -7.161136627197266,
      "step": 4840
    },
    {
      "epoch": 1.049388064551067,
      "grad_norm": 13.8125,
      "learning_rate": 1.2040019903475688e-05,
      "logits/chosen": -2.9978318214416504,
      "logits/rejected": -3.026535749435425,
      "logps/chosen": -315.6226501464844,
      "logps/rejected": -361.0539855957031,
      "loss": 0.2355,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.115020751953125,
      "rewards/margins": 2.616654872894287,
      "rewards/rejected": -7.7316765785217285,
      "step": 4845
    },
    {
      "epoch": 1.050471136142099,
      "grad_norm": 37.5,
      "learning_rate": 1.2034584998638895e-05,
      "logits/chosen": -2.9787654876708984,
      "logits/rejected": -2.9408419132232666,
      "logps/chosen": -328.7071228027344,
      "logps/rejected": -376.5032958984375,
      "loss": 0.3262,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.162481784820557,
      "rewards/margins": 2.6056582927703857,
      "rewards/rejected": -7.768139839172363,
      "step": 4850
    },
    {
      "epoch": 1.0515542077331312,
      "grad_norm": 14.625,
      "learning_rate": 1.2029145856209288e-05,
      "logits/chosen": -3.0518012046813965,
      "logits/rejected": -3.1469566822052,
      "logps/chosen": -324.21771240234375,
      "logps/rejected": -370.1889953613281,
      "loss": 0.2857,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.26211404800415,
      "rewards/margins": 2.819667339324951,
      "rewards/rejected": -8.081781387329102,
      "step": 4855
    },
    {
      "epoch": 1.0526372793241634,
      "grad_norm": 23.5,
      "learning_rate": 1.2023702483941988e-05,
      "logits/chosen": -2.982499122619629,
      "logits/rejected": -2.9022154808044434,
      "logps/chosen": -321.67376708984375,
      "logps/rejected": -374.6973571777344,
      "loss": 0.2695,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.995387554168701,
      "rewards/margins": 2.7814979553222656,
      "rewards/rejected": -7.776885986328125,
      "step": 4860
    },
    {
      "epoch": 1.0537203509151956,
      "grad_norm": 18.875,
      "learning_rate": 1.2018254889598142e-05,
      "logits/chosen": -2.9386439323425293,
      "logits/rejected": -2.9653265476226807,
      "logps/chosen": -321.98199462890625,
      "logps/rejected": -362.83154296875,
      "loss": 0.2918,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -5.101405143737793,
      "rewards/margins": 2.3764617443084717,
      "rewards/rejected": -7.477866172790527,
      "step": 4865
    },
    {
      "epoch": 1.0548034225062277,
      "grad_norm": 34.25,
      "learning_rate": 1.2012803080944923e-05,
      "logits/chosen": -2.8432459831237793,
      "logits/rejected": -2.8501715660095215,
      "logps/chosen": -321.4857482910156,
      "logps/rejected": -347.4130859375,
      "loss": 0.3563,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -4.744716167449951,
      "rewards/margins": 2.0937094688415527,
      "rewards/rejected": -6.838425636291504,
      "step": 4870
    },
    {
      "epoch": 1.0558864940972599,
      "grad_norm": 21.875,
      "learning_rate": 1.2007347065755506e-05,
      "logits/chosen": -2.7092905044555664,
      "logits/rejected": -2.8391480445861816,
      "logps/chosen": -325.26593017578125,
      "logps/rejected": -357.40167236328125,
      "loss": 0.2775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.29116678237915,
      "rewards/margins": 2.128044843673706,
      "rewards/rejected": -6.419211387634277,
      "step": 4875
    },
    {
      "epoch": 1.056969565688292,
      "grad_norm": 22.5,
      "learning_rate": 1.2001886851809066e-05,
      "logits/chosen": -2.7751259803771973,
      "logits/rejected": -2.7963993549346924,
      "logps/chosen": -317.8332214355469,
      "logps/rejected": -353.04534912109375,
      "loss": 0.318,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.088192939758301,
      "rewards/margins": 2.215643882751465,
      "rewards/rejected": -7.303836822509766,
      "step": 4880
    },
    {
      "epoch": 1.0580526372793242,
      "grad_norm": 13.3125,
      "learning_rate": 1.1996422446890766e-05,
      "logits/chosen": -3.022732734680176,
      "logits/rejected": -2.9098565578460693,
      "logps/chosen": -309.124267578125,
      "logps/rejected": -358.92999267578125,
      "loss": 0.2481,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.915335655212402,
      "rewards/margins": 2.388824939727783,
      "rewards/rejected": -7.304160118103027,
      "step": 4885
    },
    {
      "epoch": 1.0591357088703564,
      "grad_norm": 15.3125,
      "learning_rate": 1.1990953858791744e-05,
      "logits/chosen": -2.808959484100342,
      "logits/rejected": -2.7960476875305176,
      "logps/chosen": -312.7492980957031,
      "logps/rejected": -370.5310974121094,
      "loss": 0.2137,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -4.486293792724609,
      "rewards/margins": 2.714083194732666,
      "rewards/rejected": -7.200376987457275,
      "step": 4890
    },
    {
      "epoch": 1.0602187804613885,
      "grad_norm": 22.125,
      "learning_rate": 1.1985481095309098e-05,
      "logits/chosen": -2.8785338401794434,
      "logits/rejected": -2.8778960704803467,
      "logps/chosen": -310.94915771484375,
      "logps/rejected": -357.7125549316406,
      "loss": 0.213,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.510578632354736,
      "rewards/margins": 2.759395122528076,
      "rewards/rejected": -7.2699737548828125,
      "step": 4895
    },
    {
      "epoch": 1.0613018520524207,
      "grad_norm": 19.75,
      "learning_rate": 1.1980004164245892e-05,
      "logits/chosen": -3.0200698375701904,
      "logits/rejected": -2.8563625812530518,
      "logps/chosen": -312.39593505859375,
      "logps/rejected": -368.41619873046875,
      "loss": 0.2416,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.908174991607666,
      "rewards/margins": 2.6215827465057373,
      "rewards/rejected": -7.529758453369141,
      "step": 4900
    },
    {
      "epoch": 1.0623849236434528,
      "grad_norm": 25.125,
      "learning_rate": 1.1974523073411112e-05,
      "logits/chosen": -3.0433998107910156,
      "logits/rejected": -3.0251822471618652,
      "logps/chosen": -314.3352355957031,
      "logps/rejected": -364.34552001953125,
      "loss": 0.3568,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.925260543823242,
      "rewards/margins": 2.3694074153900146,
      "rewards/rejected": -7.294667720794678,
      "step": 4905
    },
    {
      "epoch": 1.063467995234485,
      "grad_norm": 19.125,
      "learning_rate": 1.1969037830619696e-05,
      "logits/chosen": -2.9487035274505615,
      "logits/rejected": -2.8155808448791504,
      "logps/chosen": -334.4192810058594,
      "logps/rejected": -375.4078369140625,
      "loss": 0.3277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.5972161293029785,
      "rewards/margins": 2.2113466262817383,
      "rewards/rejected": -7.8085618019104,
      "step": 4910
    },
    {
      "epoch": 1.0645510668255171,
      "grad_norm": 25.625,
      "learning_rate": 1.1963548443692488e-05,
      "logits/chosen": -3.0058326721191406,
      "logits/rejected": -2.8980534076690674,
      "logps/chosen": -324.68450927734375,
      "logps/rejected": -391.91143798828125,
      "loss": 0.2426,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.572323799133301,
      "rewards/margins": 2.7561750411987305,
      "rewards/rejected": -8.328498840332031,
      "step": 4915
    },
    {
      "epoch": 1.0656341384165493,
      "grad_norm": 16.75,
      "learning_rate": 1.1958054920456244e-05,
      "logits/chosen": -3.0527260303497314,
      "logits/rejected": -2.95538592338562,
      "logps/chosen": -339.6614990234375,
      "logps/rejected": -383.9004211425781,
      "loss": 0.3305,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.945977687835693,
      "rewards/margins": 2.0737709999084473,
      "rewards/rejected": -8.01974868774414,
      "step": 4920
    },
    {
      "epoch": 1.0667172100075815,
      "grad_norm": 16.875,
      "learning_rate": 1.195255726874362e-05,
      "logits/chosen": -2.9719977378845215,
      "logits/rejected": -3.0060410499572754,
      "logps/chosen": -334.288330078125,
      "logps/rejected": -381.21466064453125,
      "loss": 0.2695,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.308953762054443,
      "rewards/margins": 2.5714619159698486,
      "rewards/rejected": -7.880415439605713,
      "step": 4925
    },
    {
      "epoch": 1.0678002815986136,
      "grad_norm": 20.625,
      "learning_rate": 1.1947055496393157e-05,
      "logits/chosen": -2.9619038105010986,
      "logits/rejected": -2.8666131496429443,
      "logps/chosen": -330.406982421875,
      "logps/rejected": -374.545166015625,
      "loss": 0.368,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.422031879425049,
      "rewards/margins": 1.960541009902954,
      "rewards/rejected": -7.382573127746582,
      "step": 4930
    },
    {
      "epoch": 1.0688833531896458,
      "grad_norm": 20.875,
      "learning_rate": 1.1941549611249266e-05,
      "logits/chosen": -2.9008429050445557,
      "logits/rejected": -2.8735766410827637,
      "logps/chosen": -334.2024841308594,
      "logps/rejected": -392.1925964355469,
      "loss": 0.2809,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.43496036529541,
      "rewards/margins": 2.5163075923919678,
      "rewards/rejected": -7.951267242431641,
      "step": 4935
    },
    {
      "epoch": 1.069966424780678,
      "grad_norm": 22.75,
      "learning_rate": 1.1936039621162236e-05,
      "logits/chosen": -2.9192519187927246,
      "logits/rejected": -2.6770153045654297,
      "logps/chosen": -325.17425537109375,
      "logps/rejected": -379.9082336425781,
      "loss": 0.332,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -5.21001672744751,
      "rewards/margins": 2.049626588821411,
      "rewards/rejected": -7.259642601013184,
      "step": 4940
    },
    {
      "epoch": 1.07104949637171,
      "grad_norm": 19.5,
      "learning_rate": 1.1930525533988193e-05,
      "logits/chosen": -2.945441722869873,
      "logits/rejected": -3.051933765411377,
      "logps/chosen": -352.1643371582031,
      "logps/rejected": -399.16766357421875,
      "loss": 0.2632,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.954959392547607,
      "rewards/margins": 2.6816790103912354,
      "rewards/rejected": -8.636638641357422,
      "step": 4945
    },
    {
      "epoch": 1.0721325679627423,
      "grad_norm": 19.75,
      "learning_rate": 1.1925007357589114e-05,
      "logits/chosen": -2.946361780166626,
      "logits/rejected": -2.8812098503112793,
      "logps/chosen": -326.5589904785156,
      "logps/rejected": -381.6416931152344,
      "loss": 0.2618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.214931011199951,
      "rewards/margins": 2.5520553588867188,
      "rewards/rejected": -7.766986846923828,
      "step": 4950
    },
    {
      "epoch": 1.0732156395537744,
      "grad_norm": 14.625,
      "learning_rate": 1.1919485099832801e-05,
      "logits/chosen": -2.931492328643799,
      "logits/rejected": -2.9242916107177734,
      "logps/chosen": -320.69744873046875,
      "logps/rejected": -381.4212341308594,
      "loss": 0.1912,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -4.80952787399292,
      "rewards/margins": 2.935717821121216,
      "rewards/rejected": -7.74524450302124,
      "step": 4955
    },
    {
      "epoch": 1.0742987111448066,
      "grad_norm": 16.375,
      "learning_rate": 1.1913958768592883e-05,
      "logits/chosen": -3.0097036361694336,
      "logits/rejected": -2.8943347930908203,
      "logps/chosen": -325.7027282714844,
      "logps/rejected": -380.634521484375,
      "loss": 0.2686,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.516648769378662,
      "rewards/margins": 2.5264527797698975,
      "rewards/rejected": -8.04310131072998,
      "step": 4960
    },
    {
      "epoch": 1.075381782735839,
      "grad_norm": 26.125,
      "learning_rate": 1.1908428371748785e-05,
      "logits/chosen": -2.987274646759033,
      "logits/rejected": -2.949486494064331,
      "logps/chosen": -340.60113525390625,
      "logps/rejected": -396.90740966796875,
      "loss": 0.3218,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.756770133972168,
      "rewards/margins": 2.2918076515197754,
      "rewards/rejected": -8.048577308654785,
      "step": 4965
    },
    {
      "epoch": 1.0764648543268711,
      "grad_norm": 22.25,
      "learning_rate": 1.1902893917185744e-05,
      "logits/chosen": -3.0707881450653076,
      "logits/rejected": -3.0056064128875732,
      "logps/chosen": -329.7479553222656,
      "logps/rejected": -382.24017333984375,
      "loss": 0.2739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.6616387367248535,
      "rewards/margins": 2.631995677947998,
      "rewards/rejected": -8.293634414672852,
      "step": 4970
    },
    {
      "epoch": 1.0775479259179033,
      "grad_norm": 20.0,
      "learning_rate": 1.1897355412794767e-05,
      "logits/chosen": -2.901228427886963,
      "logits/rejected": -2.8944153785705566,
      "logps/chosen": -343.62396240234375,
      "logps/rejected": -373.1121826171875,
      "loss": 0.3338,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.388866424560547,
      "rewards/margins": 2.1418395042419434,
      "rewards/rejected": -7.530706882476807,
      "step": 4975
    },
    {
      "epoch": 1.0786309975089354,
      "grad_norm": 16.625,
      "learning_rate": 1.189181286647265e-05,
      "logits/chosen": -2.889392614364624,
      "logits/rejected": -2.778939723968506,
      "logps/chosen": -347.27093505859375,
      "logps/rejected": -396.64862060546875,
      "loss": 0.224,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -5.62418270111084,
      "rewards/margins": 2.6824891567230225,
      "rewards/rejected": -8.306673049926758,
      "step": 4980
    },
    {
      "epoch": 1.0797140690999676,
      "grad_norm": 15.5,
      "learning_rate": 1.1886266286121933e-05,
      "logits/chosen": -3.120382308959961,
      "logits/rejected": -3.125608205795288,
      "logps/chosen": -326.3967590332031,
      "logps/rejected": -379.1897888183594,
      "loss": 0.2821,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.861854076385498,
      "rewards/margins": 2.4775888919830322,
      "rewards/rejected": -8.33944320678711,
      "step": 4985
    },
    {
      "epoch": 1.0807971406909997,
      "grad_norm": 30.375,
      "learning_rate": 1.188071567965093e-05,
      "logits/chosen": -3.1408817768096924,
      "logits/rejected": -2.9878017902374268,
      "logps/chosen": -334.3415222167969,
      "logps/rejected": -378.6955871582031,
      "loss": 0.2906,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.171787738800049,
      "rewards/margins": 2.20393705368042,
      "rewards/rejected": -8.375723838806152,
      "step": 4990
    },
    {
      "epoch": 1.081880212282032,
      "grad_norm": 22.5,
      "learning_rate": 1.1875161054973678e-05,
      "logits/chosen": -2.997924327850342,
      "logits/rejected": -2.981229066848755,
      "logps/chosen": -336.75531005859375,
      "logps/rejected": -374.53021240234375,
      "loss": 0.3045,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.425130844116211,
      "rewards/margins": 2.093954086303711,
      "rewards/rejected": -7.519084930419922,
      "step": 4995
    },
    {
      "epoch": 1.082963283873064,
      "grad_norm": 22.25,
      "learning_rate": 1.1869602420009955e-05,
      "logits/chosen": -2.866912364959717,
      "logits/rejected": -2.8167977333068848,
      "logps/chosen": -314.6557312011719,
      "logps/rejected": -367.09173583984375,
      "loss": 0.3557,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.782488822937012,
      "rewards/margins": 2.588285446166992,
      "rewards/rejected": -7.370774745941162,
      "step": 5000
    },
    {
      "epoch": 1.082963283873064,
      "eval_logits/chosen": -3.05830717086792,
      "eval_logits/rejected": -2.983677864074707,
      "eval_logps/chosen": -316.782958984375,
      "eval_logps/rejected": -356.4382019042969,
      "eval_loss": 0.42869651317596436,
      "eval_rewards/accuracies": 0.7749999761581421,
      "eval_rewards/chosen": -5.129085063934326,
      "eval_rewards/margins": 1.8099684715270996,
      "eval_rewards/rejected": -6.939054489135742,
      "eval_runtime": 14.9097,
      "eval_samples_per_second": 13.414,
      "eval_steps_per_second": 13.414,
      "step": 5000
    },
    {
      "epoch": 1.0840463554640962,
      "grad_norm": 13.875,
      "learning_rate": 1.1864039782685247e-05,
      "logits/chosen": -2.9298434257507324,
      "logits/rejected": -3.0011842250823975,
      "logps/chosen": -314.8794860839844,
      "logps/rejected": -358.65350341796875,
      "loss": 0.2467,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.041143894195557,
      "rewards/margins": 2.443082094192505,
      "rewards/rejected": -7.484226226806641,
      "step": 5005
    },
    {
      "epoch": 1.0851294270551284,
      "grad_norm": 19.0,
      "learning_rate": 1.1858473150930754e-05,
      "logits/chosen": -2.9989171028137207,
      "logits/rejected": -2.8328452110290527,
      "logps/chosen": -298.327880859375,
      "logps/rejected": -351.90679931640625,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.820160865783691,
      "rewards/margins": 2.2779622077941895,
      "rewards/rejected": -7.098123073577881,
      "step": 5010
    },
    {
      "epoch": 1.0862124986461605,
      "grad_norm": 8.5625,
      "learning_rate": 1.1852902532683364e-05,
      "logits/chosen": -2.959766387939453,
      "logits/rejected": -2.817348003387451,
      "logps/chosen": -312.8594665527344,
      "logps/rejected": -398.5907897949219,
      "loss": 0.2011,
      "rewards/accuracies": 0.9583333134651184,
      "rewards/chosen": -4.845559597015381,
      "rewards/margins": 2.977017879486084,
      "rewards/rejected": -7.822577476501465,
      "step": 5015
    },
    {
      "epoch": 1.0872955702371927,
      "grad_norm": 16.625,
      "learning_rate": 1.1847327935885661e-05,
      "logits/chosen": -3.0114567279815674,
      "logits/rejected": -2.9978766441345215,
      "logps/chosen": -317.7944641113281,
      "logps/rejected": -358.8961486816406,
      "loss": 0.2364,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.232514381408691,
      "rewards/margins": 2.481210708618164,
      "rewards/rejected": -7.713724613189697,
      "step": 5020
    },
    {
      "epoch": 1.0883786418282249,
      "grad_norm": 23.875,
      "learning_rate": 1.1841749368485884e-05,
      "logits/chosen": -3.0977683067321777,
      "logits/rejected": -2.9824702739715576,
      "logps/chosen": -325.4405517578125,
      "logps/rejected": -383.63018798828125,
      "loss": 0.2318,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.1952290534973145,
      "rewards/margins": 2.822434186935425,
      "rewards/rejected": -8.017663955688477,
      "step": 5025
    },
    {
      "epoch": 1.089461713419257,
      "grad_norm": 11.625,
      "learning_rate": 1.1836166838437954e-05,
      "logits/chosen": -2.8960001468658447,
      "logits/rejected": -2.858159303665161,
      "logps/chosen": -345.2682800292969,
      "logps/rejected": -394.51470947265625,
      "loss": 0.2152,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.909478664398193,
      "rewards/margins": 2.6169610023498535,
      "rewards/rejected": -8.526440620422363,
      "step": 5030
    },
    {
      "epoch": 1.0905447850102892,
      "grad_norm": 19.875,
      "learning_rate": 1.1830580353701421e-05,
      "logits/chosen": -3.0169930458068848,
      "logits/rejected": -2.9594569206237793,
      "logps/chosen": -318.1650085449219,
      "logps/rejected": -369.26092529296875,
      "loss": 0.2536,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.9986653327941895,
      "rewards/margins": 2.5741231441497803,
      "rewards/rejected": -7.572788238525391,
      "step": 5035
    },
    {
      "epoch": 1.0916278566013213,
      "grad_norm": 19.375,
      "learning_rate": 1.1824989922241492e-05,
      "logits/chosen": -3.0389246940612793,
      "logits/rejected": -3.0775909423828125,
      "logps/chosen": -331.48040771484375,
      "logps/rejected": -367.7668151855469,
      "loss": 0.2506,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.595464706420898,
      "rewards/margins": 2.8027663230895996,
      "rewards/rejected": -8.39823055267334,
      "step": 5040
    },
    {
      "epoch": 1.0927109281923535,
      "grad_norm": 20.625,
      "learning_rate": 1.1819395552028985e-05,
      "logits/chosen": -3.0415408611297607,
      "logits/rejected": -2.8423264026641846,
      "logps/chosen": -321.376953125,
      "logps/rejected": -373.16424560546875,
      "loss": 0.3096,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.117890357971191,
      "rewards/margins": 2.267782688140869,
      "rewards/rejected": -7.385672569274902,
      "step": 5045
    },
    {
      "epoch": 1.0937939997833857,
      "grad_norm": 21.5,
      "learning_rate": 1.1813797251040349e-05,
      "logits/chosen": -2.965986967086792,
      "logits/rejected": -3.0410075187683105,
      "logps/chosen": -326.51434326171875,
      "logps/rejected": -365.88616943359375,
      "loss": 0.2843,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.029885768890381,
      "rewards/margins": 2.4292654991149902,
      "rewards/rejected": -7.4591522216796875,
      "step": 5050
    },
    {
      "epoch": 1.0948770713744178,
      "grad_norm": 20.5,
      "learning_rate": 1.1808195027257625e-05,
      "logits/chosen": -3.0103516578674316,
      "logits/rejected": -2.923748016357422,
      "logps/chosen": -310.1313781738281,
      "logps/rejected": -350.0364074707031,
      "loss": 0.2741,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.751645088195801,
      "rewards/margins": 2.4383585453033447,
      "rewards/rejected": -7.190003395080566,
      "step": 5055
    },
    {
      "epoch": 1.09596014296545,
      "grad_norm": 20.75,
      "learning_rate": 1.1802588888668457e-05,
      "logits/chosen": -3.0639548301696777,
      "logits/rejected": -3.013800621032715,
      "logps/chosen": -316.31951904296875,
      "logps/rejected": -359.0628967285156,
      "loss": 0.281,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.0403218269348145,
      "rewards/margins": 2.401057004928589,
      "rewards/rejected": -7.441378116607666,
      "step": 5060
    },
    {
      "epoch": 1.0970432145564821,
      "grad_norm": 24.0,
      "learning_rate": 1.1796978843266064e-05,
      "logits/chosen": -3.0003628730773926,
      "logits/rejected": -3.005751132965088,
      "logps/chosen": -311.59375,
      "logps/rejected": -353.3387145996094,
      "loss": 0.2723,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.667974472045898,
      "rewards/margins": 2.2996668815612793,
      "rewards/rejected": -6.9676408767700195,
      "step": 5065
    },
    {
      "epoch": 1.0981262861475143,
      "grad_norm": 22.375,
      "learning_rate": 1.1791364899049234e-05,
      "logits/chosen": -2.827502489089966,
      "logits/rejected": -2.895876884460449,
      "logps/chosen": -336.0487060546875,
      "logps/rejected": -381.8141174316406,
      "loss": 0.2996,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.112926006317139,
      "rewards/margins": 2.621868371963501,
      "rewards/rejected": -7.734795570373535,
      "step": 5070
    },
    {
      "epoch": 1.0992093577385464,
      "grad_norm": 14.75,
      "learning_rate": 1.1785747064022325e-05,
      "logits/chosen": -2.958299160003662,
      "logits/rejected": -2.8529727458953857,
      "logps/chosen": -329.0645751953125,
      "logps/rejected": -368.4658508300781,
      "loss": 0.302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.24884557723999,
      "rewards/margins": 2.3967955112457275,
      "rewards/rejected": -7.645641326904297,
      "step": 5075
    },
    {
      "epoch": 1.1002924293295786,
      "grad_norm": 17.0,
      "learning_rate": 1.1780125346195228e-05,
      "logits/chosen": -2.8976166248321533,
      "logits/rejected": -2.83986759185791,
      "logps/chosen": -327.54638671875,
      "logps/rejected": -374.480712890625,
      "loss": 0.2884,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.489624977111816,
      "rewards/margins": 2.2070298194885254,
      "rewards/rejected": -7.696653842926025,
      "step": 5080
    },
    {
      "epoch": 1.1013755009206108,
      "grad_norm": 27.875,
      "learning_rate": 1.177449975358338e-05,
      "logits/chosen": -2.9501678943634033,
      "logits/rejected": -2.862287998199463,
      "logps/chosen": -313.1620788574219,
      "logps/rejected": -360.73504638671875,
      "loss": 0.3412,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.381929874420166,
      "rewards/margins": 2.2705039978027344,
      "rewards/rejected": -6.6524338722229,
      "step": 5085
    },
    {
      "epoch": 1.102458572511643,
      "grad_norm": 22.125,
      "learning_rate": 1.1768870294207737e-05,
      "logits/chosen": -2.90181565284729,
      "logits/rejected": -2.9992504119873047,
      "logps/chosen": -311.0547180175781,
      "logps/rejected": -337.27838134765625,
      "loss": 0.3028,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -4.462460517883301,
      "rewards/margins": 2.2098498344421387,
      "rewards/rejected": -6.672310829162598,
      "step": 5090
    },
    {
      "epoch": 1.103541644102675,
      "grad_norm": 20.25,
      "learning_rate": 1.1763236976094776e-05,
      "logits/chosen": -2.911648988723755,
      "logits/rejected": -2.902916669845581,
      "logps/chosen": -305.46197509765625,
      "logps/rejected": -357.6941833496094,
      "loss": 0.2519,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.893385410308838,
      "rewards/margins": 2.5823776721954346,
      "rewards/rejected": -7.475762844085693,
      "step": 5095
    },
    {
      "epoch": 1.1046247156937072,
      "grad_norm": 15.8125,
      "learning_rate": 1.1757599807276466e-05,
      "logits/chosen": -2.6915812492370605,
      "logits/rejected": -2.6937131881713867,
      "logps/chosen": -328.281494140625,
      "logps/rejected": -370.51898193359375,
      "loss": 0.2737,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.642270088195801,
      "rewards/margins": 2.4715332984924316,
      "rewards/rejected": -7.113802909851074,
      "step": 5100
    },
    {
      "epoch": 1.1057077872847396,
      "grad_norm": 14.25,
      "learning_rate": 1.1751958795790273e-05,
      "logits/chosen": -2.805500030517578,
      "logits/rejected": -2.7630133628845215,
      "logps/chosen": -300.13580322265625,
      "logps/rejected": -350.3551940917969,
      "loss": 0.2247,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.219851970672607,
      "rewards/margins": 2.683058023452759,
      "rewards/rejected": -6.902909278869629,
      "step": 5105
    },
    {
      "epoch": 1.1067908588757718,
      "grad_norm": 20.25,
      "learning_rate": 1.174631394967914e-05,
      "logits/chosen": -2.877005100250244,
      "logits/rejected": -2.9841344356536865,
      "logps/chosen": -301.84698486328125,
      "logps/rejected": -347.6138610839844,
      "loss": 0.2841,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.58458948135376,
      "rewards/margins": 2.2472829818725586,
      "rewards/rejected": -6.831872463226318,
      "step": 5110
    },
    {
      "epoch": 1.107873930466804,
      "grad_norm": 20.375,
      "learning_rate": 1.1740665276991475e-05,
      "logits/chosen": -2.908519983291626,
      "logits/rejected": -2.9126808643341064,
      "logps/chosen": -313.43621826171875,
      "logps/rejected": -358.3125915527344,
      "loss": 0.3161,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.8433661460876465,
      "rewards/margins": 2.1054790019989014,
      "rewards/rejected": -6.948845863342285,
      "step": 5115
    },
    {
      "epoch": 1.108957002057836,
      "grad_norm": 15.375,
      "learning_rate": 1.1735012785781149e-05,
      "logits/chosen": -2.8456008434295654,
      "logits/rejected": -2.883779525756836,
      "logps/chosen": -310.776123046875,
      "logps/rejected": -352.92498779296875,
      "loss": 0.2939,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.333671569824219,
      "rewards/margins": 2.3055062294006348,
      "rewards/rejected": -6.6391777992248535,
      "step": 5120
    },
    {
      "epoch": 1.1100400736488683,
      "grad_norm": 24.75,
      "learning_rate": 1.1729356484107464e-05,
      "logits/chosen": -2.971302032470703,
      "logits/rejected": -2.832404375076294,
      "logps/chosen": -304.2981872558594,
      "logps/rejected": -357.8982849121094,
      "loss": 0.274,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.499133110046387,
      "rewards/margins": 2.581735134124756,
      "rewards/rejected": -7.080867767333984,
      "step": 5125
    },
    {
      "epoch": 1.1111231452399004,
      "grad_norm": 25.375,
      "learning_rate": 1.1723696380035174e-05,
      "logits/chosen": -2.968930959701538,
      "logits/rejected": -2.982342004776001,
      "logps/chosen": -318.5821228027344,
      "logps/rejected": -363.59722900390625,
      "loss": 0.2382,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.683012962341309,
      "rewards/margins": 2.659890651702881,
      "rewards/rejected": -7.342903137207031,
      "step": 5130
    },
    {
      "epoch": 1.1122062168309326,
      "grad_norm": 10.9375,
      "learning_rate": 1.1718032481634436e-05,
      "logits/chosen": -2.8407580852508545,
      "logits/rejected": -2.937985897064209,
      "logps/chosen": -333.840576171875,
      "logps/rejected": -369.1153869628906,
      "loss": 0.2343,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.977300643920898,
      "rewards/margins": 2.8599724769592285,
      "rewards/rejected": -7.837273597717285,
      "step": 5135
    },
    {
      "epoch": 1.1132892884219647,
      "grad_norm": 14.3125,
      "learning_rate": 1.1712364796980828e-05,
      "logits/chosen": -2.8827242851257324,
      "logits/rejected": -2.9083123207092285,
      "logps/chosen": -335.8120422363281,
      "logps/rejected": -374.0755310058594,
      "loss": 0.2541,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.811016082763672,
      "rewards/margins": 2.609537124633789,
      "rewards/rejected": -7.420553684234619,
      "step": 5140
    },
    {
      "epoch": 1.1143723600129969,
      "grad_norm": 24.75,
      "learning_rate": 1.1706693334155324e-05,
      "logits/chosen": -2.867980480194092,
      "logits/rejected": -2.8641152381896973,
      "logps/chosen": -321.7567443847656,
      "logps/rejected": -379.3270263671875,
      "loss": 0.2241,
      "rewards/accuracies": 0.9249998927116394,
      "rewards/chosen": -4.907466411590576,
      "rewards/margins": 3.146967649459839,
      "rewards/rejected": -8.054433822631836,
      "step": 5145
    },
    {
      "epoch": 1.115455431604029,
      "grad_norm": 18.375,
      "learning_rate": 1.1701018101244284e-05,
      "logits/chosen": -2.7783684730529785,
      "logits/rejected": -2.772695541381836,
      "logps/chosen": -327.6959228515625,
      "logps/rejected": -373.978515625,
      "loss": 0.2412,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.818876266479492,
      "rewards/margins": 2.5761654376983643,
      "rewards/rejected": -7.395041465759277,
      "step": 5150
    },
    {
      "epoch": 1.1165385031950612,
      "grad_norm": 17.0,
      "learning_rate": 1.1695339106339442e-05,
      "logits/chosen": -2.68461537361145,
      "logits/rejected": -2.6392428874969482,
      "logps/chosen": -335.22332763671875,
      "logps/rejected": -403.16790771484375,
      "loss": 0.2218,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.485311985015869,
      "rewards/margins": 2.9856724739074707,
      "rewards/rejected": -7.470983982086182,
      "step": 5155
    },
    {
      "epoch": 1.1176215747860934,
      "grad_norm": 15.0,
      "learning_rate": 1.16896563575379e-05,
      "logits/chosen": -2.7394678592681885,
      "logits/rejected": -2.7401514053344727,
      "logps/chosen": -305.889404296875,
      "logps/rejected": -374.47174072265625,
      "loss": 0.2318,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -4.55206823348999,
      "rewards/margins": 2.506786584854126,
      "rewards/rejected": -7.058854579925537,
      "step": 5160
    },
    {
      "epoch": 1.1187046463771255,
      "grad_norm": 24.875,
      "learning_rate": 1.1683969862942109e-05,
      "logits/chosen": -2.764909505844116,
      "logits/rejected": -2.6453678607940674,
      "logps/chosen": -324.53399658203125,
      "logps/rejected": -369.94805908203125,
      "loss": 0.3062,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -4.747719764709473,
      "rewards/margins": 2.3708205223083496,
      "rewards/rejected": -7.118539333343506,
      "step": 5165
    },
    {
      "epoch": 1.1197877179681577,
      "grad_norm": 20.5,
      "learning_rate": 1.1678279630659859e-05,
      "logits/chosen": -2.793520450592041,
      "logits/rejected": -2.703333854675293,
      "logps/chosen": -320.4947204589844,
      "logps/rejected": -365.96923828125,
      "loss": 0.2745,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -5.16190242767334,
      "rewards/margins": 2.3547072410583496,
      "rewards/rejected": -7.516609191894531,
      "step": 5170
    },
    {
      "epoch": 1.1208707895591898,
      "grad_norm": 24.875,
      "learning_rate": 1.1672585668804274e-05,
      "logits/chosen": -2.777155876159668,
      "logits/rejected": -2.8104772567749023,
      "logps/chosen": -334.9007873535156,
      "logps/rejected": -380.1591491699219,
      "loss": 0.2524,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -4.9966912269592285,
      "rewards/margins": 2.809450626373291,
      "rewards/rejected": -7.8061418533325195,
      "step": 5175
    },
    {
      "epoch": 1.121953861150222,
      "grad_norm": 23.5,
      "learning_rate": 1.1666887985493794e-05,
      "logits/chosen": -2.7658259868621826,
      "logits/rejected": -2.6086013317108154,
      "logps/chosen": -310.3309020996094,
      "logps/rejected": -368.8045654296875,
      "loss": 0.2573,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.0577545166015625,
      "rewards/margins": 2.833447217941284,
      "rewards/rejected": -6.891201972961426,
      "step": 5180
    },
    {
      "epoch": 1.1230369327412542,
      "grad_norm": 21.125,
      "learning_rate": 1.1661186588852165e-05,
      "logits/chosen": -2.95611834526062,
      "logits/rejected": -2.8349685668945312,
      "logps/chosen": -322.6859436035156,
      "logps/rejected": -372.9110412597656,
      "loss": 0.2746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.884041786193848,
      "rewards/margins": 2.685509204864502,
      "rewards/rejected": -7.56955099105835,
      "step": 5185
    },
    {
      "epoch": 1.1241200043322863,
      "grad_norm": 28.875,
      "learning_rate": 1.1655481487008422e-05,
      "logits/chosen": -3.0272583961486816,
      "logits/rejected": -2.955605983734131,
      "logps/chosen": -322.0563659667969,
      "logps/rejected": -369.1878356933594,
      "loss": 0.2811,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.638443946838379,
      "rewards/margins": 2.292686939239502,
      "rewards/rejected": -7.931130409240723,
      "step": 5190
    },
    {
      "epoch": 1.1252030759233185,
      "grad_norm": 19.25,
      "learning_rate": 1.1649772688096893e-05,
      "logits/chosen": -2.8762099742889404,
      "logits/rejected": -2.8668580055236816,
      "logps/chosen": -306.9568176269531,
      "logps/rejected": -356.97430419921875,
      "loss": 0.2564,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.442292213439941,
      "rewards/margins": 2.6777515411376953,
      "rewards/rejected": -7.1200432777404785,
      "step": 5195
    },
    {
      "epoch": 1.1262861475143506,
      "grad_norm": 8.0,
      "learning_rate": 1.164406020025717e-05,
      "logits/chosen": -2.8727612495422363,
      "logits/rejected": -2.842531681060791,
      "logps/chosen": -313.61663818359375,
      "logps/rejected": -388.28497314453125,
      "loss": 0.1951,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -4.3643598556518555,
      "rewards/margins": 2.944732904434204,
      "rewards/rejected": -7.309092044830322,
      "step": 5200
    },
    {
      "epoch": 1.1273692191053828,
      "grad_norm": 17.5,
      "learning_rate": 1.1638344031634108e-05,
      "logits/chosen": -3.050197124481201,
      "logits/rejected": -2.883702278137207,
      "logps/chosen": -298.922119140625,
      "logps/rejected": -361.1224670410156,
      "loss": 0.2803,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.49001407623291,
      "rewards/margins": 2.755986213684082,
      "rewards/rejected": -7.245999813079834,
      "step": 5205
    },
    {
      "epoch": 1.128452290696415,
      "grad_norm": 16.25,
      "learning_rate": 1.1632624190377806e-05,
      "logits/chosen": -3.060039758682251,
      "logits/rejected": -2.9530460834503174,
      "logps/chosen": -329.8490295410156,
      "logps/rejected": -388.54278564453125,
      "loss": 0.2394,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.469106197357178,
      "rewards/margins": 3.1607327461242676,
      "rewards/rejected": -8.629838943481445,
      "step": 5210
    },
    {
      "epoch": 1.129535362287447,
      "grad_norm": 17.5,
      "learning_rate": 1.1626900684643606e-05,
      "logits/chosen": -2.941904306411743,
      "logits/rejected": -2.951423406600952,
      "logps/chosen": -343.3577575683594,
      "logps/rejected": -381.97735595703125,
      "loss": 0.3227,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.652168273925781,
      "rewards/margins": 2.4316983222961426,
      "rewards/rejected": -8.083866119384766,
      "step": 5215
    },
    {
      "epoch": 1.1306184338784795,
      "grad_norm": 26.625,
      "learning_rate": 1.1621173522592063e-05,
      "logits/chosen": -3.011502504348755,
      "logits/rejected": -3.0309669971466064,
      "logps/chosen": -328.8184814453125,
      "logps/rejected": -368.65966796875,
      "loss": 0.2805,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.145689010620117,
      "rewards/margins": 2.5739190578460693,
      "rewards/rejected": -7.719608306884766,
      "step": 5220
    },
    {
      "epoch": 1.1317015054695116,
      "grad_norm": 17.5,
      "learning_rate": 1.1615442712388963e-05,
      "logits/chosen": -3.0515527725219727,
      "logits/rejected": -2.9664206504821777,
      "logps/chosen": -324.15557861328125,
      "logps/rejected": -383.5330505371094,
      "loss": 0.2424,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.971574783325195,
      "rewards/margins": 2.7741334438323975,
      "rewards/rejected": -7.745708465576172,
      "step": 5225
    },
    {
      "epoch": 1.1327845770605438,
      "grad_norm": 19.0,
      "learning_rate": 1.160970826220528e-05,
      "logits/chosen": -2.906461000442505,
      "logits/rejected": -2.922682523727417,
      "logps/chosen": -305.47540283203125,
      "logps/rejected": -360.72796630859375,
      "loss": 0.2419,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.63401460647583,
      "rewards/margins": 2.728085517883301,
      "rewards/rejected": -7.362100124359131,
      "step": 5230
    },
    {
      "epoch": 1.133867648651576,
      "grad_norm": 10.6875,
      "learning_rate": 1.1603970180217182e-05,
      "logits/chosen": -2.8629441261291504,
      "logits/rejected": -2.9176223278045654,
      "logps/chosen": -315.03533935546875,
      "logps/rejected": -348.65643310546875,
      "loss": 0.2647,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.462224006652832,
      "rewards/margins": 2.1874101161956787,
      "rewards/rejected": -6.64963436126709,
      "step": 5235
    },
    {
      "epoch": 1.1349507202426081,
      "grad_norm": 19.375,
      "learning_rate": 1.1598228474606011e-05,
      "logits/chosen": -3.017625093460083,
      "logits/rejected": -3.0090928077697754,
      "logps/chosen": -302.34979248046875,
      "logps/rejected": -360.8261413574219,
      "loss": 0.2419,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.491755962371826,
      "rewards/margins": 2.895845890045166,
      "rewards/rejected": -7.387601375579834,
      "step": 5240
    },
    {
      "epoch": 1.1360337918336403,
      "grad_norm": 15.625,
      "learning_rate": 1.1592483153558283e-05,
      "logits/chosen": -2.9343149662017822,
      "logits/rejected": -2.822826862335205,
      "logps/chosen": -317.76947021484375,
      "logps/rejected": -365.1672668457031,
      "loss": 0.3524,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -4.528138637542725,
      "rewards/margins": 2.2011427879333496,
      "rewards/rejected": -6.729280948638916,
      "step": 5245
    },
    {
      "epoch": 1.1371168634246724,
      "grad_norm": 19.5,
      "learning_rate": 1.1586734225265663e-05,
      "logits/chosen": -3.1009650230407715,
      "logits/rejected": -3.0929970741271973,
      "logps/chosen": -313.02960205078125,
      "logps/rejected": -351.4237365722656,
      "loss": 0.273,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.869574069976807,
      "rewards/margins": 2.6137521266937256,
      "rewards/rejected": -7.4833269119262695,
      "step": 5250
    },
    {
      "epoch": 1.1381999350157046,
      "grad_norm": 10.1875,
      "learning_rate": 1.1580981697924965e-05,
      "logits/chosen": -2.963125228881836,
      "logits/rejected": -2.8859708309173584,
      "logps/chosen": -314.79193115234375,
      "logps/rejected": -374.1562194824219,
      "loss": 0.2471,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.367064476013184,
      "rewards/margins": 2.9176948070526123,
      "rewards/rejected": -7.284759521484375,
      "step": 5255
    },
    {
      "epoch": 1.1392830066067368,
      "grad_norm": 18.625,
      "learning_rate": 1.1575225579738125e-05,
      "logits/chosen": -3.109464168548584,
      "logits/rejected": -3.0129051208496094,
      "logps/chosen": -299.0821838378906,
      "logps/rejected": -358.9510803222656,
      "loss": 0.3471,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -4.941549777984619,
      "rewards/margins": 2.4135260581970215,
      "rewards/rejected": -7.355075836181641,
      "step": 5260
    },
    {
      "epoch": 1.140366078197769,
      "grad_norm": 12.4375,
      "learning_rate": 1.1569465878912212e-05,
      "logits/chosen": -2.9802188873291016,
      "logits/rejected": -2.8779244422912598,
      "logps/chosen": -307.7441101074219,
      "logps/rejected": -362.3753356933594,
      "loss": 0.2351,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -4.302211761474609,
      "rewards/margins": 2.777215003967285,
      "rewards/rejected": -7.0794267654418945,
      "step": 5265
    },
    {
      "epoch": 1.141449149788801,
      "grad_norm": 23.75,
      "learning_rate": 1.156370260365939e-05,
      "logits/chosen": -3.0617001056671143,
      "logits/rejected": -2.905430316925049,
      "logps/chosen": -295.05767822265625,
      "logps/rejected": -340.69317626953125,
      "loss": 0.3767,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -4.605810165405273,
      "rewards/margins": 1.893982172012329,
      "rewards/rejected": -6.499792575836182,
      "step": 5270
    },
    {
      "epoch": 1.1425322213798332,
      "grad_norm": 21.75,
      "learning_rate": 1.1557935762196928e-05,
      "logits/chosen": -3.0227198600769043,
      "logits/rejected": -2.8559155464172363,
      "logps/chosen": -285.4443359375,
      "logps/rejected": -356.64642333984375,
      "loss": 0.2571,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -3.9075851440429688,
      "rewards/margins": 2.5006356239318848,
      "rewards/rejected": -6.4082207679748535,
      "step": 5275
    },
    {
      "epoch": 1.1436152929708654,
      "grad_norm": 19.625,
      "learning_rate": 1.1552165362747178e-05,
      "logits/chosen": -2.8952081203460693,
      "logits/rejected": -2.9223458766937256,
      "logps/chosen": -313.5982971191406,
      "logps/rejected": -355.5468444824219,
      "loss": 0.2553,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.121158123016357,
      "rewards/margins": 2.69421648979187,
      "rewards/rejected": -6.81537389755249,
      "step": 5280
    },
    {
      "epoch": 1.1446983645618976,
      "grad_norm": 19.875,
      "learning_rate": 1.1546391413537559e-05,
      "logits/chosen": -3.006455183029175,
      "logits/rejected": -2.9599080085754395,
      "logps/chosen": -325.5584716796875,
      "logps/rejected": -369.7048645019531,
      "loss": 0.3215,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.160589218139648,
      "rewards/margins": 2.3124489784240723,
      "rewards/rejected": -7.4730377197265625,
      "step": 5285
    },
    {
      "epoch": 1.1457814361529297,
      "grad_norm": 22.25,
      "learning_rate": 1.1540613922800563e-05,
      "logits/chosen": -3.0857372283935547,
      "logits/rejected": -2.9748024940490723,
      "logps/chosen": -318.0557556152344,
      "logps/rejected": -373.3168029785156,
      "loss": 0.2867,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.983092308044434,
      "rewards/margins": 2.615866184234619,
      "rewards/rejected": -7.598957061767578,
      "step": 5290
    },
    {
      "epoch": 1.1468645077439619,
      "grad_norm": 16.5,
      "learning_rate": 1.153483289877372e-05,
      "logits/chosen": -3.006498098373413,
      "logits/rejected": -2.974785566329956,
      "logps/chosen": -312.24664306640625,
      "logps/rejected": -366.6042785644531,
      "loss": 0.237,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.960713863372803,
      "rewards/margins": 2.6812949180603027,
      "rewards/rejected": -7.642009735107422,
      "step": 5295
    },
    {
      "epoch": 1.147947579334994,
      "grad_norm": 16.25,
      "learning_rate": 1.1529048349699603e-05,
      "logits/chosen": -2.8746631145477295,
      "logits/rejected": -2.8956782817840576,
      "logps/chosen": -330.98944091796875,
      "logps/rejected": -376.8467102050781,
      "loss": 0.3049,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.1800971031188965,
      "rewards/margins": 2.3564066886901855,
      "rewards/rejected": -7.536503791809082,
      "step": 5300
    },
    {
      "epoch": 1.1490306509260262,
      "grad_norm": 25.125,
      "learning_rate": 1.1523260283825812e-05,
      "logits/chosen": -3.0967235565185547,
      "logits/rejected": -3.1356875896453857,
      "logps/chosen": -305.1015625,
      "logps/rejected": -346.9869384765625,
      "loss": 0.2872,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.8840460777282715,
      "rewards/margins": 2.56093168258667,
      "rewards/rejected": -7.444977760314941,
      "step": 5305
    },
    {
      "epoch": 1.1501137225170583,
      "grad_norm": 19.75,
      "learning_rate": 1.1517468709404956e-05,
      "logits/chosen": -3.0625548362731934,
      "logits/rejected": -3.051947832107544,
      "logps/chosen": -301.66888427734375,
      "logps/rejected": -346.9801940917969,
      "loss": 0.2532,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.690339088439941,
      "rewards/margins": 2.743130683898926,
      "rewards/rejected": -7.433469295501709,
      "step": 5310
    },
    {
      "epoch": 1.1511967941080905,
      "grad_norm": 18.875,
      "learning_rate": 1.1511673634694654e-05,
      "logits/chosen": -3.0007145404815674,
      "logits/rejected": -2.9524407386779785,
      "logps/chosen": -326.64495849609375,
      "logps/rejected": -378.7975769042969,
      "loss": 0.2812,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.424916744232178,
      "rewards/margins": 2.4220285415649414,
      "rewards/rejected": -7.846945762634277,
      "step": 5315
    },
    {
      "epoch": 1.1522798656991227,
      "grad_norm": 15.75,
      "learning_rate": 1.1505875067957512e-05,
      "logits/chosen": -3.1419291496276855,
      "logits/rejected": -3.1027414798736572,
      "logps/chosen": -310.489013671875,
      "logps/rejected": -353.13433837890625,
      "loss": 0.2772,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.14328145980835,
      "rewards/margins": 2.453856945037842,
      "rewards/rejected": -7.597138404846191,
      "step": 5320
    },
    {
      "epoch": 1.1533629372901548,
      "grad_norm": 21.625,
      "learning_rate": 1.1500073017461111e-05,
      "logits/chosen": -3.046548843383789,
      "logits/rejected": -2.9795851707458496,
      "logps/chosen": -310.34674072265625,
      "logps/rejected": -384.6014709472656,
      "loss": 0.3006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.933616638183594,
      "rewards/margins": 2.816103458404541,
      "rewards/rejected": -7.749720096588135,
      "step": 5325
    },
    {
      "epoch": 1.154446008881187,
      "grad_norm": 30.375,
      "learning_rate": 1.1494267491478007e-05,
      "logits/chosen": -2.8674519062042236,
      "logits/rejected": -2.8449227809906006,
      "logps/chosen": -329.46942138671875,
      "logps/rejected": -368.8901062011719,
      "loss": 0.267,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.564370155334473,
      "rewards/margins": 2.674287796020508,
      "rewards/rejected": -7.238657474517822,
      "step": 5330
    },
    {
      "epoch": 1.1555290804722191,
      "grad_norm": 27.875,
      "learning_rate": 1.1488458498285707e-05,
      "logits/chosen": -3.0214812755584717,
      "logits/rejected": -3.108987331390381,
      "logps/chosen": -338.6521911621094,
      "logps/rejected": -358.90789794921875,
      "loss": 0.3345,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.092566013336182,
      "rewards/margins": 2.1202564239501953,
      "rewards/rejected": -7.212821960449219,
      "step": 5335
    },
    {
      "epoch": 1.1566121520632513,
      "grad_norm": 20.625,
      "learning_rate": 1.1482646046166658e-05,
      "logits/chosen": -2.952584743499756,
      "logits/rejected": -2.8713583946228027,
      "logps/chosen": -330.6754150390625,
      "logps/rejected": -379.182861328125,
      "loss": 0.2737,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -4.821787357330322,
      "rewards/margins": 2.8346617221832275,
      "rewards/rejected": -7.656449794769287,
      "step": 5340
    },
    {
      "epoch": 1.1576952236542835,
      "grad_norm": 15.5625,
      "learning_rate": 1.1476830143408245e-05,
      "logits/chosen": -2.9374566078186035,
      "logits/rejected": -2.9809229373931885,
      "logps/chosen": -333.97894287109375,
      "logps/rejected": -377.7020263671875,
      "loss": 0.2047,
      "rewards/accuracies": 0.9666666984558105,
      "rewards/chosen": -4.818951606750488,
      "rewards/margins": 2.7981386184692383,
      "rewards/rejected": -7.617089748382568,
      "step": 5345
    },
    {
      "epoch": 1.1587782952453156,
      "grad_norm": 22.75,
      "learning_rate": 1.1471010798302772e-05,
      "logits/chosen": -2.913024425506592,
      "logits/rejected": -2.857499599456787,
      "logps/chosen": -313.3147277832031,
      "logps/rejected": -361.52789306640625,
      "loss": 0.2821,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.1384596824646,
      "rewards/margins": 2.6930644512176514,
      "rewards/rejected": -6.831523895263672,
      "step": 5350
    },
    {
      "epoch": 1.1598613668363478,
      "grad_norm": 24.0,
      "learning_rate": 1.1465188019147448e-05,
      "logits/chosen": -2.806962013244629,
      "logits/rejected": -2.8128042221069336,
      "logps/chosen": -301.88153076171875,
      "logps/rejected": -359.8642272949219,
      "loss": 0.3008,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.43951940536499,
      "rewards/margins": 2.2937557697296143,
      "rewards/rejected": -6.733275413513184,
      "step": 5355
    },
    {
      "epoch": 1.16094443842738,
      "grad_norm": 25.875,
      "learning_rate": 1.1459361814244379e-05,
      "logits/chosen": -2.921674966812134,
      "logits/rejected": -2.9407358169555664,
      "logps/chosen": -323.6402282714844,
      "logps/rejected": -360.4979553222656,
      "loss": 0.2711,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.601588249206543,
      "rewards/margins": 2.845003843307495,
      "rewards/rejected": -7.446592807769775,
      "step": 5360
    },
    {
      "epoch": 1.1620275100184123,
      "grad_norm": 17.125,
      "learning_rate": 1.1453532191900558e-05,
      "logits/chosen": -2.9033379554748535,
      "logits/rejected": -2.9265198707580566,
      "logps/chosen": -323.7586975097656,
      "logps/rejected": -370.90966796875,
      "loss": 0.2607,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.762800693511963,
      "rewards/margins": 2.8570330142974854,
      "rewards/rejected": -7.619833946228027,
      "step": 5365
    },
    {
      "epoch": 1.1631105816094445,
      "grad_norm": 17.5,
      "learning_rate": 1.1447699160427847e-05,
      "logits/chosen": -2.918327808380127,
      "logits/rejected": -2.951490640640259,
      "logps/chosen": -330.8597412109375,
      "logps/rejected": -382.0935363769531,
      "loss": 0.2182,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.087030410766602,
      "rewards/margins": 3.0031559467315674,
      "rewards/rejected": -8.090187072753906,
      "step": 5370
    },
    {
      "epoch": 1.1641936532004766,
      "grad_norm": 18.5,
      "learning_rate": 1.1441862728142972e-05,
      "logits/chosen": -2.885468006134033,
      "logits/rejected": -2.848456382751465,
      "logps/chosen": -340.20050048828125,
      "logps/rejected": -380.2904357910156,
      "loss": 0.2569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.096328258514404,
      "rewards/margins": 2.6620709896087646,
      "rewards/rejected": -7.75839900970459,
      "step": 5375
    },
    {
      "epoch": 1.1652767247915088,
      "grad_norm": 17.375,
      "learning_rate": 1.1436022903367505e-05,
      "logits/chosen": -3.020047903060913,
      "logits/rejected": -2.9201793670654297,
      "logps/chosen": -320.3584289550781,
      "logps/rejected": -360.7699279785156,
      "loss": 0.2681,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.823559761047363,
      "rewards/margins": 2.578702688217163,
      "rewards/rejected": -7.402262210845947,
      "step": 5380
    },
    {
      "epoch": 1.166359796382541,
      "grad_norm": 28.0,
      "learning_rate": 1.1430179694427858e-05,
      "logits/chosen": -2.9410171508789062,
      "logits/rejected": -2.9440746307373047,
      "logps/chosen": -313.79034423828125,
      "logps/rejected": -361.2711181640625,
      "loss": 0.2886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.992279052734375,
      "rewards/margins": 2.641594886779785,
      "rewards/rejected": -7.633874416351318,
      "step": 5385
    },
    {
      "epoch": 1.167442867973573,
      "grad_norm": 27.5,
      "learning_rate": 1.1424333109655264e-05,
      "logits/chosen": -2.9567553997039795,
      "logits/rejected": -2.900160789489746,
      "logps/chosen": -341.37506103515625,
      "logps/rejected": -393.0235595703125,
      "loss": 0.2718,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.377872467041016,
      "rewards/margins": 2.5999629497528076,
      "rewards/rejected": -7.977835655212402,
      "step": 5390
    },
    {
      "epoch": 1.1685259395646053,
      "grad_norm": 21.0,
      "learning_rate": 1.1418483157385777e-05,
      "logits/chosen": -2.9131650924682617,
      "logits/rejected": -2.886038303375244,
      "logps/chosen": -341.2619323730469,
      "logps/rejected": -383.41790771484375,
      "loss": 0.3403,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.098568439483643,
      "rewards/margins": 2.706672191619873,
      "rewards/rejected": -7.805241584777832,
      "step": 5395
    },
    {
      "epoch": 1.1696090111556374,
      "grad_norm": 9.5,
      "learning_rate": 1.141262984596024e-05,
      "logits/chosen": -2.880826950073242,
      "logits/rejected": -2.8149924278259277,
      "logps/chosen": -312.03314208984375,
      "logps/rejected": -363.06463623046875,
      "loss": 0.2728,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.335701942443848,
      "rewards/margins": 2.6459782123565674,
      "rewards/rejected": -6.981679439544678,
      "step": 5400
    },
    {
      "epoch": 1.1706920827466696,
      "grad_norm": 24.25,
      "learning_rate": 1.14067731837243e-05,
      "logits/chosen": -2.8456084728240967,
      "logits/rejected": -2.9532272815704346,
      "logps/chosen": -306.45391845703125,
      "logps/rejected": -340.8958435058594,
      "loss": 0.3399,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.717177391052246,
      "rewards/margins": 2.3803486824035645,
      "rewards/rejected": -7.097525596618652,
      "step": 5405
    },
    {
      "epoch": 1.1717751543377017,
      "grad_norm": 28.125,
      "learning_rate": 1.1400913179028367e-05,
      "logits/chosen": -2.7453770637512207,
      "logits/rejected": -2.7653980255126953,
      "logps/chosen": -314.2389221191406,
      "logps/rejected": -357.98687744140625,
      "loss": 0.2319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9723281860351562,
      "rewards/margins": 2.9033188819885254,
      "rewards/rejected": -6.875647068023682,
      "step": 5410
    },
    {
      "epoch": 1.172858225928734,
      "grad_norm": 28.625,
      "learning_rate": 1.1395049840227631e-05,
      "logits/chosen": -2.839628219604492,
      "logits/rejected": -2.857290744781494,
      "logps/chosen": -308.24798583984375,
      "logps/rejected": -349.9514465332031,
      "loss": 0.2392,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -4.236536026000977,
      "rewards/margins": 2.649643898010254,
      "rewards/rejected": -6.886179447174072,
      "step": 5415
    },
    {
      "epoch": 1.173941297519766,
      "grad_norm": 23.0,
      "learning_rate": 1.1389183175682022e-05,
      "logits/chosen": -2.8655295372009277,
      "logits/rejected": -2.852989912033081,
      "logps/chosen": -310.88336181640625,
      "logps/rejected": -350.5017395019531,
      "loss": 0.2695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.581418991088867,
      "rewards/margins": 2.521850109100342,
      "rewards/rejected": -7.103269100189209,
      "step": 5420
    },
    {
      "epoch": 1.1750243691107982,
      "grad_norm": 17.625,
      "learning_rate": 1.1383313193756224e-05,
      "logits/chosen": -2.809936046600342,
      "logits/rejected": -2.757683277130127,
      "logps/chosen": -299.8343811035156,
      "logps/rejected": -354.88836669921875,
      "loss": 0.2443,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.527406692504883,
      "rewards/margins": 2.516167163848877,
      "rewards/rejected": -7.04357385635376,
      "step": 5425
    },
    {
      "epoch": 1.1761074407018304,
      "grad_norm": 19.625,
      "learning_rate": 1.1377439902819642e-05,
      "logits/chosen": -2.6945853233337402,
      "logits/rejected": -2.7290892601013184,
      "logps/chosen": -315.0670166015625,
      "logps/rejected": -368.22406005859375,
      "loss": 0.2344,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.873349189758301,
      "rewards/margins": 2.844116687774658,
      "rewards/rejected": -7.717465877532959,
      "step": 5430
    },
    {
      "epoch": 1.1771905122928625,
      "grad_norm": 22.375,
      "learning_rate": 1.1371563311246404e-05,
      "logits/chosen": -2.8593192100524902,
      "logits/rejected": -2.8325796127319336,
      "logps/chosen": -304.3038024902344,
      "logps/rejected": -336.5018615722656,
      "loss": 0.3465,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -4.3054680824279785,
      "rewards/margins": 2.2770943641662598,
      "rewards/rejected": -6.582562446594238,
      "step": 5435
    },
    {
      "epoch": 1.1782735838838947,
      "grad_norm": 15.6875,
      "learning_rate": 1.1365683427415343e-05,
      "logits/chosen": -2.9897663593292236,
      "logits/rejected": -2.9704155921936035,
      "logps/chosen": -302.6545104980469,
      "logps/rejected": -346.9750061035156,
      "loss": 0.2233,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -4.778778076171875,
      "rewards/margins": 2.650946617126465,
      "rewards/rejected": -7.42972469329834,
      "step": 5440
    },
    {
      "epoch": 1.1793566554749269,
      "grad_norm": 16.125,
      "learning_rate": 1.1359800259709982e-05,
      "logits/chosen": -3.0201001167297363,
      "logits/rejected": -2.8827993869781494,
      "logps/chosen": -314.4952087402344,
      "logps/rejected": -360.69256591796875,
      "loss": 0.2994,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.237941265106201,
      "rewards/margins": 2.5942606925964355,
      "rewards/rejected": -7.832201957702637,
      "step": 5445
    },
    {
      "epoch": 1.180439727065959,
      "grad_norm": 27.25,
      "learning_rate": 1.1353913816518533e-05,
      "logits/chosen": -2.7104175090789795,
      "logits/rejected": -2.7049365043640137,
      "logps/chosen": -320.6121520996094,
      "logps/rejected": -369.0080871582031,
      "loss": 0.27,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.923180103302002,
      "rewards/margins": 2.532107353210449,
      "rewards/rejected": -7.455286979675293,
      "step": 5450
    },
    {
      "epoch": 1.1815227986569912,
      "grad_norm": 19.25,
      "learning_rate": 1.1348024106233877e-05,
      "logits/chosen": -2.8369784355163574,
      "logits/rejected": -2.8396096229553223,
      "logps/chosen": -336.0149841308594,
      "logps/rejected": -369.4625244140625,
      "loss": 0.2963,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.9410505294799805,
      "rewards/margins": 2.3933968544006348,
      "rewards/rejected": -7.334447383880615,
      "step": 5455
    },
    {
      "epoch": 1.1826058702480233,
      "grad_norm": 13.625,
      "learning_rate": 1.1342131137253545e-05,
      "logits/chosen": -2.719464063644409,
      "logits/rejected": -2.730768918991089,
      "logps/chosen": -309.7929992675781,
      "logps/rejected": -362.4224548339844,
      "loss": 0.2497,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.174858093261719,
      "rewards/margins": 2.932610273361206,
      "rewards/rejected": -7.107468605041504,
      "step": 5460
    },
    {
      "epoch": 1.1836889418390555,
      "grad_norm": 20.625,
      "learning_rate": 1.1336234917979724e-05,
      "logits/chosen": -2.8230838775634766,
      "logits/rejected": -2.833725690841675,
      "logps/chosen": -330.9817810058594,
      "logps/rejected": -361.52716064453125,
      "loss": 0.3286,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.385636329650879,
      "rewards/margins": 2.330066442489624,
      "rewards/rejected": -7.71570348739624,
      "step": 5465
    },
    {
      "epoch": 1.1847720134300876,
      "grad_norm": 26.75,
      "learning_rate": 1.1330335456819232e-05,
      "logits/chosen": -2.902616500854492,
      "logits/rejected": -2.960984945297241,
      "logps/chosen": -330.39605712890625,
      "logps/rejected": -372.9693603515625,
      "loss": 0.2758,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.198241710662842,
      "rewards/margins": 2.667201042175293,
      "rewards/rejected": -7.865444183349609,
      "step": 5470
    },
    {
      "epoch": 1.1858550850211198,
      "grad_norm": 24.5,
      "learning_rate": 1.1324432762183508e-05,
      "logits/chosen": -2.9198901653289795,
      "logits/rejected": -2.8549675941467285,
      "logps/chosen": -306.70660400390625,
      "logps/rejected": -371.6899719238281,
      "loss": 0.2906,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -4.750134468078613,
      "rewards/margins": 2.786588430404663,
      "rewards/rejected": -7.536722660064697,
      "step": 5475
    },
    {
      "epoch": 1.1869381566121522,
      "grad_norm": 16.25,
      "learning_rate": 1.13185268424886e-05,
      "logits/chosen": -2.8030970096588135,
      "logits/rejected": -2.7689261436462402,
      "logps/chosen": -334.2020568847656,
      "logps/rejected": -352.9264221191406,
      "loss": 0.3043,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.87109899520874,
      "rewards/margins": 2.200471878051758,
      "rewards/rejected": -7.071571350097656,
      "step": 5480
    },
    {
      "epoch": 1.1880212282031843,
      "grad_norm": 21.75,
      "learning_rate": 1.131261770615516e-05,
      "logits/chosen": -2.651186943054199,
      "logits/rejected": -2.7190451622009277,
      "logps/chosen": -333.97564697265625,
      "logps/rejected": -390.098876953125,
      "loss": 0.2391,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.143041610717773,
      "rewards/margins": 2.6562273502349854,
      "rewards/rejected": -7.7992682456970215,
      "step": 5485
    },
    {
      "epoch": 1.1891042997942165,
      "grad_norm": 18.875,
      "learning_rate": 1.1306705361608421e-05,
      "logits/chosen": -2.7548320293426514,
      "logits/rejected": -2.7743473052978516,
      "logps/chosen": -327.77996826171875,
      "logps/rejected": -367.03411865234375,
      "loss": 0.2832,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.08829927444458,
      "rewards/margins": 2.425875186920166,
      "rewards/rejected": -7.514174461364746,
      "step": 5490
    },
    {
      "epoch": 1.1901873713852487,
      "grad_norm": 13.5625,
      "learning_rate": 1.1300789817278193e-05,
      "logits/chosen": -2.82240629196167,
      "logits/rejected": -2.7647624015808105,
      "logps/chosen": -309.332275390625,
      "logps/rejected": -358.1824645996094,
      "loss": 0.2614,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.667954444885254,
      "rewards/margins": 2.3639588356018066,
      "rewards/rejected": -7.031912803649902,
      "step": 5495
    },
    {
      "epoch": 1.1912704429762808,
      "grad_norm": 22.625,
      "learning_rate": 1.1294871081598844e-05,
      "logits/chosen": -2.7854502201080322,
      "logits/rejected": -2.864598035812378,
      "logps/chosen": -321.42889404296875,
      "logps/rejected": -380.06353759765625,
      "loss": 0.2842,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -4.902697563171387,
      "rewards/margins": 2.5830631256103516,
      "rewards/rejected": -7.4857611656188965,
      "step": 5500
    },
    {
      "epoch": 1.1912704429762808,
      "eval_logits/chosen": -2.920480251312256,
      "eval_logits/rejected": -2.8478851318359375,
      "eval_logps/chosen": -303.5321960449219,
      "eval_logps/rejected": -345.245849609375,
      "eval_loss": 0.4320386052131653,
      "eval_rewards/accuracies": 0.8100000023841858,
      "eval_rewards/chosen": -4.466545581817627,
      "eval_rewards/margins": 1.9128895998001099,
      "eval_rewards/rejected": -6.3794355392456055,
      "eval_runtime": 14.415,
      "eval_samples_per_second": 13.874,
      "eval_steps_per_second": 13.874,
      "step": 5500
    },
    {
      "epoch": 1.192353514567313,
      "grad_norm": 17.375,
      "learning_rate": 1.12889491630093e-05,
      "logits/chosen": -2.9167532920837402,
      "logits/rejected": -2.9165523052215576,
      "logps/chosen": -333.07537841796875,
      "logps/rejected": -369.79913330078125,
      "loss": 0.3001,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.549914360046387,
      "rewards/margins": 2.3391475677490234,
      "rewards/rejected": -7.889062404632568,
      "step": 5505
    },
    {
      "epoch": 1.1934365861583451,
      "grad_norm": 21.0,
      "learning_rate": 1.1283024069953016e-05,
      "logits/chosen": -2.673137664794922,
      "logits/rejected": -2.7820017337799072,
      "logps/chosen": -328.48712158203125,
      "logps/rejected": -361.35247802734375,
      "loss": 0.2758,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.745781421661377,
      "rewards/margins": 2.3199706077575684,
      "rewards/rejected": -7.065751552581787,
      "step": 5510
    },
    {
      "epoch": 1.1945196577493773,
      "grad_norm": 24.25,
      "learning_rate": 1.1277095810877982e-05,
      "logits/chosen": -2.892275810241699,
      "logits/rejected": -2.7924652099609375,
      "logps/chosen": -309.9503479003906,
      "logps/rejected": -368.9506530761719,
      "loss": 0.2849,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -4.6037797927856445,
      "rewards/margins": 2.5284857749938965,
      "rewards/rejected": -7.132265567779541,
      "step": 5515
    },
    {
      "epoch": 1.1956027293404095,
      "grad_norm": 14.875,
      "learning_rate": 1.1271164394236696e-05,
      "logits/chosen": -2.831387758255005,
      "logits/rejected": -2.6471800804138184,
      "logps/chosen": -324.0531311035156,
      "logps/rejected": -381.2491760253906,
      "loss": 0.2483,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.645190715789795,
      "rewards/margins": 2.730055570602417,
      "rewards/rejected": -7.375245571136475,
      "step": 5520
    },
    {
      "epoch": 1.1966858009314416,
      "grad_norm": 32.5,
      "learning_rate": 1.1265229828486161e-05,
      "logits/chosen": -2.9058291912078857,
      "logits/rejected": -2.7981197834014893,
      "logps/chosen": -322.2016296386719,
      "logps/rejected": -365.9845886230469,
      "loss": 0.2755,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.255465984344482,
      "rewards/margins": 2.4291844367980957,
      "rewards/rejected": -7.684649467468262,
      "step": 5525
    },
    {
      "epoch": 1.1977688725224738,
      "grad_norm": 17.0,
      "learning_rate": 1.1259292122087865e-05,
      "logits/chosen": -3.064091205596924,
      "logits/rejected": -2.9423511028289795,
      "logps/chosen": -322.0274963378906,
      "logps/rejected": -384.5496520996094,
      "loss": 0.2192,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.221628189086914,
      "rewards/margins": 2.8469090461730957,
      "rewards/rejected": -8.068536758422852,
      "step": 5530
    },
    {
      "epoch": 1.198851944113506,
      "grad_norm": 31.875,
      "learning_rate": 1.1253351283507781e-05,
      "logits/chosen": -2.927833080291748,
      "logits/rejected": -2.9325602054595947,
      "logps/chosen": -343.3148498535156,
      "logps/rejected": -371.60101318359375,
      "loss": 0.3486,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.413395881652832,
      "rewards/margins": 2.273406505584717,
      "rewards/rejected": -7.686801910400391,
      "step": 5535
    },
    {
      "epoch": 1.199935015704538,
      "grad_norm": 32.75,
      "learning_rate": 1.1247407321216346e-05,
      "logits/chosen": -2.9667463302612305,
      "logits/rejected": -2.96681547164917,
      "logps/chosen": -335.94952392578125,
      "logps/rejected": -382.74334716796875,
      "loss": 0.3013,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.51339054107666,
      "rewards/margins": 2.6603832244873047,
      "rewards/rejected": -8.173773765563965,
      "step": 5540
    },
    {
      "epoch": 1.2010180872955702,
      "grad_norm": 16.875,
      "learning_rate": 1.1241460243688448e-05,
      "logits/chosen": -2.867199659347534,
      "logits/rejected": -2.908792495727539,
      "logps/chosen": -319.82391357421875,
      "logps/rejected": -367.37872314453125,
      "loss": 0.2715,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.814332962036133,
      "rewards/margins": 2.596827983856201,
      "rewards/rejected": -7.411160945892334,
      "step": 5545
    },
    {
      "epoch": 1.2021011588866024,
      "grad_norm": 29.0,
      "learning_rate": 1.1235510059403417e-05,
      "logits/chosen": -2.9524881839752197,
      "logits/rejected": -2.9857394695281982,
      "logps/chosen": -324.26129150390625,
      "logps/rejected": -363.54254150390625,
      "loss": 0.3122,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.032627582550049,
      "rewards/margins": 2.3991103172302246,
      "rewards/rejected": -7.431738376617432,
      "step": 5550
    },
    {
      "epoch": 1.2031842304776346,
      "grad_norm": 17.375,
      "learning_rate": 1.1229556776845014e-05,
      "logits/chosen": -3.070835828781128,
      "logits/rejected": -3.1740610599517822,
      "logps/chosen": -319.2527160644531,
      "logps/rejected": -355.9009094238281,
      "loss": 0.261,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.213276386260986,
      "rewards/margins": 2.708881139755249,
      "rewards/rejected": -7.922157287597656,
      "step": 5555
    },
    {
      "epoch": 1.2042673020686667,
      "grad_norm": 20.5,
      "learning_rate": 1.1223600404501418e-05,
      "logits/chosen": -3.095405101776123,
      "logits/rejected": -3.0353243350982666,
      "logps/chosen": -308.9761657714844,
      "logps/rejected": -356.8370361328125,
      "loss": 0.208,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.7469892501831055,
      "rewards/margins": 2.6621763706207275,
      "rewards/rejected": -7.409165382385254,
      "step": 5560
    },
    {
      "epoch": 1.2053503736596989,
      "grad_norm": 26.375,
      "learning_rate": 1.1217640950865214e-05,
      "logits/chosen": -2.921715021133423,
      "logits/rejected": -2.901141405105591,
      "logps/chosen": -329.7908630371094,
      "logps/rejected": -359.5177307128906,
      "loss": 0.3448,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.198519229888916,
      "rewards/margins": 2.1215968132019043,
      "rewards/rejected": -7.320115566253662,
      "step": 5565
    },
    {
      "epoch": 1.206433445250731,
      "grad_norm": 25.125,
      "learning_rate": 1.1211678424433378e-05,
      "logits/chosen": -3.0555450916290283,
      "logits/rejected": -2.9317820072174072,
      "logps/chosen": -316.43548583984375,
      "logps/rejected": -376.3172912597656,
      "loss": 0.2362,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.385037422180176,
      "rewards/margins": 2.838263988494873,
      "rewards/rejected": -8.223302841186523,
      "step": 5570
    },
    {
      "epoch": 1.2075165168417632,
      "grad_norm": 29.5,
      "learning_rate": 1.1205712833707267e-05,
      "logits/chosen": -3.1788041591644287,
      "logits/rejected": -3.054051160812378,
      "logps/chosen": -303.30487060546875,
      "logps/rejected": -352.6898193359375,
      "loss": 0.2451,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.702012062072754,
      "rewards/margins": 2.920327663421631,
      "rewards/rejected": -7.622338771820068,
      "step": 5575
    },
    {
      "epoch": 1.2085995884327954,
      "grad_norm": 14.1875,
      "learning_rate": 1.119974418719261e-05,
      "logits/chosen": -2.915924549102783,
      "logits/rejected": -2.800495147705078,
      "logps/chosen": -317.6298828125,
      "logps/rejected": -372.6432189941406,
      "loss": 0.2761,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.99164342880249,
      "rewards/margins": 2.7336373329162598,
      "rewards/rejected": -7.72528076171875,
      "step": 5580
    },
    {
      "epoch": 1.2096826600238275,
      "grad_norm": 13.6875,
      "learning_rate": 1.119377249339949e-05,
      "logits/chosen": -2.9874155521392822,
      "logits/rejected": -2.9420485496520996,
      "logps/chosen": -306.9132080078125,
      "logps/rejected": -362.4388122558594,
      "loss": 0.292,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.966670989990234,
      "rewards/margins": 2.599680185317993,
      "rewards/rejected": -7.566351413726807,
      "step": 5585
    },
    {
      "epoch": 1.2107657316148597,
      "grad_norm": 38.5,
      "learning_rate": 1.1187797760842337e-05,
      "logits/chosen": -3.279672145843506,
      "logits/rejected": -3.1006264686584473,
      "logps/chosen": -309.078857421875,
      "logps/rejected": -378.21148681640625,
      "loss": 0.2853,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.686314105987549,
      "rewards/margins": 2.6358706951141357,
      "rewards/rejected": -8.322184562683105,
      "step": 5590
    },
    {
      "epoch": 1.2118488032058918,
      "grad_norm": 23.625,
      "learning_rate": 1.1181819998039911e-05,
      "logits/chosen": -2.9429993629455566,
      "logits/rejected": -2.9987587928771973,
      "logps/chosen": -326.060791015625,
      "logps/rejected": -373.5211486816406,
      "loss": 0.35,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.329251289367676,
      "rewards/margins": 2.4928581714630127,
      "rewards/rejected": -7.822110176086426,
      "step": 5595
    },
    {
      "epoch": 1.212931874796924,
      "grad_norm": 17.875,
      "learning_rate": 1.1175839213515297e-05,
      "logits/chosen": -2.8231072425842285,
      "logits/rejected": -2.9349260330200195,
      "logps/chosen": -325.45648193359375,
      "logps/rejected": -372.9749450683594,
      "loss": 0.2478,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.259548664093018,
      "rewards/margins": 3.0960419178009033,
      "rewards/rejected": -7.3555908203125,
      "step": 5600
    },
    {
      "epoch": 1.2140149463879562,
      "grad_norm": 14.0625,
      "learning_rate": 1.1169855415795882e-05,
      "logits/chosen": -2.902475595474243,
      "logits/rejected": -3.0087292194366455,
      "logps/chosen": -341.58868408203125,
      "logps/rejected": -364.7960510253906,
      "loss": 0.295,
      "rewards/accuracies": 0.8916667699813843,
      "rewards/chosen": -4.56096076965332,
      "rewards/margins": 2.4828219413757324,
      "rewards/rejected": -7.043783664703369,
      "step": 5605
    },
    {
      "epoch": 1.2150980179789883,
      "grad_norm": 22.25,
      "learning_rate": 1.1163868613413355e-05,
      "logits/chosen": -2.9173474311828613,
      "logits/rejected": -2.8826985359191895,
      "logps/chosen": -322.7054443359375,
      "logps/rejected": -366.1441650390625,
      "loss": 0.2915,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.633702754974365,
      "rewards/margins": 2.5375962257385254,
      "rewards/rejected": -7.171298980712891,
      "step": 5610
    },
    {
      "epoch": 1.2161810895700205,
      "grad_norm": 14.6875,
      "learning_rate": 1.1157878814903685e-05,
      "logits/chosen": -3.0869851112365723,
      "logits/rejected": -3.031777858734131,
      "logps/chosen": -306.411865234375,
      "logps/rejected": -353.49151611328125,
      "loss": 0.236,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.882421016693115,
      "rewards/margins": 2.669191837310791,
      "rewards/rejected": -7.551612854003906,
      "step": 5615
    },
    {
      "epoch": 1.2172641611610526,
      "grad_norm": 25.125,
      "learning_rate": 1.1151886028807119e-05,
      "logits/chosen": -2.879128932952881,
      "logits/rejected": -2.8309624195098877,
      "logps/chosen": -339.2496643066406,
      "logps/rejected": -380.70721435546875,
      "loss": 0.3105,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.704929351806641,
      "rewards/margins": 2.310060977935791,
      "rewards/rejected": -8.014989852905273,
      "step": 5620
    },
    {
      "epoch": 1.218347232752085,
      "grad_norm": 14.0625,
      "learning_rate": 1.1145890263668153e-05,
      "logits/chosen": -2.9012041091918945,
      "logits/rejected": -2.9262819290161133,
      "logps/chosen": -314.0087890625,
      "logps/rejected": -357.82977294921875,
      "loss": 0.2409,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.826700687408447,
      "rewards/margins": 2.5857138633728027,
      "rewards/rejected": -7.412413120269775,
      "step": 5625
    },
    {
      "epoch": 1.2194303043431172,
      "grad_norm": 26.5,
      "learning_rate": 1.1139891528035542e-05,
      "logits/chosen": -2.8453445434570312,
      "logits/rejected": -2.864506959915161,
      "logps/chosen": -325.62701416015625,
      "logps/rejected": -360.1942443847656,
      "loss": 0.3398,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.016104221343994,
      "rewards/margins": 2.5516905784606934,
      "rewards/rejected": -7.5677947998046875,
      "step": 5630
    },
    {
      "epoch": 1.2205133759341493,
      "grad_norm": 19.125,
      "learning_rate": 1.1133889830462269e-05,
      "logits/chosen": -2.9722588062286377,
      "logits/rejected": -2.973005533218384,
      "logps/chosen": -311.5088806152344,
      "logps/rejected": -352.54193115234375,
      "loss": 0.2691,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.631992340087891,
      "rewards/margins": 2.519477367401123,
      "rewards/rejected": -7.1514692306518555,
      "step": 5635
    },
    {
      "epoch": 1.2215964475251815,
      "grad_norm": 20.0,
      "learning_rate": 1.1127885179505544e-05,
      "logits/chosen": -3.0197908878326416,
      "logits/rejected": -2.971198797225952,
      "logps/chosen": -312.25689697265625,
      "logps/rejected": -368.4163513183594,
      "loss": 0.2619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.799557685852051,
      "rewards/margins": 2.844108819961548,
      "rewards/rejected": -7.6436662673950195,
      "step": 5640
    },
    {
      "epoch": 1.2226795191162136,
      "grad_norm": 18.875,
      "learning_rate": 1.1121877583726784e-05,
      "logits/chosen": -2.8629708290100098,
      "logits/rejected": -2.8590989112854004,
      "logps/chosen": -319.55657958984375,
      "logps/rejected": -355.053466796875,
      "loss": 0.3393,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.80111837387085,
      "rewards/margins": 2.2818775177001953,
      "rewards/rejected": -7.082995414733887,
      "step": 5645
    },
    {
      "epoch": 1.2237625907072458,
      "grad_norm": 15.9375,
      "learning_rate": 1.1115867051691608e-05,
      "logits/chosen": -2.8788726329803467,
      "logits/rejected": -2.8535239696502686,
      "logps/chosen": -339.25604248046875,
      "logps/rejected": -377.3362121582031,
      "loss": 0.315,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.505650520324707,
      "rewards/margins": 2.660068988800049,
      "rewards/rejected": -8.165718078613281,
      "step": 5650
    },
    {
      "epoch": 1.224845662298278,
      "grad_norm": 22.875,
      "learning_rate": 1.110985359196982e-05,
      "logits/chosen": -2.684196710586548,
      "logits/rejected": -2.7570483684539795,
      "logps/chosen": -319.1610107421875,
      "logps/rejected": -357.5515441894531,
      "loss": 0.323,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.328387260437012,
      "rewards/margins": 2.564757823944092,
      "rewards/rejected": -6.8931450843811035,
      "step": 5655
    },
    {
      "epoch": 1.2259287338893101,
      "grad_norm": 17.0,
      "learning_rate": 1.1103837213135402e-05,
      "logits/chosen": -2.750816822052002,
      "logits/rejected": -2.762791633605957,
      "logps/chosen": -307.9979248046875,
      "logps/rejected": -340.2948303222656,
      "loss": 0.3237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.199837684631348,
      "rewards/margins": 2.2009923458099365,
      "rewards/rejected": -6.400830268859863,
      "step": 5660
    },
    {
      "epoch": 1.2270118054803423,
      "grad_norm": 15.75,
      "learning_rate": 1.109781792376649e-05,
      "logits/chosen": -2.812495231628418,
      "logits/rejected": -2.6204590797424316,
      "logps/chosen": -308.58587646484375,
      "logps/rejected": -366.861083984375,
      "loss": 0.2262,
      "rewards/accuracies": 0.9416666030883789,
      "rewards/chosen": -3.8237621784210205,
      "rewards/margins": 2.8054888248443604,
      "rewards/rejected": -6.629251003265381,
      "step": 5665
    },
    {
      "epoch": 1.2280948770713744,
      "grad_norm": 17.125,
      "learning_rate": 1.1091795732445377e-05,
      "logits/chosen": -2.871924638748169,
      "logits/rejected": -2.9185028076171875,
      "logps/chosen": -310.59808349609375,
      "logps/rejected": -359.0186767578125,
      "loss": 0.2194,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -4.559218406677246,
      "rewards/margins": 2.6851117610931396,
      "rewards/rejected": -7.244329929351807,
      "step": 5670
    },
    {
      "epoch": 1.2291779486624066,
      "grad_norm": 18.25,
      "learning_rate": 1.108577064775849e-05,
      "logits/chosen": -2.8771440982818604,
      "logits/rejected": -2.958346128463745,
      "logps/chosen": -306.0693359375,
      "logps/rejected": -344.82489013671875,
      "loss": 0.2499,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.12489128112793,
      "rewards/margins": 2.606116533279419,
      "rewards/rejected": -6.7310075759887695,
      "step": 5675
    },
    {
      "epoch": 1.2302610202534388,
      "grad_norm": 21.5,
      "learning_rate": 1.1079742678296386e-05,
      "logits/chosen": -2.992905855178833,
      "logits/rejected": -2.9366345405578613,
      "logps/chosen": -312.6063537597656,
      "logps/rejected": -370.27386474609375,
      "loss": 0.253,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.780673503875732,
      "rewards/margins": 2.765998125076294,
      "rewards/rejected": -7.5466718673706055,
      "step": 5680
    },
    {
      "epoch": 1.231344091844471,
      "grad_norm": 11.1875,
      "learning_rate": 1.1073711832653727e-05,
      "logits/chosen": -3.093364715576172,
      "logits/rejected": -3.070919990539551,
      "logps/chosen": -319.67034912109375,
      "logps/rejected": -363.86541748046875,
      "loss": 0.2465,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.304293632507324,
      "rewards/margins": 2.652754545211792,
      "rewards/rejected": -7.9570488929748535,
      "step": 5685
    },
    {
      "epoch": 1.232427163435503,
      "grad_norm": 13.0,
      "learning_rate": 1.1067678119429285e-05,
      "logits/chosen": -2.8731884956359863,
      "logits/rejected": -2.7797341346740723,
      "logps/chosen": -336.4526672363281,
      "logps/rejected": -394.0983581542969,
      "loss": 0.2145,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.660905838012695,
      "rewards/margins": 3.0771965980529785,
      "rewards/rejected": -7.738102912902832,
      "step": 5690
    },
    {
      "epoch": 1.2335102350265352,
      "grad_norm": 24.75,
      "learning_rate": 1.1061641547225911e-05,
      "logits/chosen": -3.0624725818634033,
      "logits/rejected": -2.9033045768737793,
      "logps/chosen": -336.77557373046875,
      "logps/rejected": -381.4901123046875,
      "loss": 0.3479,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -5.54068660736084,
      "rewards/margins": 2.3895325660705566,
      "rewards/rejected": -7.930220127105713,
      "step": 5695
    },
    {
      "epoch": 1.2345933066175674,
      "grad_norm": 16.875,
      "learning_rate": 1.1055602124650544e-05,
      "logits/chosen": -3.1942124366760254,
      "logits/rejected": -3.1609654426574707,
      "logps/chosen": -325.01678466796875,
      "logps/rejected": -387.4505310058594,
      "loss": 0.2408,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.113271236419678,
      "rewards/margins": 2.9102120399475098,
      "rewards/rejected": -8.023483276367188,
      "step": 5700
    },
    {
      "epoch": 1.2356763782085995,
      "grad_norm": 14.6875,
      "learning_rate": 1.1049559860314174e-05,
      "logits/chosen": -3.2032783031463623,
      "logits/rejected": -3.1310842037200928,
      "logps/chosen": -320.2510986328125,
      "logps/rejected": -371.97015380859375,
      "loss": 0.2761,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.427734851837158,
      "rewards/margins": 2.725999355316162,
      "rewards/rejected": -8.15373420715332,
      "step": 5705
    },
    {
      "epoch": 1.2367594497996317,
      "grad_norm": 28.625,
      "learning_rate": 1.1043514762831855e-05,
      "logits/chosen": -3.147210121154785,
      "logits/rejected": -3.1793601512908936,
      "logps/chosen": -326.7197570800781,
      "logps/rejected": -371.3775634765625,
      "loss": 0.2921,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.497369766235352,
      "rewards/margins": 2.4769537448883057,
      "rewards/rejected": -7.974323272705078,
      "step": 5710
    },
    {
      "epoch": 1.2378425213906639,
      "grad_norm": 35.25,
      "learning_rate": 1.1037466840822672e-05,
      "logits/chosen": -3.1751210689544678,
      "logits/rejected": -3.000579833984375,
      "logps/chosen": -328.47509765625,
      "logps/rejected": -377.4613037109375,
      "loss": 0.3487,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.455537796020508,
      "rewards/margins": 2.328071355819702,
      "rewards/rejected": -7.783608913421631,
      "step": 5715
    },
    {
      "epoch": 1.238925592981696,
      "grad_norm": 16.25,
      "learning_rate": 1.1031416102909742e-05,
      "logits/chosen": -2.9700613021850586,
      "logits/rejected": -2.9545657634735107,
      "logps/chosen": -328.4125061035156,
      "logps/rejected": -368.2803649902344,
      "loss": 0.3025,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.7081522941589355,
      "rewards/margins": 2.3556830883026123,
      "rewards/rejected": -7.063835144042969,
      "step": 5720
    },
    {
      "epoch": 1.2400086645727282,
      "grad_norm": 15.625,
      "learning_rate": 1.102536255772019e-05,
      "logits/chosen": -3.1667978763580322,
      "logits/rejected": -3.1173088550567627,
      "logps/chosen": -303.1298828125,
      "logps/rejected": -347.6133728027344,
      "loss": 0.3213,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.0096964836120605,
      "rewards/margins": 2.199877977371216,
      "rewards/rejected": -7.209573268890381,
      "step": 5725
    },
    {
      "epoch": 1.2410917361637603,
      "grad_norm": 21.25,
      "learning_rate": 1.1019306213885155e-05,
      "logits/chosen": -2.979358673095703,
      "logits/rejected": -2.9728705883026123,
      "logps/chosen": -328.80548095703125,
      "logps/rejected": -371.2505798339844,
      "loss": 0.267,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.159667015075684,
      "rewards/margins": 2.559731960296631,
      "rewards/rejected": -7.719399452209473,
      "step": 5730
    },
    {
      "epoch": 1.2421748077547925,
      "grad_norm": 16.875,
      "learning_rate": 1.1013247080039755e-05,
      "logits/chosen": -3.0845377445220947,
      "logits/rejected": -3.0834763050079346,
      "logps/chosen": -341.6955871582031,
      "logps/rejected": -396.30279541015625,
      "loss": 0.2373,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -5.6686811447143555,
      "rewards/margins": 2.796114444732666,
      "rewards/rejected": -8.46479606628418,
      "step": 5735
    },
    {
      "epoch": 1.2432578793458249,
      "grad_norm": 10.5625,
      "learning_rate": 1.1007185164823092e-05,
      "logits/chosen": -3.09165620803833,
      "logits/rejected": -3.0186405181884766,
      "logps/chosen": -313.3529052734375,
      "logps/rejected": -380.88848876953125,
      "loss": 0.236,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -4.904000759124756,
      "rewards/margins": 2.9433836936950684,
      "rewards/rejected": -7.847384452819824,
      "step": 5740
    },
    {
      "epoch": 1.244340950936857,
      "grad_norm": 26.875,
      "learning_rate": 1.1001120476878233e-05,
      "logits/chosen": -3.048595905303955,
      "logits/rejected": -3.0212454795837402,
      "logps/chosen": -320.8490905761719,
      "logps/rejected": -377.32257080078125,
      "loss": 0.2429,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.722134590148926,
      "rewards/margins": 3.0293612480163574,
      "rewards/rejected": -7.751496315002441,
      "step": 5745
    },
    {
      "epoch": 1.2454240225278892,
      "grad_norm": 26.125,
      "learning_rate": 1.0995053024852201e-05,
      "logits/chosen": -3.0470457077026367,
      "logits/rejected": -3.083350419998169,
      "logps/chosen": -339.48968505859375,
      "logps/rejected": -398.4544372558594,
      "loss": 0.2369,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.415940761566162,
      "rewards/margins": 2.8609631061553955,
      "rewards/rejected": -8.276905059814453,
      "step": 5750
    },
    {
      "epoch": 1.2465070941189214,
      "grad_norm": 23.75,
      "learning_rate": 1.098898281739595e-05,
      "logits/chosen": -3.035067081451416,
      "logits/rejected": -3.0193798542022705,
      "logps/chosen": -332.65313720703125,
      "logps/rejected": -388.72259521484375,
      "loss": 0.2826,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.229769706726074,
      "rewards/margins": 2.799334764480591,
      "rewards/rejected": -8.029104232788086,
      "step": 5755
    },
    {
      "epoch": 1.2475901657099535,
      "grad_norm": 28.625,
      "learning_rate": 1.0982909863164377e-05,
      "logits/chosen": -3.005302667617798,
      "logits/rejected": -2.8202970027923584,
      "logps/chosen": -350.9735107421875,
      "logps/rejected": -407.66278076171875,
      "loss": 0.2827,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.998074054718018,
      "rewards/margins": 2.6197943687438965,
      "rewards/rejected": -8.617868423461914,
      "step": 5760
    },
    {
      "epoch": 1.2486732373009857,
      "grad_norm": 23.25,
      "learning_rate": 1.0976834170816282e-05,
      "logits/chosen": -3.029982805252075,
      "logits/rejected": -3.0009026527404785,
      "logps/chosen": -357.8543395996094,
      "logps/rejected": -407.3343505859375,
      "loss": 0.3202,
      "rewards/accuracies": 0.8250001072883606,
      "rewards/chosen": -5.60605001449585,
      "rewards/margins": 2.871216058731079,
      "rewards/rejected": -8.477266311645508,
      "step": 5765
    },
    {
      "epoch": 1.2497563088920178,
      "grad_norm": 25.5,
      "learning_rate": 1.097075574901438e-05,
      "logits/chosen": -2.963892698287964,
      "logits/rejected": -2.8440299034118652,
      "logps/chosen": -343.18408203125,
      "logps/rejected": -390.2073974609375,
      "loss": 0.3392,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.129645347595215,
      "rewards/margins": 2.1965243816375732,
      "rewards/rejected": -7.326170444488525,
      "step": 5770
    },
    {
      "epoch": 1.25083938048305,
      "grad_norm": 15.25,
      "learning_rate": 1.0964674606425267e-05,
      "logits/chosen": -3.0746421813964844,
      "logits/rejected": -2.9862639904022217,
      "logps/chosen": -320.9520263671875,
      "logps/rejected": -374.8645935058594,
      "loss": 0.229,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -5.473803520202637,
      "rewards/margins": 2.538437843322754,
      "rewards/rejected": -8.012240409851074,
      "step": 5775
    },
    {
      "epoch": 1.2519224520740821,
      "grad_norm": 23.125,
      "learning_rate": 1.0958590751719428e-05,
      "logits/chosen": -3.2039828300476074,
      "logits/rejected": -3.143651247024536,
      "logps/chosen": -313.99224853515625,
      "logps/rejected": -360.52264404296875,
      "loss": 0.262,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.176756858825684,
      "rewards/margins": 2.2236669063568115,
      "rewards/rejected": -7.400423526763916,
      "step": 5780
    },
    {
      "epoch": 1.2530055236651143,
      "grad_norm": 15.9375,
      "learning_rate": 1.095250419357121e-05,
      "logits/chosen": -3.0499978065490723,
      "logits/rejected": -3.0161657333374023,
      "logps/chosen": -345.4108581542969,
      "logps/rejected": -385.2159118652344,
      "loss": 0.2434,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -4.947136878967285,
      "rewards/margins": 2.605712890625,
      "rewards/rejected": -7.552849769592285,
      "step": 5785
    },
    {
      "epoch": 1.2540885952561465,
      "grad_norm": 20.5,
      "learning_rate": 1.0946414940658815e-05,
      "logits/chosen": -2.9658071994781494,
      "logits/rejected": -3.0858635902404785,
      "logps/chosen": -328.3982238769531,
      "logps/rejected": -367.3298034667969,
      "loss": 0.2378,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.697236061096191,
      "rewards/margins": 2.717258930206299,
      "rewards/rejected": -7.41449499130249,
      "step": 5790
    },
    {
      "epoch": 1.2551716668471786,
      "grad_norm": 26.375,
      "learning_rate": 1.0940323001664286e-05,
      "logits/chosen": -3.0018441677093506,
      "logits/rejected": -3.070457935333252,
      "logps/chosen": -327.6575622558594,
      "logps/rejected": -370.64288330078125,
      "loss": 0.2801,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.104820728302002,
      "rewards/margins": 2.680741786956787,
      "rewards/rejected": -7.785562038421631,
      "step": 5795
    },
    {
      "epoch": 1.2562547384382108,
      "grad_norm": 15.4375,
      "learning_rate": 1.09342283852735e-05,
      "logits/chosen": -3.180093765258789,
      "logits/rejected": -3.1200671195983887,
      "logps/chosen": -330.6824645996094,
      "logps/rejected": -366.82928466796875,
      "loss": 0.301,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.636427402496338,
      "rewards/margins": 2.4763150215148926,
      "rewards/rejected": -8.11274242401123,
      "step": 5800
    },
    {
      "epoch": 1.257337810029243,
      "grad_norm": 23.5,
      "learning_rate": 1.0928131100176147e-05,
      "logits/chosen": -3.1345810890197754,
      "logits/rejected": -3.124800682067871,
      "logps/chosen": -314.8672180175781,
      "logps/rejected": -360.98480224609375,
      "loss": 0.2768,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.0404953956604,
      "rewards/margins": 2.5577423572540283,
      "rewards/rejected": -7.59823751449585,
      "step": 5805
    },
    {
      "epoch": 1.258420881620275,
      "grad_norm": 19.875,
      "learning_rate": 1.0922031155065727e-05,
      "logits/chosen": -2.978874683380127,
      "logits/rejected": -3.1023755073547363,
      "logps/chosen": -333.7210388183594,
      "logps/rejected": -375.69134521484375,
      "loss": 0.3145,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -4.981715202331543,
      "rewards/margins": 2.6444168090820312,
      "rewards/rejected": -7.626131534576416,
      "step": 5810
    },
    {
      "epoch": 1.2595039532113073,
      "grad_norm": 17.875,
      "learning_rate": 1.0915928558639523e-05,
      "logits/chosen": -3.007841110229492,
      "logits/rejected": -2.93815541267395,
      "logps/chosen": -341.398681640625,
      "logps/rejected": -394.36529541015625,
      "loss": 0.2999,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.429516792297363,
      "rewards/margins": 2.22894549369812,
      "rewards/rejected": -7.6584625244140625,
      "step": 5815
    },
    {
      "epoch": 1.2605870248023394,
      "grad_norm": 15.875,
      "learning_rate": 1.0909823319598612e-05,
      "logits/chosen": -3.1136701107025146,
      "logits/rejected": -3.1179871559143066,
      "logps/chosen": -317.00927734375,
      "logps/rejected": -375.01495361328125,
      "loss": 0.2383,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.2244391441345215,
      "rewards/margins": 2.6890788078308105,
      "rewards/rejected": -7.91351842880249,
      "step": 5820
    },
    {
      "epoch": 1.2616700963933716,
      "grad_norm": 18.625,
      "learning_rate": 1.0903715446647825e-05,
      "logits/chosen": -3.1970250606536865,
      "logits/rejected": -3.057323932647705,
      "logps/chosen": -303.05242919921875,
      "logps/rejected": -350.616455078125,
      "loss": 0.3238,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.877738952636719,
      "rewards/margins": 2.142457962036133,
      "rewards/rejected": -7.020196437835693,
      "step": 5825
    },
    {
      "epoch": 1.2627531679844037,
      "grad_norm": 15.625,
      "learning_rate": 1.0897604948495761e-05,
      "logits/chosen": -3.1172804832458496,
      "logits/rejected": -3.0315747261047363,
      "logps/chosen": -315.5918884277344,
      "logps/rejected": -371.2701721191406,
      "loss": 0.292,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.989575386047363,
      "rewards/margins": 2.706392765045166,
      "rewards/rejected": -7.695967674255371,
      "step": 5830
    },
    {
      "epoch": 1.263836239575436,
      "grad_norm": 16.125,
      "learning_rate": 1.089149183385475e-05,
      "logits/chosen": -3.0823469161987305,
      "logits/rejected": -3.03135347366333,
      "logps/chosen": -327.08013916015625,
      "logps/rejected": -389.0831298828125,
      "loss": 0.2643,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.3990912437438965,
      "rewards/margins": 2.5588622093200684,
      "rewards/rejected": -7.957952976226807,
      "step": 5835
    },
    {
      "epoch": 1.264919311166468,
      "grad_norm": 17.25,
      "learning_rate": 1.0885376111440866e-05,
      "logits/chosen": -3.199946880340576,
      "logits/rejected": -3.1101856231689453,
      "logps/chosen": -319.2005920410156,
      "logps/rejected": -380.82373046875,
      "loss": 0.2153,
      "rewards/accuracies": 0.9083334803581238,
      "rewards/chosen": -5.298712730407715,
      "rewards/margins": 3.0102646350860596,
      "rewards/rejected": -8.308978080749512,
      "step": 5840
    },
    {
      "epoch": 1.2660023827575002,
      "grad_norm": 11.6875,
      "learning_rate": 1.0879257789973886e-05,
      "logits/chosen": -3.2694060802459717,
      "logits/rejected": -3.155984878540039,
      "logps/chosen": -326.58563232421875,
      "logps/rejected": -387.0062561035156,
      "loss": 0.201,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -5.333475112915039,
      "rewards/margins": 3.0603365898132324,
      "rewards/rejected": -8.39381217956543,
      "step": 5845
    },
    {
      "epoch": 1.2670854543485324,
      "grad_norm": 29.75,
      "learning_rate": 1.0873136878177306e-05,
      "logits/chosen": -3.1596341133117676,
      "logits/rejected": -3.1318774223327637,
      "logps/chosen": -345.2782897949219,
      "logps/rejected": -396.2611389160156,
      "loss": 0.308,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.932945251464844,
      "rewards/margins": 2.8151464462280273,
      "rewards/rejected": -8.748092651367188,
      "step": 5850
    },
    {
      "epoch": 1.2681685259395645,
      "grad_norm": 18.625,
      "learning_rate": 1.0867013384778304e-05,
      "logits/chosen": -3.134035110473633,
      "logits/rejected": -3.044124126434326,
      "logps/chosen": -323.16741943359375,
      "logps/rejected": -374.8368835449219,
      "loss": 0.221,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.378701686859131,
      "rewards/margins": 2.6282997131347656,
      "rewards/rejected": -8.007001876831055,
      "step": 5855
    },
    {
      "epoch": 1.2692515975305967,
      "grad_norm": 22.125,
      "learning_rate": 1.0860887318507753e-05,
      "logits/chosen": -3.031609535217285,
      "logits/rejected": -3.0466599464416504,
      "logps/chosen": -341.20751953125,
      "logps/rejected": -377.48846435546875,
      "loss": 0.3248,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.825469970703125,
      "rewards/margins": 2.4904417991638184,
      "rewards/rejected": -8.315912246704102,
      "step": 5860
    },
    {
      "epoch": 1.2703346691216288,
      "grad_norm": 31.75,
      "learning_rate": 1.085475868810018e-05,
      "logits/chosen": -3.2069315910339355,
      "logits/rejected": -3.077960729598999,
      "logps/chosen": -335.66229248046875,
      "logps/rejected": -392.8728332519531,
      "loss": 0.3252,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.947548866271973,
      "rewards/margins": 2.779942750930786,
      "rewards/rejected": -8.72749137878418,
      "step": 5865
    },
    {
      "epoch": 1.271417740712661,
      "grad_norm": 28.125,
      "learning_rate": 1.0848627502293772e-05,
      "logits/chosen": -3.1056036949157715,
      "logits/rejected": -3.0852389335632324,
      "logps/chosen": -340.1831970214844,
      "logps/rejected": -377.218017578125,
      "loss": 0.3007,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.428526878356934,
      "rewards/margins": 2.5784859657287598,
      "rewards/rejected": -8.007013320922852,
      "step": 5870
    },
    {
      "epoch": 1.2725008123036932,
      "grad_norm": 24.875,
      "learning_rate": 1.0842493769830366e-05,
      "logits/chosen": -3.248342990875244,
      "logits/rejected": -3.136357069015503,
      "logps/chosen": -309.7193298339844,
      "logps/rejected": -377.22857666015625,
      "loss": 0.2863,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.987521171569824,
      "rewards/margins": 3.1182823181152344,
      "rewards/rejected": -8.105803489685059,
      "step": 5875
    },
    {
      "epoch": 1.2735838838947253,
      "grad_norm": 13.75,
      "learning_rate": 1.0836357499455424e-05,
      "logits/chosen": -3.0904831886291504,
      "logits/rejected": -3.087261438369751,
      "logps/chosen": -297.6565856933594,
      "logps/rejected": -357.70831298828125,
      "loss": 0.201,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.330855846405029,
      "rewards/margins": 3.4999947547912598,
      "rewards/rejected": -7.830851078033447,
      "step": 5880
    },
    {
      "epoch": 1.2746669554857575,
      "grad_norm": 18.25,
      "learning_rate": 1.0830218699918028e-05,
      "logits/chosen": -3.0837340354919434,
      "logits/rejected": -3.0427005290985107,
      "logps/chosen": -326.55609130859375,
      "logps/rejected": -383.1168518066406,
      "loss": 0.3032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.209286212921143,
      "rewards/margins": 2.6413540840148926,
      "rewards/rejected": -7.850639343261719,
      "step": 5885
    },
    {
      "epoch": 1.2757500270767896,
      "grad_norm": 28.625,
      "learning_rate": 1.0824077379970865e-05,
      "logits/chosen": -3.1735730171203613,
      "logits/rejected": -3.1587131023406982,
      "logps/chosen": -324.1385803222656,
      "logps/rejected": -378.71746826171875,
      "loss": 0.3342,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.762078762054443,
      "rewards/margins": 2.4118123054504395,
      "rewards/rejected": -8.173891067504883,
      "step": 5890
    },
    {
      "epoch": 1.276833098667822,
      "grad_norm": 19.375,
      "learning_rate": 1.0817933548370218e-05,
      "logits/chosen": -2.929447889328003,
      "logits/rejected": -3.050015926361084,
      "logps/chosen": -354.573974609375,
      "logps/rejected": -386.526123046875,
      "loss": 0.3015,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.464478492736816,
      "rewards/margins": 2.5762996673583984,
      "rewards/rejected": -8.040778160095215,
      "step": 5895
    },
    {
      "epoch": 1.2779161702588542,
      "grad_norm": 21.375,
      "learning_rate": 1.081178721387595e-05,
      "logits/chosen": -3.1629276275634766,
      "logits/rejected": -3.157005548477173,
      "logps/chosen": -326.55364990234375,
      "logps/rejected": -369.0660400390625,
      "loss": 0.3001,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.721410274505615,
      "rewards/margins": 2.4201242923736572,
      "rewards/rejected": -8.141534805297852,
      "step": 5900
    },
    {
      "epoch": 1.2789992418498863,
      "grad_norm": 11.0,
      "learning_rate": 1.080563838525149e-05,
      "logits/chosen": -3.057692050933838,
      "logits/rejected": -3.123286247253418,
      "logps/chosen": -313.6417541503906,
      "logps/rejected": -351.7977294921875,
      "loss": 0.2475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.885014533996582,
      "rewards/margins": 2.6576437950134277,
      "rewards/rejected": -7.54265832901001,
      "step": 5905
    },
    {
      "epoch": 1.2800823134409185,
      "grad_norm": 22.375,
      "learning_rate": 1.0799487071263828e-05,
      "logits/chosen": -2.8775198459625244,
      "logits/rejected": -2.9307851791381836,
      "logps/chosen": -316.8709716796875,
      "logps/rejected": -370.9345703125,
      "loss": 0.2937,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.78914737701416,
      "rewards/margins": 2.2409462928771973,
      "rewards/rejected": -7.030093193054199,
      "step": 5910
    },
    {
      "epoch": 1.2811653850319507,
      "grad_norm": 10.6875,
      "learning_rate": 1.0793333280683492e-05,
      "logits/chosen": -2.9545817375183105,
      "logits/rejected": -3.0689260959625244,
      "logps/chosen": -323.0229797363281,
      "logps/rejected": -351.8775939941406,
      "loss": 0.3038,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -4.736842155456543,
      "rewards/margins": 2.3470587730407715,
      "rewards/rejected": -7.083900451660156,
      "step": 5915
    },
    {
      "epoch": 1.2822484566229828,
      "grad_norm": 19.375,
      "learning_rate": 1.078717702228455e-05,
      "logits/chosen": -3.1603894233703613,
      "logits/rejected": -3.1210830211639404,
      "logps/chosen": -334.40185546875,
      "logps/rejected": -384.3031005859375,
      "loss": 0.2565,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.765501976013184,
      "rewards/margins": 2.676474094390869,
      "rewards/rejected": -8.441976547241211,
      "step": 5920
    },
    {
      "epoch": 1.283331528214015,
      "grad_norm": 8.9375,
      "learning_rate": 1.0781018304844578e-05,
      "logits/chosen": -2.986077070236206,
      "logits/rejected": -2.9124131202697754,
      "logps/chosen": -317.772216796875,
      "logps/rejected": -367.75030517578125,
      "loss": 0.2212,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -4.8614583015441895,
      "rewards/margins": 2.6009578704833984,
      "rewards/rejected": -7.4624152183532715,
      "step": 5925
    },
    {
      "epoch": 1.2844145998050471,
      "grad_norm": 19.875,
      "learning_rate": 1.0774857137144661e-05,
      "logits/chosen": -2.989952564239502,
      "logits/rejected": -2.9411864280700684,
      "logps/chosen": -330.1673889160156,
      "logps/rejected": -380.3165588378906,
      "loss": 0.3119,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.058451175689697,
      "rewards/margins": 2.4343879222869873,
      "rewards/rejected": -7.4928388595581055,
      "step": 5930
    },
    {
      "epoch": 1.2854976713960793,
      "grad_norm": 26.5,
      "learning_rate": 1.0768693527969385e-05,
      "logits/chosen": -2.9194655418395996,
      "logits/rejected": -2.8935647010803223,
      "logps/chosen": -318.12158203125,
      "logps/rejected": -372.59307861328125,
      "loss": 0.3058,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -4.805605888366699,
      "rewards/margins": 2.3248112201690674,
      "rewards/rejected": -7.1304168701171875,
      "step": 5935
    },
    {
      "epoch": 1.2865807429871114,
      "grad_norm": 19.0,
      "learning_rate": 1.0762527486106812e-05,
      "logits/chosen": -3.1485657691955566,
      "logits/rejected": -3.101994276046753,
      "logps/chosen": -326.7908020019531,
      "logps/rejected": -378.8125305175781,
      "loss": 0.3366,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.273049354553223,
      "rewards/margins": 2.703970432281494,
      "rewards/rejected": -7.977019309997559,
      "step": 5940
    },
    {
      "epoch": 1.2876638145781436,
      "grad_norm": 23.5,
      "learning_rate": 1.0756359020348464e-05,
      "logits/chosen": -2.976914644241333,
      "logits/rejected": -2.949021816253662,
      "logps/chosen": -294.29827880859375,
      "logps/rejected": -346.1313781738281,
      "loss": 0.2775,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.196505069732666,
      "rewards/margins": 2.6123547554016113,
      "rewards/rejected": -6.808859825134277,
      "step": 5945
    },
    {
      "epoch": 1.2887468861691758,
      "grad_norm": 22.75,
      "learning_rate": 1.0750188139489336e-05,
      "logits/chosen": -3.046260356903076,
      "logits/rejected": -3.113056182861328,
      "logps/chosen": -302.9249572753906,
      "logps/rejected": -346.08599853515625,
      "loss": 0.2574,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.769841194152832,
      "rewards/margins": 2.6289172172546387,
      "rewards/rejected": -7.398758888244629,
      "step": 5950
    },
    {
      "epoch": 1.289829957760208,
      "grad_norm": 21.375,
      "learning_rate": 1.0744014852327853e-05,
      "logits/chosen": -3.1827638149261475,
      "logits/rejected": -3.015235424041748,
      "logps/chosen": -312.2342224121094,
      "logps/rejected": -369.0506286621094,
      "loss": 0.3096,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.996426582336426,
      "rewards/margins": 2.6022467613220215,
      "rewards/rejected": -7.598673343658447,
      "step": 5955
    },
    {
      "epoch": 1.29091302935124,
      "grad_norm": 24.75,
      "learning_rate": 1.0737839167665877e-05,
      "logits/chosen": -2.9422409534454346,
      "logits/rejected": -2.9018921852111816,
      "logps/chosen": -331.22308349609375,
      "logps/rejected": -369.29034423828125,
      "loss": 0.275,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.558159828186035,
      "rewards/margins": 2.670886516571045,
      "rewards/rejected": -7.229046821594238,
      "step": 5960
    },
    {
      "epoch": 1.2919961009422722,
      "grad_norm": 22.375,
      "learning_rate": 1.0731661094308683e-05,
      "logits/chosen": -3.0260837078094482,
      "logits/rejected": -2.9857375621795654,
      "logps/chosen": -326.9279479980469,
      "logps/rejected": -371.94781494140625,
      "loss": 0.2837,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.895761966705322,
      "rewards/margins": 2.5662035942077637,
      "rewards/rejected": -7.461966037750244,
      "step": 5965
    },
    {
      "epoch": 1.2930791725333044,
      "grad_norm": 18.125,
      "learning_rate": 1.0725480641064963e-05,
      "logits/chosen": -3.1082606315612793,
      "logits/rejected": -3.2466228008270264,
      "logps/chosen": -312.1835021972656,
      "logps/rejected": -342.5710754394531,
      "loss": 0.2271,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.6102800369262695,
      "rewards/margins": 2.835742473602295,
      "rewards/rejected": -7.446022987365723,
      "step": 5970
    },
    {
      "epoch": 1.2941622441243366,
      "grad_norm": 22.25,
      "learning_rate": 1.0719297816746785e-05,
      "logits/chosen": -2.899869680404663,
      "logits/rejected": -2.943584442138672,
      "logps/chosen": -313.1765441894531,
      "logps/rejected": -352.2496032714844,
      "loss": 0.3178,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.1309380531311035,
      "rewards/margins": 2.457705497741699,
      "rewards/rejected": -6.5886430740356445,
      "step": 5975
    },
    {
      "epoch": 1.2952453157153687,
      "grad_norm": 26.625,
      "learning_rate": 1.0713112630169616e-05,
      "logits/chosen": -3.146419048309326,
      "logits/rejected": -2.9647157192230225,
      "logps/chosen": -302.82470703125,
      "logps/rejected": -363.29754638671875,
      "loss": 0.2193,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.401900291442871,
      "rewards/margins": 2.6870851516723633,
      "rewards/rejected": -7.088985443115234,
      "step": 5980
    },
    {
      "epoch": 1.2963283873064009,
      "grad_norm": 20.25,
      "learning_rate": 1.070692509015228e-05,
      "logits/chosen": -3.0241315364837646,
      "logits/rejected": -3.0123157501220703,
      "logps/chosen": -298.6112365722656,
      "logps/rejected": -336.13348388671875,
      "loss": 0.2768,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.344757556915283,
      "rewards/margins": 2.3117973804473877,
      "rewards/rejected": -6.65655517578125,
      "step": 5985
    },
    {
      "epoch": 1.2974114588974333,
      "grad_norm": 20.625,
      "learning_rate": 1.0700735205516959e-05,
      "logits/chosen": -2.9703032970428467,
      "logits/rejected": -3.033684015274048,
      "logps/chosen": -314.86907958984375,
      "logps/rejected": -360.64288330078125,
      "loss": 0.2615,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -4.349424362182617,
      "rewards/margins": 2.670215606689453,
      "rewards/rejected": -7.019639015197754,
      "step": 5990
    },
    {
      "epoch": 1.2984945304884654,
      "grad_norm": 24.125,
      "learning_rate": 1.0694542985089176e-05,
      "logits/chosen": -2.9770874977111816,
      "logits/rejected": -3.0060620307922363,
      "logps/chosen": -317.1051025390625,
      "logps/rejected": -372.3232116699219,
      "loss": 0.2775,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.6001362800598145,
      "rewards/margins": 2.916315793991089,
      "rewards/rejected": -7.516451835632324,
      "step": 5995
    },
    {
      "epoch": 1.2995776020794976,
      "grad_norm": 18.625,
      "learning_rate": 1.0688348437697788e-05,
      "logits/chosen": -3.1532998085021973,
      "logits/rejected": -3.076730251312256,
      "logps/chosen": -301.53240966796875,
      "logps/rejected": -352.5526428222656,
      "loss": 0.2299,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -4.016139507293701,
      "rewards/margins": 2.888875722885132,
      "rewards/rejected": -6.905014991760254,
      "step": 6000
    },
    {
      "epoch": 1.2995776020794976,
      "eval_logits/chosen": -3.129425048828125,
      "eval_logits/rejected": -3.0613110065460205,
      "eval_logps/chosen": -300.5923156738281,
      "eval_logps/rejected": -343.0413818359375,
      "eval_loss": 0.4141290783882141,
      "eval_rewards/accuracies": 0.8050000071525574,
      "eval_rewards/chosen": -4.319552421569824,
      "eval_rewards/margins": 1.9496605396270752,
      "eval_rewards/rejected": -6.26921272277832,
      "eval_runtime": 14.6676,
      "eval_samples_per_second": 13.635,
      "eval_steps_per_second": 13.635,
      "step": 6000
    },
    {
      "epoch": 1.3006606736705297,
      "grad_norm": 15.1875,
      "learning_rate": 1.0682151572174967e-05,
      "logits/chosen": -3.0155301094055176,
      "logits/rejected": -3.0080976486206055,
      "logps/chosen": -296.72247314453125,
      "logps/rejected": -346.89495849609375,
      "loss": 0.2217,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -4.077033519744873,
      "rewards/margins": 2.6798722743988037,
      "rewards/rejected": -6.756906032562256,
      "step": 6005
    },
    {
      "epoch": 1.301743745261562,
      "grad_norm": 22.25,
      "learning_rate": 1.0675952397356198e-05,
      "logits/chosen": -3.195451498031616,
      "logits/rejected": -3.1201367378234863,
      "logps/chosen": -310.18951416015625,
      "logps/rejected": -356.0938720703125,
      "loss": 0.2559,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.766565322875977,
      "rewards/margins": 2.635601043701172,
      "rewards/rejected": -7.402166843414307,
      "step": 6010
    },
    {
      "epoch": 1.302826816852594,
      "grad_norm": 17.0,
      "learning_rate": 1.0669750922080242e-05,
      "logits/chosen": -3.083357095718384,
      "logits/rejected": -3.0918052196502686,
      "logps/chosen": -324.7391052246094,
      "logps/rejected": -370.17449951171875,
      "loss": 0.2458,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.783339500427246,
      "rewards/margins": 2.62338924407959,
      "rewards/rejected": -7.406729221343994,
      "step": 6015
    },
    {
      "epoch": 1.3039098884436262,
      "grad_norm": 18.5,
      "learning_rate": 1.0663547155189156e-05,
      "logits/chosen": -3.0830483436584473,
      "logits/rejected": -3.0489375591278076,
      "logps/chosen": -327.0721435546875,
      "logps/rejected": -372.4896240234375,
      "loss": 0.2323,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.266324043273926,
      "rewards/margins": 2.8745815753936768,
      "rewards/rejected": -7.140905857086182,
      "step": 6020
    },
    {
      "epoch": 1.3049929600346584,
      "grad_norm": 14.0625,
      "learning_rate": 1.0657341105528256e-05,
      "logits/chosen": -3.105966567993164,
      "logits/rejected": -3.197953224182129,
      "logps/chosen": -314.03521728515625,
      "logps/rejected": -358.35888671875,
      "loss": 0.3123,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.952166557312012,
      "rewards/margins": 2.6430373191833496,
      "rewards/rejected": -7.5952043533325195,
      "step": 6025
    },
    {
      "epoch": 1.3060760316256905,
      "grad_norm": 12.0625,
      "learning_rate": 1.0651132781946119e-05,
      "logits/chosen": -3.041498899459839,
      "logits/rejected": -2.9318885803222656,
      "logps/chosen": -306.776123046875,
      "logps/rejected": -359.6872253417969,
      "loss": 0.2226,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.171546459197998,
      "rewards/margins": 3.097689628601074,
      "rewards/rejected": -7.269236087799072,
      "step": 6030
    },
    {
      "epoch": 1.3071591032167227,
      "grad_norm": 20.75,
      "learning_rate": 1.0644922193294554e-05,
      "logits/chosen": -3.0277645587921143,
      "logits/rejected": -3.045985460281372,
      "logps/chosen": -311.87054443359375,
      "logps/rejected": -366.1253967285156,
      "loss": 0.1957,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -4.390000343322754,
      "rewards/margins": 3.018247127532959,
      "rewards/rejected": -7.4082465171813965,
      "step": 6035
    },
    {
      "epoch": 1.3082421748077548,
      "grad_norm": 22.25,
      "learning_rate": 1.063870934842861e-05,
      "logits/chosen": -3.065904140472412,
      "logits/rejected": -3.1358280181884766,
      "logps/chosen": -309.68798828125,
      "logps/rejected": -349.1609802246094,
      "loss": 0.317,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.517958641052246,
      "rewards/margins": 2.6461081504821777,
      "rewards/rejected": -7.164067268371582,
      "step": 6040
    },
    {
      "epoch": 1.309325246398787,
      "grad_norm": 16.125,
      "learning_rate": 1.0632494256206547e-05,
      "logits/chosen": -3.1141040325164795,
      "logits/rejected": -3.1680116653442383,
      "logps/chosen": -318.3184509277344,
      "logps/rejected": -367.09674072265625,
      "loss": 0.3066,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.096056938171387,
      "rewards/margins": 2.6661264896392822,
      "rewards/rejected": -7.76218318939209,
      "step": 6045
    },
    {
      "epoch": 1.3104083179898192,
      "grad_norm": 22.875,
      "learning_rate": 1.0626276925489833e-05,
      "logits/chosen": -3.0401196479797363,
      "logits/rejected": -2.98860764503479,
      "logps/chosen": -305.2593078613281,
      "logps/rejected": -358.7225646972656,
      "loss": 0.2307,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.315443515777588,
      "rewards/margins": 2.7880702018737793,
      "rewards/rejected": -7.103513240814209,
      "step": 6050
    },
    {
      "epoch": 1.3114913895808513,
      "grad_norm": 24.75,
      "learning_rate": 1.0620057365143124e-05,
      "logits/chosen": -3.1458706855773926,
      "logits/rejected": -3.061649799346924,
      "logps/chosen": -331.65875244140625,
      "logps/rejected": -377.994873046875,
      "loss": 0.3604,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -5.355137348175049,
      "rewards/margins": 2.4361307621002197,
      "rewards/rejected": -7.791268348693848,
      "step": 6055
    },
    {
      "epoch": 1.3125744611718835,
      "grad_norm": 16.125,
      "learning_rate": 1.061383558403426e-05,
      "logits/chosen": -2.9835402965545654,
      "logits/rejected": -2.927891254425049,
      "logps/chosen": -317.83233642578125,
      "logps/rejected": -379.6717529296875,
      "loss": 0.2287,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.484260082244873,
      "rewards/margins": 2.794806957244873,
      "rewards/rejected": -7.279066562652588,
      "step": 6060
    },
    {
      "epoch": 1.3136575327629156,
      "grad_norm": 17.375,
      "learning_rate": 1.0607611591034239e-05,
      "logits/chosen": -3.043379783630371,
      "logits/rejected": -2.980273723602295,
      "logps/chosen": -310.42523193359375,
      "logps/rejected": -366.92022705078125,
      "loss": 0.205,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.358177661895752,
      "rewards/margins": 2.836945056915283,
      "rewards/rejected": -7.195122718811035,
      "step": 6065
    },
    {
      "epoch": 1.3147406043539478,
      "grad_norm": 23.375,
      "learning_rate": 1.060138539501722e-05,
      "logits/chosen": -3.1290836334228516,
      "logits/rejected": -3.0759005546569824,
      "logps/chosen": -309.87432861328125,
      "logps/rejected": -363.46246337890625,
      "loss": 0.3546,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -4.603633403778076,
      "rewards/margins": 2.4828743934631348,
      "rewards/rejected": -7.086508274078369,
      "step": 6070
    },
    {
      "epoch": 1.31582367594498,
      "grad_norm": 26.125,
      "learning_rate": 1.05951570048605e-05,
      "logits/chosen": -3.1667397022247314,
      "logits/rejected": -3.0980701446533203,
      "logps/chosen": -312.37164306640625,
      "logps/rejected": -365.97991943359375,
      "loss": 0.3036,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.423161029815674,
      "rewards/margins": 2.4883551597595215,
      "rewards/rejected": -6.911515712738037,
      "step": 6075
    },
    {
      "epoch": 1.3169067475360121,
      "grad_norm": 24.125,
      "learning_rate": 1.0588926429444506e-05,
      "logits/chosen": -3.1985268592834473,
      "logits/rejected": -3.037935972213745,
      "logps/chosen": -303.3695373535156,
      "logps/rejected": -362.5956726074219,
      "loss": 0.2579,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.593846321105957,
      "rewards/margins": 2.87857985496521,
      "rewards/rejected": -7.472426414489746,
      "step": 6080
    },
    {
      "epoch": 1.3179898191270443,
      "grad_norm": 21.125,
      "learning_rate": 1.0582693677652779e-05,
      "logits/chosen": -2.941063404083252,
      "logits/rejected": -2.967134952545166,
      "logps/chosen": -307.1399841308594,
      "logps/rejected": -357.4988708496094,
      "loss": 0.3135,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.295450687408447,
      "rewards/margins": 2.527141571044922,
      "rewards/rejected": -6.822592258453369,
      "step": 6085
    },
    {
      "epoch": 1.3190728907180764,
      "grad_norm": 21.0,
      "learning_rate": 1.0576458758371965e-05,
      "logits/chosen": -2.923325300216675,
      "logits/rejected": -2.841299533843994,
      "logps/chosen": -306.7635498046875,
      "logps/rejected": -353.08038330078125,
      "loss": 0.298,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -4.206625938415527,
      "rewards/margins": 2.3285529613494873,
      "rewards/rejected": -6.535179138183594,
      "step": 6090
    },
    {
      "epoch": 1.3201559623091086,
      "grad_norm": 18.25,
      "learning_rate": 1.05702216804918e-05,
      "logits/chosen": -2.8585293292999268,
      "logits/rejected": -2.956233263015747,
      "logps/chosen": -306.43450927734375,
      "logps/rejected": -344.3274230957031,
      "loss": 0.2661,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.276177883148193,
      "rewards/margins": 2.6884946823120117,
      "rewards/rejected": -6.964673042297363,
      "step": 6095
    },
    {
      "epoch": 1.3212390339001407,
      "grad_norm": 18.5,
      "learning_rate": 1.0563982452905095e-05,
      "logits/chosen": -3.013235569000244,
      "logits/rejected": -2.953650951385498,
      "logps/chosen": -303.94000244140625,
      "logps/rejected": -359.276123046875,
      "loss": 0.2215,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.422074794769287,
      "rewards/margins": 2.8506364822387695,
      "rewards/rejected": -7.272711753845215,
      "step": 6100
    },
    {
      "epoch": 1.322322105491173,
      "grad_norm": 20.875,
      "learning_rate": 1.055774108450773e-05,
      "logits/chosen": -3.0823113918304443,
      "logits/rejected": -3.0923328399658203,
      "logps/chosen": -316.2342529296875,
      "logps/rejected": -364.44085693359375,
      "loss": 0.3012,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.744819641113281,
      "rewards/margins": 2.655000925064087,
      "rewards/rejected": -7.399819850921631,
      "step": 6105
    },
    {
      "epoch": 1.323405177082205,
      "grad_norm": 27.625,
      "learning_rate": 1.0551497584198635e-05,
      "logits/chosen": -2.849191188812256,
      "logits/rejected": -3.013546943664551,
      "logps/chosen": -309.2489318847656,
      "logps/rejected": -356.624755859375,
      "loss": 0.3298,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.40834903717041,
      "rewards/margins": 2.4389290809631348,
      "rewards/rejected": -6.847278594970703,
      "step": 6110
    },
    {
      "epoch": 1.3244882486732372,
      "grad_norm": 17.375,
      "learning_rate": 1.0545251960879777e-05,
      "logits/chosen": -2.908191680908203,
      "logits/rejected": -2.8685975074768066,
      "logps/chosen": -308.7104797363281,
      "logps/rejected": -369.33599853515625,
      "loss": 0.248,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.513687610626221,
      "rewards/margins": 2.6511070728302,
      "rewards/rejected": -7.164793968200684,
      "step": 6115
    },
    {
      "epoch": 1.3255713202642694,
      "grad_norm": 12.5625,
      "learning_rate": 1.053900422345616e-05,
      "logits/chosen": -2.91652250289917,
      "logits/rejected": -2.7970528602600098,
      "logps/chosen": -306.05377197265625,
      "logps/rejected": -372.6474609375,
      "loss": 0.2277,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.279263496398926,
      "rewards/margins": 2.728107452392578,
      "rewards/rejected": -7.0073699951171875,
      "step": 6120
    },
    {
      "epoch": 1.3266543918553015,
      "grad_norm": 23.875,
      "learning_rate": 1.0532754380835789e-05,
      "logits/chosen": -3.00355863571167,
      "logits/rejected": -2.770104169845581,
      "logps/chosen": -307.6023864746094,
      "logps/rejected": -367.7640380859375,
      "loss": 0.2804,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.627732276916504,
      "rewards/margins": 2.440418243408203,
      "rewards/rejected": -7.068150520324707,
      "step": 6125
    },
    {
      "epoch": 1.3277374634463337,
      "grad_norm": 26.5,
      "learning_rate": 1.052650244192968e-05,
      "logits/chosen": -2.903996706008911,
      "logits/rejected": -2.891838550567627,
      "logps/chosen": -314.5550231933594,
      "logps/rejected": -360.7840270996094,
      "loss": 0.3443,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -4.871306419372559,
      "rewards/margins": 2.240732192993164,
      "rewards/rejected": -7.112039089202881,
      "step": 6130
    },
    {
      "epoch": 1.3288205350373659,
      "grad_norm": 17.75,
      "learning_rate": 1.0520248415651833e-05,
      "logits/chosen": -2.7732765674591064,
      "logits/rejected": -2.7939486503601074,
      "logps/chosen": -325.8810729980469,
      "logps/rejected": -374.45489501953125,
      "loss": 0.2636,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.300878047943115,
      "rewards/margins": 2.72587513923645,
      "rewards/rejected": -7.0267534255981445,
      "step": 6135
    },
    {
      "epoch": 1.329903606628398,
      "grad_norm": 15.4375,
      "learning_rate": 1.051399231091923e-05,
      "logits/chosen": -2.9330201148986816,
      "logits/rejected": -3.05928635597229,
      "logps/chosen": -297.5533447265625,
      "logps/rejected": -350.97955322265625,
      "loss": 0.2123,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.507937431335449,
      "rewards/margins": 2.6475930213928223,
      "rewards/rejected": -7.155529975891113,
      "step": 6140
    },
    {
      "epoch": 1.3309866782194302,
      "grad_norm": 26.875,
      "learning_rate": 1.0507734136651811e-05,
      "logits/chosen": -3.036648750305176,
      "logits/rejected": -3.0185296535491943,
      "logps/chosen": -309.8866882324219,
      "logps/rejected": -364.9476623535156,
      "loss": 0.3033,
      "rewards/accuracies": 0.8083333969116211,
      "rewards/chosen": -4.938461780548096,
      "rewards/margins": 2.6940178871154785,
      "rewards/rejected": -7.632479190826416,
      "step": 6145
    },
    {
      "epoch": 1.3320697498104623,
      "grad_norm": 25.5,
      "learning_rate": 1.0501473901772464e-05,
      "logits/chosen": -3.055542230606079,
      "logits/rejected": -2.9825077056884766,
      "logps/chosen": -316.8515625,
      "logps/rejected": -371.39129638671875,
      "loss": 0.3408,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -4.879720687866211,
      "rewards/margins": 2.534566640853882,
      "rewards/rejected": -7.4142866134643555,
      "step": 6150
    },
    {
      "epoch": 1.3331528214014947,
      "grad_norm": 23.625,
      "learning_rate": 1.0495211615207025e-05,
      "logits/chosen": -2.903456211090088,
      "logits/rejected": -2.8877556324005127,
      "logps/chosen": -324.33880615234375,
      "logps/rejected": -376.82794189453125,
      "loss": 0.2458,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.556795597076416,
      "rewards/margins": 2.792792797088623,
      "rewards/rejected": -7.349587917327881,
      "step": 6155
    },
    {
      "epoch": 1.3342358929925269,
      "grad_norm": 19.625,
      "learning_rate": 1.0488947285884246e-05,
      "logits/chosen": -2.985050916671753,
      "logits/rejected": -2.951732635498047,
      "logps/chosen": -313.0109558105469,
      "logps/rejected": -357.62506103515625,
      "loss": 0.2615,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.564778804779053,
      "rewards/margins": 2.6334128379821777,
      "rewards/rejected": -7.1981916427612305,
      "step": 6160
    },
    {
      "epoch": 1.335318964583559,
      "grad_norm": 17.125,
      "learning_rate": 1.0482680922735798e-05,
      "logits/chosen": -3.1888086795806885,
      "logits/rejected": -3.1578993797302246,
      "logps/chosen": -310.9721374511719,
      "logps/rejected": -353.1031188964844,
      "loss": 0.3033,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -4.800003528594971,
      "rewards/margins": 2.5504229068756104,
      "rewards/rejected": -7.350427150726318,
      "step": 6165
    },
    {
      "epoch": 1.3364020361745912,
      "grad_norm": 20.875,
      "learning_rate": 1.0476412534696244e-05,
      "logits/chosen": -3.0960333347320557,
      "logits/rejected": -3.0649161338806152,
      "logps/chosen": -321.1448974609375,
      "logps/rejected": -373.02801513671875,
      "loss": 0.2385,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.539035797119141,
      "rewards/margins": 2.9871888160705566,
      "rewards/rejected": -7.5262250900268555,
      "step": 6170
    },
    {
      "epoch": 1.3374851077656233,
      "grad_norm": 25.5,
      "learning_rate": 1.0470142130703045e-05,
      "logits/chosen": -3.172499179840088,
      "logits/rejected": -3.1861650943756104,
      "logps/chosen": -314.2149963378906,
      "logps/rejected": -362.16180419921875,
      "loss": 0.2427,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.998209476470947,
      "rewards/margins": 2.7660281658172607,
      "rewards/rejected": -7.764237403869629,
      "step": 6175
    },
    {
      "epoch": 1.3385681793566555,
      "grad_norm": 19.375,
      "learning_rate": 1.0463869719696528e-05,
      "logits/chosen": -2.994249105453491,
      "logits/rejected": -2.9972808361053467,
      "logps/chosen": -317.7293701171875,
      "logps/rejected": -362.0760803222656,
      "loss": 0.297,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.505670070648193,
      "rewards/margins": 2.595961809158325,
      "rewards/rejected": -7.101632118225098,
      "step": 6180
    },
    {
      "epoch": 1.3396512509476877,
      "grad_norm": 24.375,
      "learning_rate": 1.0457595310619881e-05,
      "logits/chosen": -2.9340693950653076,
      "logits/rejected": -3.040144205093384,
      "logps/chosen": -336.251708984375,
      "logps/rejected": -383.70062255859375,
      "loss": 0.2876,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.850746154785156,
      "rewards/margins": 2.9153060913085938,
      "rewards/rejected": -7.76605224609375,
      "step": 6185
    },
    {
      "epoch": 1.3407343225387198,
      "grad_norm": 12.75,
      "learning_rate": 1.045131891241915e-05,
      "logits/chosen": -3.010021448135376,
      "logits/rejected": -2.9751667976379395,
      "logps/chosen": -319.7196350097656,
      "logps/rejected": -368.21893310546875,
      "loss": 0.2252,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.55413293838501,
      "rewards/margins": 3.028226375579834,
      "rewards/rejected": -7.582359313964844,
      "step": 6190
    },
    {
      "epoch": 1.341817394129752,
      "grad_norm": 22.125,
      "learning_rate": 1.0445040534043208e-05,
      "logits/chosen": -3.047685146331787,
      "logits/rejected": -2.899512529373169,
      "logps/chosen": -327.83843994140625,
      "logps/rejected": -375.2582092285156,
      "loss": 0.235,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.620074272155762,
      "rewards/margins": 2.667555570602417,
      "rewards/rejected": -7.2876296043396,
      "step": 6195
    },
    {
      "epoch": 1.3429004657207841,
      "grad_norm": 18.5,
      "learning_rate": 1.0438760184443752e-05,
      "logits/chosen": -3.2132277488708496,
      "logits/rejected": -3.119208335876465,
      "logps/chosen": -311.8704528808594,
      "logps/rejected": -368.13037109375,
      "loss": 0.3334,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.052960395812988,
      "rewards/margins": 2.6256120204925537,
      "rewards/rejected": -7.678572177886963,
      "step": 6200
    },
    {
      "epoch": 1.3439835373118163,
      "grad_norm": 18.75,
      "learning_rate": 1.0432477872575293e-05,
      "logits/chosen": -3.087966203689575,
      "logits/rejected": -3.087881326675415,
      "logps/chosen": -310.404296875,
      "logps/rejected": -355.86322021484375,
      "loss": 0.2251,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -4.284920692443848,
      "rewards/margins": 2.9574267864227295,
      "rewards/rejected": -7.242347717285156,
      "step": 6205
    },
    {
      "epoch": 1.3450666089028485,
      "grad_norm": 31.0,
      "learning_rate": 1.042619360739514e-05,
      "logits/chosen": -3.122197389602661,
      "logits/rejected": -3.18550443649292,
      "logps/chosen": -317.04864501953125,
      "logps/rejected": -355.5166015625,
      "loss": 0.3145,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.531319618225098,
      "rewards/margins": 2.4433531761169434,
      "rewards/rejected": -6.974671840667725,
      "step": 6210
    },
    {
      "epoch": 1.3461496804938806,
      "grad_norm": 15.25,
      "learning_rate": 1.0419907397863385e-05,
      "logits/chosen": -3.0467348098754883,
      "logits/rejected": -3.036858081817627,
      "logps/chosen": -297.4718017578125,
      "logps/rejected": -360.32598876953125,
      "loss": 0.2316,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -4.383322715759277,
      "rewards/margins": 2.8837356567382812,
      "rewards/rejected": -7.2670578956604,
      "step": 6215
    },
    {
      "epoch": 1.3472327520849128,
      "grad_norm": 21.75,
      "learning_rate": 1.0413619252942895e-05,
      "logits/chosen": -3.003507137298584,
      "logits/rejected": -3.0198495388031006,
      "logps/chosen": -327.5364074707031,
      "logps/rejected": -370.81787109375,
      "loss": 0.2548,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.773963928222656,
      "rewards/margins": 2.8295817375183105,
      "rewards/rejected": -7.603545188903809,
      "step": 6220
    },
    {
      "epoch": 1.348315823675945,
      "grad_norm": 26.0,
      "learning_rate": 1.0407329181599289e-05,
      "logits/chosen": -3.0448946952819824,
      "logits/rejected": -3.134932041168213,
      "logps/chosen": -322.92572021484375,
      "logps/rejected": -358.35418701171875,
      "loss": 0.3328,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.716402053833008,
      "rewards/margins": 2.5825116634368896,
      "rewards/rejected": -7.298914432525635,
      "step": 6225
    },
    {
      "epoch": 1.349398895266977,
      "grad_norm": 18.625,
      "learning_rate": 1.0401037192800942e-05,
      "logits/chosen": -3.0836129188537598,
      "logits/rejected": -3.0890040397644043,
      "logps/chosen": -328.3659973144531,
      "logps/rejected": -375.02996826171875,
      "loss": 0.2323,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.211712837219238,
      "rewards/margins": 2.840402126312256,
      "rewards/rejected": -8.052114486694336,
      "step": 6230
    },
    {
      "epoch": 1.3504819668580093,
      "grad_norm": 17.625,
      "learning_rate": 1.039474329551896e-05,
      "logits/chosen": -3.136312484741211,
      "logits/rejected": -3.1804442405700684,
      "logps/chosen": -311.181884765625,
      "logps/rejected": -367.85772705078125,
      "loss": 0.2372,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -5.3709330558776855,
      "rewards/margins": 2.9869086742401123,
      "rewards/rejected": -8.357841491699219,
      "step": 6235
    },
    {
      "epoch": 1.3515650384490414,
      "grad_norm": 21.5,
      "learning_rate": 1.0388447498727164e-05,
      "logits/chosen": -3.161449909210205,
      "logits/rejected": -2.983397960662842,
      "logps/chosen": -309.5649108886719,
      "logps/rejected": -373.44744873046875,
      "loss": 0.2366,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.9461541175842285,
      "rewards/margins": 3.1183462142944336,
      "rewards/rejected": -8.064499855041504,
      "step": 6240
    },
    {
      "epoch": 1.3526481100400736,
      "grad_norm": 15.875,
      "learning_rate": 1.0382149811402094e-05,
      "logits/chosen": -3.174330472946167,
      "logits/rejected": -3.1142027378082275,
      "logps/chosen": -316.9427490234375,
      "logps/rejected": -367.22442626953125,
      "loss": 0.2383,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -4.937924385070801,
      "rewards/margins": 2.724611759185791,
      "rewards/rejected": -7.662535667419434,
      "step": 6245
    },
    {
      "epoch": 1.353731181631106,
      "grad_norm": 17.875,
      "learning_rate": 1.0375850242522978e-05,
      "logits/chosen": -3.1190381050109863,
      "logits/rejected": -3.0978846549987793,
      "logps/chosen": -311.3733215332031,
      "logps/rejected": -371.2100830078125,
      "loss": 0.2361,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.643267631530762,
      "rewards/margins": 2.982309341430664,
      "rewards/rejected": -7.625576019287109,
      "step": 6250
    },
    {
      "epoch": 1.354814253222138,
      "grad_norm": 34.5,
      "learning_rate": 1.0369548801071727e-05,
      "logits/chosen": -3.145299196243286,
      "logits/rejected": -3.0685622692108154,
      "logps/chosen": -327.93524169921875,
      "logps/rejected": -379.909423828125,
      "loss": 0.2592,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.785292625427246,
      "rewards/margins": 3.0452170372009277,
      "rewards/rejected": -7.830510139465332,
      "step": 6255
    },
    {
      "epoch": 1.3558973248131703,
      "grad_norm": 17.0,
      "learning_rate": 1.0363245496032925e-05,
      "logits/chosen": -3.187196969985962,
      "logits/rejected": -3.2231571674346924,
      "logps/chosen": -322.618896484375,
      "logps/rejected": -383.9452209472656,
      "loss": 0.214,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -4.909748077392578,
      "rewards/margins": 3.2038750648498535,
      "rewards/rejected": -8.113622665405273,
      "step": 6260
    },
    {
      "epoch": 1.3569803964042024,
      "grad_norm": 24.25,
      "learning_rate": 1.0356940336393814e-05,
      "logits/chosen": -3.0175445079803467,
      "logits/rejected": -2.952637195587158,
      "logps/chosen": -337.2830810546875,
      "logps/rejected": -398.1259460449219,
      "loss": 0.2602,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.030255317687988,
      "rewards/margins": 2.7670724391937256,
      "rewards/rejected": -7.797327995300293,
      "step": 6265
    },
    {
      "epoch": 1.3580634679952346,
      "grad_norm": 21.0,
      "learning_rate": 1.0350633331144272e-05,
      "logits/chosen": -3.1500964164733887,
      "logits/rejected": -3.1534087657928467,
      "logps/chosen": -320.10394287109375,
      "logps/rejected": -372.7421875,
      "loss": 0.2814,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.1551313400268555,
      "rewards/margins": 2.702780246734619,
      "rewards/rejected": -7.857911586761475,
      "step": 6270
    },
    {
      "epoch": 1.3591465395862667,
      "grad_norm": 23.0,
      "learning_rate": 1.0344324489276821e-05,
      "logits/chosen": -3.204277515411377,
      "logits/rejected": -3.2856247425079346,
      "logps/chosen": -337.4677429199219,
      "logps/rejected": -374.2984619140625,
      "loss": 0.2879,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.204538345336914,
      "rewards/margins": 2.637478828430176,
      "rewards/rejected": -7.84201717376709,
      "step": 6275
    },
    {
      "epoch": 1.360229611177299,
      "grad_norm": 32.75,
      "learning_rate": 1.0338013819786588e-05,
      "logits/chosen": -3.0984253883361816,
      "logits/rejected": -3.197808027267456,
      "logps/chosen": -328.1217346191406,
      "logps/rejected": -371.6477966308594,
      "loss": 0.2644,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.780097484588623,
      "rewards/margins": 2.767369270324707,
      "rewards/rejected": -7.547467231750488,
      "step": 6280
    },
    {
      "epoch": 1.361312682768331,
      "grad_norm": 16.375,
      "learning_rate": 1.033170133167132e-05,
      "logits/chosen": -3.294830322265625,
      "logits/rejected": -3.223712205886841,
      "logps/chosen": -335.594970703125,
      "logps/rejected": -377.6457824707031,
      "loss": 0.2503,
      "rewards/accuracies": 0.8916667699813843,
      "rewards/chosen": -5.638089656829834,
      "rewards/margins": 2.623359203338623,
      "rewards/rejected": -8.261448860168457,
      "step": 6285
    },
    {
      "epoch": 1.3623957543593632,
      "grad_norm": 11.375,
      "learning_rate": 1.0325387033931346e-05,
      "logits/chosen": -3.174211263656616,
      "logits/rejected": -3.2436108589172363,
      "logps/chosen": -342.1929626464844,
      "logps/rejected": -391.52166748046875,
      "loss": 0.2747,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.240675449371338,
      "rewards/margins": 3.1578924655914307,
      "rewards/rejected": -8.398568153381348,
      "step": 6290
    },
    {
      "epoch": 1.3634788259503954,
      "grad_norm": 17.625,
      "learning_rate": 1.0319070935569578e-05,
      "logits/chosen": -3.15863037109375,
      "logits/rejected": -3.098733425140381,
      "logps/chosen": -340.1280517578125,
      "logps/rejected": -390.55804443359375,
      "loss": 0.2836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.390357971191406,
      "rewards/margins": 2.7144012451171875,
      "rewards/rejected": -8.104758262634277,
      "step": 6295
    },
    {
      "epoch": 1.3645618975414275,
      "grad_norm": 14.5625,
      "learning_rate": 1.0312753045591498e-05,
      "logits/chosen": -3.1805202960968018,
      "logits/rejected": -3.1500096321105957,
      "logps/chosen": -330.9845275878906,
      "logps/rejected": -377.0622253417969,
      "loss": 0.2556,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.414775371551514,
      "rewards/margins": 2.6838057041168213,
      "rewards/rejected": -8.098581314086914,
      "step": 6300
    },
    {
      "epoch": 1.3656449691324597,
      "grad_norm": 29.875,
      "learning_rate": 1.0306433373005143e-05,
      "logits/chosen": -3.118060350418091,
      "logits/rejected": -3.076564311981201,
      "logps/chosen": -324.40899658203125,
      "logps/rejected": -374.7588806152344,
      "loss": 0.2998,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.26946496963501,
      "rewards/margins": 2.775048017501831,
      "rewards/rejected": -8.044512748718262,
      "step": 6305
    },
    {
      "epoch": 1.3667280407234919,
      "grad_norm": 17.0,
      "learning_rate": 1.0300111926821086e-05,
      "logits/chosen": -2.992932081222534,
      "logits/rejected": -2.935206890106201,
      "logps/chosen": -314.10467529296875,
      "logps/rejected": -383.8005676269531,
      "loss": 0.2246,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -4.8480119705200195,
      "rewards/margins": 2.906261920928955,
      "rewards/rejected": -7.754273891448975,
      "step": 6310
    },
    {
      "epoch": 1.367811112314524,
      "grad_norm": 17.625,
      "learning_rate": 1.0293788716052433e-05,
      "logits/chosen": -3.203254222869873,
      "logits/rejected": -3.1960396766662598,
      "logps/chosen": -323.8249816894531,
      "logps/rejected": -378.65509033203125,
      "loss": 0.2365,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.469396114349365,
      "rewards/margins": 2.990618944168091,
      "rewards/rejected": -8.460015296936035,
      "step": 6315
    },
    {
      "epoch": 1.3688941839055562,
      "grad_norm": 20.25,
      "learning_rate": 1.0287463749714806e-05,
      "logits/chosen": -3.0903470516204834,
      "logits/rejected": -3.074910879135132,
      "logps/chosen": -314.44757080078125,
      "logps/rejected": -361.234619140625,
      "loss": 0.2697,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.75827693939209,
      "rewards/margins": 2.538146495819092,
      "rewards/rejected": -7.296423435211182,
      "step": 6320
    },
    {
      "epoch": 1.3699772554965883,
      "grad_norm": 25.375,
      "learning_rate": 1.028113703682633e-05,
      "logits/chosen": -3.172330617904663,
      "logits/rejected": -3.2907555103302,
      "logps/chosen": -295.919677734375,
      "logps/rejected": -352.119140625,
      "loss": 0.2677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.456972599029541,
      "rewards/margins": 2.94743275642395,
      "rewards/rejected": -7.404405117034912,
      "step": 6325
    },
    {
      "epoch": 1.3710603270876205,
      "grad_norm": 24.75,
      "learning_rate": 1.0274808586407615e-05,
      "logits/chosen": -3.0538134574890137,
      "logits/rejected": -3.0002975463867188,
      "logps/chosen": -316.48712158203125,
      "logps/rejected": -370.37957763671875,
      "loss": 0.2864,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.674925804138184,
      "rewards/margins": 2.5109922885894775,
      "rewards/rejected": -7.185917854309082,
      "step": 6330
    },
    {
      "epoch": 1.3721433986786526,
      "grad_norm": 24.625,
      "learning_rate": 1.026847840748176e-05,
      "logits/chosen": -3.131197452545166,
      "logits/rejected": -3.2179324626922607,
      "logps/chosen": -310.97540283203125,
      "logps/rejected": -352.1275329589844,
      "loss": 0.2343,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.528408050537109,
      "rewards/margins": 2.8413357734680176,
      "rewards/rejected": -7.369743347167969,
      "step": 6335
    },
    {
      "epoch": 1.3732264702696848,
      "grad_norm": 21.125,
      "learning_rate": 1.0262146509074317e-05,
      "logits/chosen": -3.1680843830108643,
      "logits/rejected": -3.1491901874542236,
      "logps/chosen": -322.6315002441406,
      "logps/rejected": -366.0670166015625,
      "loss": 0.2354,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.085851192474365,
      "rewards/margins": 2.788428783416748,
      "rewards/rejected": -7.874279975891113,
      "step": 6340
    },
    {
      "epoch": 1.374309541860717,
      "grad_norm": 16.625,
      "learning_rate": 1.0255812900213295e-05,
      "logits/chosen": -3.2067954540252686,
      "logits/rejected": -3.1469414234161377,
      "logps/chosen": -328.2745056152344,
      "logps/rejected": -383.2586975097656,
      "loss": 0.2668,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.391452312469482,
      "rewards/margins": 2.890854835510254,
      "rewards/rejected": -8.282307624816895,
      "step": 6345
    },
    {
      "epoch": 1.3753926134517491,
      "grad_norm": 22.375,
      "learning_rate": 1.0249477589929138e-05,
      "logits/chosen": -3.213787078857422,
      "logits/rejected": -3.2210731506347656,
      "logps/chosen": -326.5201721191406,
      "logps/rejected": -388.22808837890625,
      "loss": 0.1952,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.606820106506348,
      "rewards/margins": 3.0073485374450684,
      "rewards/rejected": -8.614169120788574,
      "step": 6350
    },
    {
      "epoch": 1.3764756850427813,
      "grad_norm": 30.25,
      "learning_rate": 1.0243140587254723e-05,
      "logits/chosen": -3.2146358489990234,
      "logits/rejected": -3.25372052192688,
      "logps/chosen": -325.6421813964844,
      "logps/rejected": -371.33477783203125,
      "loss": 0.2942,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.20993709564209,
      "rewards/margins": 2.3742167949676514,
      "rewards/rejected": -7.584153652191162,
      "step": 6355
    },
    {
      "epoch": 1.3775587566338134,
      "grad_norm": 25.875,
      "learning_rate": 1.0236801901225334e-05,
      "logits/chosen": -3.1271474361419678,
      "logits/rejected": -3.212050199508667,
      "logps/chosen": -334.38104248046875,
      "logps/rejected": -383.27569580078125,
      "loss": 0.2606,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.075222969055176,
      "rewards/margins": 3.2338337898254395,
      "rewards/rejected": -8.30905532836914,
      "step": 6360
    },
    {
      "epoch": 1.3786418282248456,
      "grad_norm": 22.875,
      "learning_rate": 1.0230461540878655e-05,
      "logits/chosen": -3.06294846534729,
      "logits/rejected": -3.1647067070007324,
      "logps/chosen": -340.28289794921875,
      "logps/rejected": -379.3400573730469,
      "loss": 0.2704,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.060542106628418,
      "rewards/margins": 2.8624510765075684,
      "rewards/rejected": -7.9229936599731445,
      "step": 6365
    },
    {
      "epoch": 1.3797248998158778,
      "grad_norm": 23.5,
      "learning_rate": 1.0224119515254761e-05,
      "logits/chosen": -3.1181674003601074,
      "logits/rejected": -3.1057257652282715,
      "logps/chosen": -328.48748779296875,
      "logps/rejected": -376.4448547363281,
      "loss": 0.2533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.15308141708374,
      "rewards/margins": 2.581195592880249,
      "rewards/rejected": -7.734278202056885,
      "step": 6370
    },
    {
      "epoch": 1.38080797140691,
      "grad_norm": 9.625,
      "learning_rate": 1.02177758333961e-05,
      "logits/chosen": -2.968372106552124,
      "logits/rejected": -3.0150744915008545,
      "logps/chosen": -345.1783752441406,
      "logps/rejected": -393.9994812011719,
      "loss": 0.2611,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.9308977127075195,
      "rewards/margins": 3.0389528274536133,
      "rewards/rejected": -7.969851016998291,
      "step": 6375
    },
    {
      "epoch": 1.381891042997942,
      "grad_norm": 26.75,
      "learning_rate": 1.0211430504347478e-05,
      "logits/chosen": -3.142197370529175,
      "logits/rejected": -3.1153724193573,
      "logps/chosen": -322.02313232421875,
      "logps/rejected": -373.09967041015625,
      "loss": 0.3413,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -5.184882164001465,
      "rewards/margins": 2.166511058807373,
      "rewards/rejected": -7.351393699645996,
      "step": 6380
    },
    {
      "epoch": 1.3829741145889742,
      "grad_norm": 28.875,
      "learning_rate": 1.0205083537156057e-05,
      "logits/chosen": -3.1337733268737793,
      "logits/rejected": -3.050189256668091,
      "logps/chosen": -300.73992919921875,
      "logps/rejected": -364.0346374511719,
      "loss": 0.3553,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.484007358551025,
      "rewards/margins": 2.50458025932312,
      "rewards/rejected": -6.988587856292725,
      "step": 6385
    },
    {
      "epoch": 1.3840571861800064,
      "grad_norm": 21.5,
      "learning_rate": 1.0198734940871324e-05,
      "logits/chosen": -3.07379150390625,
      "logits/rejected": -2.975837469100952,
      "logps/chosen": -314.924560546875,
      "logps/rejected": -374.07672119140625,
      "loss": 0.3491,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.142820835113525,
      "rewards/margins": 2.372718572616577,
      "rewards/rejected": -7.515539646148682,
      "step": 6390
    },
    {
      "epoch": 1.3851402577710386,
      "grad_norm": 18.125,
      "learning_rate": 1.01923847245451e-05,
      "logits/chosen": -2.9436838626861572,
      "logits/rejected": -2.9099957942962646,
      "logps/chosen": -331.5326232910156,
      "logps/rejected": -386.37542724609375,
      "loss": 0.2723,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.924625396728516,
      "rewards/margins": 2.760284185409546,
      "rewards/rejected": -7.684910774230957,
      "step": 6395
    },
    {
      "epoch": 1.3862233293620707,
      "grad_norm": 16.25,
      "learning_rate": 1.0186032897231509e-05,
      "logits/chosen": -3.116699695587158,
      "logits/rejected": -3.1400630474090576,
      "logps/chosen": -305.48614501953125,
      "logps/rejected": -340.1701965332031,
      "loss": 0.2792,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -4.22272253036499,
      "rewards/margins": 2.7567498683929443,
      "rewards/rejected": -6.9794721603393555,
      "step": 6400
    },
    {
      "epoch": 1.3873064009531029,
      "grad_norm": 20.0,
      "learning_rate": 1.0179679467986975e-05,
      "logits/chosen": -3.0715906620025635,
      "logits/rejected": -2.9928178787231445,
      "logps/chosen": -307.9637756347656,
      "logps/rejected": -374.84210205078125,
      "loss": 0.2963,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.558981895446777,
      "rewards/margins": 2.5715789794921875,
      "rewards/rejected": -7.130561828613281,
      "step": 6405
    },
    {
      "epoch": 1.3883894725441353,
      "grad_norm": 19.75,
      "learning_rate": 1.01733244458702e-05,
      "logits/chosen": -3.069126844406128,
      "logits/rejected": -2.858613967895508,
      "logps/chosen": -316.3505554199219,
      "logps/rejected": -385.1528625488281,
      "loss": 0.2775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.776829719543457,
      "rewards/margins": 2.5650973320007324,
      "rewards/rejected": -7.341927528381348,
      "step": 6410
    },
    {
      "epoch": 1.3894725441351674,
      "grad_norm": 19.25,
      "learning_rate": 1.0166967839942168e-05,
      "logits/chosen": -3.221590518951416,
      "logits/rejected": -3.1429622173309326,
      "logps/chosen": -308.0515441894531,
      "logps/rejected": -355.439453125,
      "loss": 0.3014,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -4.804394721984863,
      "rewards/margins": 2.4329166412353516,
      "rewards/rejected": -7.237312316894531,
      "step": 6415
    },
    {
      "epoch": 1.3905556157261996,
      "grad_norm": 17.0,
      "learning_rate": 1.0160609659266107e-05,
      "logits/chosen": -3.0534827709198,
      "logits/rejected": -3.1773366928100586,
      "logps/chosen": -315.1832580566406,
      "logps/rejected": -355.71624755859375,
      "loss": 0.2246,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.895639896392822,
      "rewards/margins": 2.9344773292541504,
      "rewards/rejected": -7.830118656158447,
      "step": 6420
    },
    {
      "epoch": 1.3916386873172317,
      "grad_norm": 22.75,
      "learning_rate": 1.0154249912907505e-05,
      "logits/chosen": -3.0032150745391846,
      "logits/rejected": -2.9852092266082764,
      "logps/chosen": -326.09759521484375,
      "logps/rejected": -376.2630615234375,
      "loss": 0.2569,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.2761969566345215,
      "rewards/margins": 2.635972499847412,
      "rewards/rejected": -7.912168979644775,
      "step": 6425
    },
    {
      "epoch": 1.3927217589082639,
      "grad_norm": 19.375,
      "learning_rate": 1.0147888609934075e-05,
      "logits/chosen": -3.1505069732666016,
      "logits/rejected": -3.144770860671997,
      "logps/chosen": -341.9070739746094,
      "logps/rejected": -383.01568603515625,
      "loss": 0.2347,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.78622579574585,
      "rewards/margins": 2.7167398929595947,
      "rewards/rejected": -8.502964973449707,
      "step": 6430
    },
    {
      "epoch": 1.393804830499296,
      "grad_norm": 32.25,
      "learning_rate": 1.0141525759415749e-05,
      "logits/chosen": -3.2509818077087402,
      "logits/rejected": -3.113563060760498,
      "logps/chosen": -323.01715087890625,
      "logps/rejected": -362.99456787109375,
      "loss": 0.3066,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -5.4922990798950195,
      "rewards/margins": 2.3959403038024902,
      "rewards/rejected": -7.888239860534668,
      "step": 6435
    },
    {
      "epoch": 1.3948879020903282,
      "grad_norm": 15.5625,
      "learning_rate": 1.0135161370424663e-05,
      "logits/chosen": -3.196747303009033,
      "logits/rejected": -3.163665771484375,
      "logps/chosen": -314.0989685058594,
      "logps/rejected": -358.50439453125,
      "loss": 0.279,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.949930191040039,
      "rewards/margins": 2.4744627475738525,
      "rewards/rejected": -7.424391746520996,
      "step": 6440
    },
    {
      "epoch": 1.3959709736813604,
      "grad_norm": 34.0,
      "learning_rate": 1.0128795452035153e-05,
      "logits/chosen": -3.1333296298980713,
      "logits/rejected": -3.0821750164031982,
      "logps/chosen": -317.8971862792969,
      "logps/rejected": -372.0641174316406,
      "loss": 0.2983,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.33318567276001,
      "rewards/margins": 2.8096697330474854,
      "rewards/rejected": -8.142855644226074,
      "step": 6445
    },
    {
      "epoch": 1.3970540452723925,
      "grad_norm": 29.125,
      "learning_rate": 1.0122428013323733e-05,
      "logits/chosen": -3.0514702796936035,
      "logits/rejected": -3.0396995544433594,
      "logps/chosen": -338.3207702636719,
      "logps/rejected": -373.91632080078125,
      "loss": 0.3316,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.278558731079102,
      "rewards/margins": 2.319575071334839,
      "rewards/rejected": -7.598133087158203,
      "step": 6450
    },
    {
      "epoch": 1.3981371168634247,
      "grad_norm": 16.75,
      "learning_rate": 1.0116059063369087e-05,
      "logits/chosen": -3.0831422805786133,
      "logits/rejected": -3.1514015197753906,
      "logps/chosen": -318.0684814453125,
      "logps/rejected": -353.5467529296875,
      "loss": 0.2821,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -4.7872700691223145,
      "rewards/margins": 2.7464444637298584,
      "rewards/rejected": -7.53371524810791,
      "step": 6455
    },
    {
      "epoch": 1.3992201884544568,
      "grad_norm": 17.75,
      "learning_rate": 1.0109688611252046e-05,
      "logits/chosen": -3.023886203765869,
      "logits/rejected": -2.939612865447998,
      "logps/chosen": -315.62689208984375,
      "logps/rejected": -368.85504150390625,
      "loss": 0.2521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.164239406585693,
      "rewards/margins": 2.824399948120117,
      "rewards/rejected": -6.988638877868652,
      "step": 6460
    },
    {
      "epoch": 1.400303260045489,
      "grad_norm": 15.9375,
      "learning_rate": 1.0103316666055587e-05,
      "logits/chosen": -3.0556883811950684,
      "logits/rejected": -3.04371976852417,
      "logps/chosen": -323.9380187988281,
      "logps/rejected": -362.9095764160156,
      "loss": 0.1919,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.710166931152344,
      "rewards/margins": 3.0439651012420654,
      "rewards/rejected": -7.754131317138672,
      "step": 6465
    },
    {
      "epoch": 1.4013863316365212,
      "grad_norm": 24.375,
      "learning_rate": 1.0096943236864823e-05,
      "logits/chosen": -2.905780792236328,
      "logits/rejected": -3.042336940765381,
      "logps/chosen": -336.3863220214844,
      "logps/rejected": -387.5533447265625,
      "loss": 0.2268,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.445866584777832,
      "rewards/margins": 2.9071810245513916,
      "rewards/rejected": -8.353047370910645,
      "step": 6470
    },
    {
      "epoch": 1.4024694032275533,
      "grad_norm": 23.5,
      "learning_rate": 1.0090568332766973e-05,
      "logits/chosen": -3.1349644660949707,
      "logits/rejected": -2.9797186851501465,
      "logps/chosen": -313.53460693359375,
      "logps/rejected": -368.92926025390625,
      "loss": 0.2904,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.798738479614258,
      "rewards/margins": 2.472109317779541,
      "rewards/rejected": -7.270848274230957,
      "step": 6475
    },
    {
      "epoch": 1.4035524748185855,
      "grad_norm": 20.75,
      "learning_rate": 1.008419196285136e-05,
      "logits/chosen": -3.2333004474639893,
      "logits/rejected": -3.219202756881714,
      "logps/chosen": -322.06390380859375,
      "logps/rejected": -370.7413330078125,
      "loss": 0.2558,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.438675880432129,
      "rewards/margins": 2.747386932373047,
      "rewards/rejected": -8.186063766479492,
      "step": 6480
    },
    {
      "epoch": 1.4046355464096176,
      "grad_norm": 20.375,
      "learning_rate": 1.0077814136209402e-05,
      "logits/chosen": -2.870008945465088,
      "logits/rejected": -3.028120517730713,
      "logps/chosen": -335.29974365234375,
      "logps/rejected": -391.85845947265625,
      "loss": 0.2759,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.402560234069824,
      "rewards/margins": 2.944777727127075,
      "rewards/rejected": -8.34733772277832,
      "step": 6485
    },
    {
      "epoch": 1.4057186180006498,
      "grad_norm": 15.6875,
      "learning_rate": 1.007143486193459e-05,
      "logits/chosen": -3.1518614292144775,
      "logits/rejected": -3.277221202850342,
      "logps/chosen": -329.1944885253906,
      "logps/rejected": -372.53668212890625,
      "loss": 0.2395,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.513443946838379,
      "rewards/margins": 2.9066855907440186,
      "rewards/rejected": -8.420129776000977,
      "step": 6490
    },
    {
      "epoch": 1.406801689591682,
      "grad_norm": 13.1875,
      "learning_rate": 1.0065054149122484e-05,
      "logits/chosen": -3.110258102416992,
      "logits/rejected": -3.002289295196533,
      "logps/chosen": -299.7094421386719,
      "logps/rejected": -357.091064453125,
      "loss": 0.2129,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.64835786819458,
      "rewards/margins": 2.8854830265045166,
      "rewards/rejected": -7.533840179443359,
      "step": 6495
    },
    {
      "epoch": 1.407884761182714,
      "grad_norm": 16.75,
      "learning_rate": 1.0058672006870687e-05,
      "logits/chosen": -3.026257038116455,
      "logits/rejected": -3.0440454483032227,
      "logps/chosen": -337.65045166015625,
      "logps/rejected": -391.91571044921875,
      "loss": 0.2467,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.294532775878906,
      "rewards/margins": 3.0808653831481934,
      "rewards/rejected": -8.375397682189941,
      "step": 6500
    },
    {
      "epoch": 1.407884761182714,
      "eval_logits/chosen": -3.138395309448242,
      "eval_logits/rejected": -3.067551374435425,
      "eval_logps/chosen": -311.17926025390625,
      "eval_logps/rejected": -354.2960205078125,
      "eval_loss": 0.4409726858139038,
      "eval_rewards/accuracies": 0.800000011920929,
      "eval_rewards/chosen": -4.8488993644714355,
      "eval_rewards/margins": 1.983046531677246,
      "eval_rewards/rejected": -6.831945419311523,
      "eval_runtime": 14.7941,
      "eval_samples_per_second": 13.519,
      "eval_steps_per_second": 13.519,
      "step": 6500
    },
    {
      "epoch": 1.4089678327737463,
      "grad_norm": 20.5,
      "learning_rate": 1.005228844427885e-05,
      "logits/chosen": -3.107631206512451,
      "logits/rejected": -3.087475538253784,
      "logps/chosen": -310.6731262207031,
      "logps/rejected": -366.03814697265625,
      "loss": 0.2358,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.639066219329834,
      "rewards/margins": 3.234971523284912,
      "rewards/rejected": -7.874037742614746,
      "step": 6505
    },
    {
      "epoch": 1.4100509043647786,
      "grad_norm": 27.875,
      "learning_rate": 1.0045903470448638e-05,
      "logits/chosen": -2.9398787021636963,
      "logits/rejected": -2.9564929008483887,
      "logps/chosen": -329.48577880859375,
      "logps/rejected": -397.5646667480469,
      "loss": 0.1923,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.076545715332031,
      "rewards/margins": 3.5204761028289795,
      "rewards/rejected": -8.597021102905273,
      "step": 6510
    },
    {
      "epoch": 1.4111339759558108,
      "grad_norm": 27.0,
      "learning_rate": 1.0039517094483734e-05,
      "logits/chosen": -3.1200623512268066,
      "logits/rejected": -3.173692226409912,
      "logps/chosen": -313.0102233886719,
      "logps/rejected": -354.974853515625,
      "loss": 0.2678,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.910219669342041,
      "rewards/margins": 2.6408421993255615,
      "rewards/rejected": -7.551061153411865,
      "step": 6515
    },
    {
      "epoch": 1.412217047546843,
      "grad_norm": 22.25,
      "learning_rate": 1.0033129325489827e-05,
      "logits/chosen": -3.1238291263580322,
      "logits/rejected": -3.1048970222473145,
      "logps/chosen": -340.2206115722656,
      "logps/rejected": -391.7366943359375,
      "loss": 0.2769,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.603354454040527,
      "rewards/margins": 2.6920077800750732,
      "rewards/rejected": -8.29536247253418,
      "step": 6520
    },
    {
      "epoch": 1.4133001191378751,
      "grad_norm": 29.125,
      "learning_rate": 1.0026740172574575e-05,
      "logits/chosen": -3.1506729125976562,
      "logits/rejected": -3.2528679370880127,
      "logps/chosen": -317.2520446777344,
      "logps/rejected": -363.11126708984375,
      "loss": 0.3158,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -5.313124179840088,
      "rewards/margins": 2.369401216506958,
      "rewards/rejected": -7.682525634765625,
      "step": 6525
    },
    {
      "epoch": 1.4143831907289073,
      "grad_norm": 20.875,
      "learning_rate": 1.002034964484763e-05,
      "logits/chosen": -3.2330055236816406,
      "logits/rejected": -3.155683994293213,
      "logps/chosen": -311.0859375,
      "logps/rejected": -365.8779602050781,
      "loss": 0.3022,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.092386722564697,
      "rewards/margins": 2.8344388008117676,
      "rewards/rejected": -7.926825523376465,
      "step": 6530
    },
    {
      "epoch": 1.4154662623199394,
      "grad_norm": 29.5,
      "learning_rate": 1.0013957751420583e-05,
      "logits/chosen": -3.04109525680542,
      "logits/rejected": -3.1344056129455566,
      "logps/chosen": -330.28131103515625,
      "logps/rejected": -372.97222900390625,
      "loss": 0.2541,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.537692546844482,
      "rewards/margins": 2.6065402030944824,
      "rewards/rejected": -8.144231796264648,
      "step": 6535
    },
    {
      "epoch": 1.4165493339109716,
      "grad_norm": 11.75,
      "learning_rate": 1.0007564501406993e-05,
      "logits/chosen": -3.029484272003174,
      "logits/rejected": -3.0991687774658203,
      "logps/chosen": -348.01904296875,
      "logps/rejected": -399.304931640625,
      "loss": 0.2205,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.322739601135254,
      "rewards/margins": 3.3354568481445312,
      "rewards/rejected": -8.658197402954102,
      "step": 6540
    },
    {
      "epoch": 1.4176324055020038,
      "grad_norm": 31.0,
      "learning_rate": 1.0001169903922335e-05,
      "logits/chosen": -2.9197354316711426,
      "logits/rejected": -2.8940930366516113,
      "logps/chosen": -338.9026794433594,
      "logps/rejected": -380.46124267578125,
      "loss": 0.3518,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.180726051330566,
      "rewards/margins": 2.629176616668701,
      "rewards/rejected": -7.809902191162109,
      "step": 6545
    },
    {
      "epoch": 1.418715477093036,
      "grad_norm": 21.75,
      "learning_rate": 9.994773968084019e-06,
      "logits/chosen": -3.1522135734558105,
      "logits/rejected": -3.042480230331421,
      "logps/chosen": -306.677490234375,
      "logps/rejected": -356.11566162109375,
      "loss": 0.2858,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.104304313659668,
      "rewards/margins": 2.524052143096924,
      "rewards/rejected": -7.628355979919434,
      "step": 6550
    },
    {
      "epoch": 1.419798548684068,
      "grad_norm": 20.625,
      "learning_rate": 9.988376703011353e-06,
      "logits/chosen": -3.017210006713867,
      "logits/rejected": -3.059549570083618,
      "logps/chosen": -315.18231201171875,
      "logps/rejected": -368.90191650390625,
      "loss": 0.2303,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.05557918548584,
      "rewards/margins": 2.929673671722412,
      "rewards/rejected": -7.98525333404541,
      "step": 6555
    },
    {
      "epoch": 1.4208816202751002,
      "grad_norm": 17.125,
      "learning_rate": 9.981978117825544e-06,
      "logits/chosen": -3.0560507774353027,
      "logits/rejected": -3.020932674407959,
      "logps/chosen": -320.1767578125,
      "logps/rejected": -371.28753662109375,
      "loss": 0.2571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.100651741027832,
      "rewards/margins": 2.8494603633880615,
      "rewards/rejected": -7.950112819671631,
      "step": 6560
    },
    {
      "epoch": 1.4219646918661324,
      "grad_norm": 26.625,
      "learning_rate": 9.975578221649687e-06,
      "logits/chosen": -2.9872453212738037,
      "logits/rejected": -2.9978251457214355,
      "logps/chosen": -321.91436767578125,
      "logps/rejected": -375.6042175292969,
      "loss": 0.2568,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.863023281097412,
      "rewards/margins": 3.0086286067962646,
      "rewards/rejected": -7.871651649475098,
      "step": 6565
    },
    {
      "epoch": 1.4230477634571645,
      "grad_norm": 30.25,
      "learning_rate": 9.96917702360874e-06,
      "logits/chosen": -3.0774805545806885,
      "logits/rejected": -3.140839099884033,
      "logps/chosen": -324.61187744140625,
      "logps/rejected": -367.0957946777344,
      "loss": 0.3014,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -5.633880615234375,
      "rewards/margins": 2.6572813987731934,
      "rewards/rejected": -8.291162490844727,
      "step": 6570
    },
    {
      "epoch": 1.4241308350481967,
      "grad_norm": 23.375,
      "learning_rate": 9.962774532829514e-06,
      "logits/chosen": -2.9487414360046387,
      "logits/rejected": -2.9561288356781006,
      "logps/chosen": -345.681396484375,
      "logps/rejected": -394.67755126953125,
      "loss": 0.3088,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.585224628448486,
      "rewards/margins": 2.734950542449951,
      "rewards/rejected": -8.320175170898438,
      "step": 6575
    },
    {
      "epoch": 1.4252139066392289,
      "grad_norm": 12.625,
      "learning_rate": 9.95637075844067e-06,
      "logits/chosen": -3.0579824447631836,
      "logits/rejected": -3.0473389625549316,
      "logps/chosen": -317.33221435546875,
      "logps/rejected": -356.14666748046875,
      "loss": 0.2636,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.292137622833252,
      "rewards/margins": 2.65661358833313,
      "rewards/rejected": -7.948751926422119,
      "step": 6580
    },
    {
      "epoch": 1.426296978230261,
      "grad_norm": 32.25,
      "learning_rate": 9.949965709572699e-06,
      "logits/chosen": -2.9639928340911865,
      "logits/rejected": -2.922116756439209,
      "logps/chosen": -322.3084716796875,
      "logps/rejected": -372.1573486328125,
      "loss": 0.333,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.543591022491455,
      "rewards/margins": 2.4829680919647217,
      "rewards/rejected": -8.026558876037598,
      "step": 6585
    },
    {
      "epoch": 1.4273800498212932,
      "grad_norm": 13.375,
      "learning_rate": 9.943559395357904e-06,
      "logits/chosen": -2.921011209487915,
      "logits/rejected": -2.950202465057373,
      "logps/chosen": -328.94281005859375,
      "logps/rejected": -383.89398193359375,
      "loss": 0.2305,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.748935699462891,
      "rewards/margins": 3.1916873455047607,
      "rewards/rejected": -7.9406232833862305,
      "step": 6590
    },
    {
      "epoch": 1.4284631214123253,
      "grad_norm": 23.875,
      "learning_rate": 9.937151824930393e-06,
      "logits/chosen": -3.024729013442993,
      "logits/rejected": -3.074603319168091,
      "logps/chosen": -331.5130310058594,
      "logps/rejected": -386.37139892578125,
      "loss": 0.2698,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.740936756134033,
      "rewards/margins": 2.898160219192505,
      "rewards/rejected": -8.639097213745117,
      "step": 6595
    },
    {
      "epoch": 1.4295461930033575,
      "grad_norm": 20.875,
      "learning_rate": 9.930743007426074e-06,
      "logits/chosen": -3.119143009185791,
      "logits/rejected": -3.072573661804199,
      "logps/chosen": -335.49578857421875,
      "logps/rejected": -385.36553955078125,
      "loss": 0.2481,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.985945701599121,
      "rewards/margins": 2.6958158016204834,
      "rewards/rejected": -8.681760787963867,
      "step": 6600
    },
    {
      "epoch": 1.4306292645943897,
      "grad_norm": 19.875,
      "learning_rate": 9.924332951982618e-06,
      "logits/chosen": -3.0081162452697754,
      "logits/rejected": -2.967133045196533,
      "logps/chosen": -337.9064636230469,
      "logps/rejected": -391.11212158203125,
      "loss": 0.2665,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.397648811340332,
      "rewards/margins": 2.8139500617980957,
      "rewards/rejected": -8.211599349975586,
      "step": 6605
    },
    {
      "epoch": 1.4317123361854218,
      "grad_norm": 24.625,
      "learning_rate": 9.917921667739474e-06,
      "logits/chosen": -3.160097599029541,
      "logits/rejected": -3.1660666465759277,
      "logps/chosen": -321.82049560546875,
      "logps/rejected": -370.0487060546875,
      "loss": 0.3101,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.652372360229492,
      "rewards/margins": 2.1996140480041504,
      "rewards/rejected": -7.851986885070801,
      "step": 6610
    },
    {
      "epoch": 1.432795407776454,
      "grad_norm": 23.875,
      "learning_rate": 9.911509163837837e-06,
      "logits/chosen": -3.136996030807495,
      "logits/rejected": -3.1440489292144775,
      "logps/chosen": -309.23321533203125,
      "logps/rejected": -360.04022216796875,
      "loss": 0.2653,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.292004585266113,
      "rewards/margins": 2.7787718772888184,
      "rewards/rejected": -8.070775985717773,
      "step": 6615
    },
    {
      "epoch": 1.4338784793674861,
      "grad_norm": 18.125,
      "learning_rate": 9.905095449420643e-06,
      "logits/chosen": -3.19917368888855,
      "logits/rejected": -3.0151124000549316,
      "logps/chosen": -329.4129943847656,
      "logps/rejected": -395.5152893066406,
      "loss": 0.2856,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.475763320922852,
      "rewards/margins": 2.7803711891174316,
      "rewards/rejected": -8.256134033203125,
      "step": 6620
    },
    {
      "epoch": 1.4349615509585183,
      "grad_norm": 19.125,
      "learning_rate": 9.898680533632552e-06,
      "logits/chosen": -3.1650097370147705,
      "logits/rejected": -3.1352148056030273,
      "logps/chosen": -328.70648193359375,
      "logps/rejected": -377.76434326171875,
      "loss": 0.2733,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.289421081542969,
      "rewards/margins": 2.870349407196045,
      "rewards/rejected": -8.159770965576172,
      "step": 6625
    },
    {
      "epoch": 1.4360446225495505,
      "grad_norm": 29.125,
      "learning_rate": 9.892264425619936e-06,
      "logits/chosen": -3.043574810028076,
      "logits/rejected": -3.0136282444000244,
      "logps/chosen": -350.4503173828125,
      "logps/rejected": -395.36163330078125,
      "loss": 0.3139,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.418923377990723,
      "rewards/margins": 2.4928641319274902,
      "rewards/rejected": -8.911787033081055,
      "step": 6630
    },
    {
      "epoch": 1.4371276941405826,
      "grad_norm": 26.5,
      "learning_rate": 9.885847134530873e-06,
      "logits/chosen": -3.0810399055480957,
      "logits/rejected": -3.000697135925293,
      "logps/chosen": -315.0095520019531,
      "logps/rejected": -378.23443603515625,
      "loss": 0.2519,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.289761543273926,
      "rewards/margins": 2.7837650775909424,
      "rewards/rejected": -8.073527336120605,
      "step": 6635
    },
    {
      "epoch": 1.4382107657316148,
      "grad_norm": 24.0,
      "learning_rate": 9.879428669515123e-06,
      "logits/chosen": -3.06984543800354,
      "logits/rejected": -2.9397311210632324,
      "logps/chosen": -329.99871826171875,
      "logps/rejected": -387.5128479003906,
      "loss": 0.2442,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.2909836769104,
      "rewards/margins": 2.998056411743164,
      "rewards/rejected": -8.289039611816406,
      "step": 6640
    },
    {
      "epoch": 1.439293837322647,
      "grad_norm": 35.0,
      "learning_rate": 9.873009039724117e-06,
      "logits/chosen": -3.1042940616607666,
      "logits/rejected": -3.0659339427948,
      "logps/chosen": -340.75958251953125,
      "logps/rejected": -387.731201171875,
      "loss": 0.2743,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.679214954376221,
      "rewards/margins": 2.7571961879730225,
      "rewards/rejected": -8.436410903930664,
      "step": 6645
    },
    {
      "epoch": 1.440376908913679,
      "grad_norm": 24.0,
      "learning_rate": 9.866588254310955e-06,
      "logits/chosen": -3.0422403812408447,
      "logits/rejected": -3.0526368618011475,
      "logps/chosen": -325.7805480957031,
      "logps/rejected": -381.4814147949219,
      "loss": 0.2181,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.098733425140381,
      "rewards/margins": 2.6565370559692383,
      "rewards/rejected": -7.755270957946777,
      "step": 6650
    },
    {
      "epoch": 1.4414599805047112,
      "grad_norm": 23.5,
      "learning_rate": 9.860166322430377e-06,
      "logits/chosen": -3.0632128715515137,
      "logits/rejected": -2.990349531173706,
      "logps/chosen": -352.4147033691406,
      "logps/rejected": -402.93646240234375,
      "loss": 0.3181,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.442287921905518,
      "rewards/margins": 2.8208117485046387,
      "rewards/rejected": -8.263099670410156,
      "step": 6655
    },
    {
      "epoch": 1.4425430520957434,
      "grad_norm": 25.0,
      "learning_rate": 9.853743253238763e-06,
      "logits/chosen": -3.3110270500183105,
      "logits/rejected": -3.2634029388427734,
      "logps/chosen": -318.69207763671875,
      "logps/rejected": -374.0821228027344,
      "loss": 0.2641,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.713545799255371,
      "rewards/margins": 2.709125518798828,
      "rewards/rejected": -8.4226713180542,
      "step": 6660
    },
    {
      "epoch": 1.4436261236867756,
      "grad_norm": 35.75,
      "learning_rate": 9.847319055894107e-06,
      "logits/chosen": -3.202690601348877,
      "logits/rejected": -3.1490187644958496,
      "logps/chosen": -337.3077697753906,
      "logps/rejected": -395.32861328125,
      "loss": 0.3173,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -5.939998626708984,
      "rewards/margins": 2.7264115810394287,
      "rewards/rejected": -8.666410446166992,
      "step": 6665
    },
    {
      "epoch": 1.444709195277808,
      "grad_norm": 23.625,
      "learning_rate": 9.840893739556018e-06,
      "logits/chosen": -3.058971881866455,
      "logits/rejected": -3.1121134757995605,
      "logps/chosen": -342.4585266113281,
      "logps/rejected": -378.8442687988281,
      "loss": 0.2866,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.029562950134277,
      "rewards/margins": 2.612069606781006,
      "rewards/rejected": -7.641633033752441,
      "step": 6670
    },
    {
      "epoch": 1.44579226686884,
      "grad_norm": 16.5,
      "learning_rate": 9.834467313385706e-06,
      "logits/chosen": -3.1781668663024902,
      "logits/rejected": -3.2779464721679688,
      "logps/chosen": -328.47039794921875,
      "logps/rejected": -374.5408630371094,
      "loss": 0.2491,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.69696569442749,
      "rewards/margins": 2.9405202865600586,
      "rewards/rejected": -8.637486457824707,
      "step": 6675
    },
    {
      "epoch": 1.4468753384598723,
      "grad_norm": 20.125,
      "learning_rate": 9.82803978654595e-06,
      "logits/chosen": -3.046985387802124,
      "logits/rejected": -3.0592198371887207,
      "logps/chosen": -329.106201171875,
      "logps/rejected": -395.46697998046875,
      "loss": 0.2473,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.436577796936035,
      "rewards/margins": 3.220172882080078,
      "rewards/rejected": -8.656750679016113,
      "step": 6680
    },
    {
      "epoch": 1.4479584100509044,
      "grad_norm": 22.875,
      "learning_rate": 9.821611168201105e-06,
      "logits/chosen": -2.995206832885742,
      "logits/rejected": -3.027879476547241,
      "logps/chosen": -336.87518310546875,
      "logps/rejected": -382.3934020996094,
      "loss": 0.2856,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.517508029937744,
      "rewards/margins": 2.7490553855895996,
      "rewards/rejected": -8.266563415527344,
      "step": 6685
    },
    {
      "epoch": 1.4490414816419366,
      "grad_norm": 23.5,
      "learning_rate": 9.815181467517086e-06,
      "logits/chosen": -3.2574782371520996,
      "logits/rejected": -3.1795825958251953,
      "logps/chosen": -310.74822998046875,
      "logps/rejected": -366.74078369140625,
      "loss": 0.2751,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.088016986846924,
      "rewards/margins": 2.7770957946777344,
      "rewards/rejected": -7.865113258361816,
      "step": 6690
    },
    {
      "epoch": 1.4501245532329687,
      "grad_norm": 16.25,
      "learning_rate": 9.808750693661342e-06,
      "logits/chosen": -3.241199016571045,
      "logits/rejected": -3.1932101249694824,
      "logps/chosen": -332.2509765625,
      "logps/rejected": -388.971435546875,
      "loss": 0.264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.401240348815918,
      "rewards/margins": 3.029315710067749,
      "rewards/rejected": -8.430556297302246,
      "step": 6695
    },
    {
      "epoch": 1.451207624824001,
      "grad_norm": 24.25,
      "learning_rate": 9.802318855802864e-06,
      "logits/chosen": -2.7881789207458496,
      "logits/rejected": -2.9787418842315674,
      "logps/chosen": -347.40460205078125,
      "logps/rejected": -379.0799865722656,
      "loss": 0.2869,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.982597827911377,
      "rewards/margins": 2.531099557876587,
      "rewards/rejected": -7.513697147369385,
      "step": 6700
    },
    {
      "epoch": 1.452290696415033,
      "grad_norm": 16.75,
      "learning_rate": 9.795885963112144e-06,
      "logits/chosen": -3.095085859298706,
      "logits/rejected": -3.115355968475342,
      "logps/chosen": -319.49578857421875,
      "logps/rejected": -382.9137878417969,
      "loss": 0.19,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -5.167126655578613,
      "rewards/margins": 3.284860610961914,
      "rewards/rejected": -8.451987266540527,
      "step": 6705
    },
    {
      "epoch": 1.4533737680060652,
      "grad_norm": 20.125,
      "learning_rate": 9.789452024761198e-06,
      "logits/chosen": -3.173558473587036,
      "logits/rejected": -3.0684709548950195,
      "logps/chosen": -316.66998291015625,
      "logps/rejected": -371.78173828125,
      "loss": 0.3065,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.752496242523193,
      "rewards/margins": 2.4917550086975098,
      "rewards/rejected": -8.244250297546387,
      "step": 6710
    },
    {
      "epoch": 1.4544568395970974,
      "grad_norm": 23.75,
      "learning_rate": 9.783017049923516e-06,
      "logits/chosen": -3.012725830078125,
      "logits/rejected": -2.9152073860168457,
      "logps/chosen": -335.68511962890625,
      "logps/rejected": -387.9934997558594,
      "loss": 0.3652,
      "rewards/accuracies": 0.8000000715255737,
      "rewards/chosen": -5.827768802642822,
      "rewards/margins": 2.5722546577453613,
      "rewards/rejected": -8.4000244140625,
      "step": 6715
    },
    {
      "epoch": 1.4555399111881295,
      "grad_norm": 28.375,
      "learning_rate": 9.776581047774073e-06,
      "logits/chosen": -2.874389171600342,
      "logits/rejected": -2.8622756004333496,
      "logps/chosen": -320.12506103515625,
      "logps/rejected": -378.2082214355469,
      "loss": 0.3187,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -4.9170098304748535,
      "rewards/margins": 2.7802281379699707,
      "rewards/rejected": -7.697237491607666,
      "step": 6720
    },
    {
      "epoch": 1.4566229827791617,
      "grad_norm": 21.5,
      "learning_rate": 9.770144027489308e-06,
      "logits/chosen": -2.9200475215911865,
      "logits/rejected": -3.0528054237365723,
      "logps/chosen": -343.9949951171875,
      "logps/rejected": -382.3630065917969,
      "loss": 0.2922,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.365740776062012,
      "rewards/margins": 2.732023239135742,
      "rewards/rejected": -8.097764015197754,
      "step": 6725
    },
    {
      "epoch": 1.4577060543701938,
      "grad_norm": 26.375,
      "learning_rate": 9.763705998247115e-06,
      "logits/chosen": -2.9978156089782715,
      "logits/rejected": -3.071347713470459,
      "logps/chosen": -330.8571472167969,
      "logps/rejected": -360.32366943359375,
      "loss": 0.3087,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.358300685882568,
      "rewards/margins": 2.4880735874176025,
      "rewards/rejected": -7.846373558044434,
      "step": 6730
    },
    {
      "epoch": 1.458789125961226,
      "grad_norm": 23.125,
      "learning_rate": 9.757266969226817e-06,
      "logits/chosen": -2.876255750656128,
      "logits/rejected": -2.9443163871765137,
      "logps/chosen": -324.33740234375,
      "logps/rejected": -377.70355224609375,
      "loss": 0.238,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.233353614807129,
      "rewards/margins": 2.7474241256713867,
      "rewards/rejected": -7.980776786804199,
      "step": 6735
    },
    {
      "epoch": 1.4598721975522582,
      "grad_norm": 31.5,
      "learning_rate": 9.750826949609175e-06,
      "logits/chosen": -2.934077739715576,
      "logits/rejected": -3.054363965988159,
      "logps/chosen": -324.67010498046875,
      "logps/rejected": -365.02850341796875,
      "loss": 0.3087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.2798004150390625,
      "rewards/margins": 2.596240997314453,
      "rewards/rejected": -7.876041412353516,
      "step": 6740
    },
    {
      "epoch": 1.4609552691432903,
      "grad_norm": 20.25,
      "learning_rate": 9.744385948576351e-06,
      "logits/chosen": -2.9789557456970215,
      "logits/rejected": -3.0051140785217285,
      "logps/chosen": -329.91363525390625,
      "logps/rejected": -380.33349609375,
      "loss": 0.2378,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.07149600982666,
      "rewards/margins": 2.868314027786255,
      "rewards/rejected": -7.939810276031494,
      "step": 6745
    },
    {
      "epoch": 1.4620383407343225,
      "grad_norm": 17.875,
      "learning_rate": 9.737943975311916e-06,
      "logits/chosen": -3.0410563945770264,
      "logits/rejected": -2.995421886444092,
      "logps/chosen": -324.1555480957031,
      "logps/rejected": -371.50286865234375,
      "loss": 0.2991,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.199015140533447,
      "rewards/margins": 2.7597880363464355,
      "rewards/rejected": -7.958802700042725,
      "step": 6750
    },
    {
      "epoch": 1.4631214123253546,
      "grad_norm": 20.5,
      "learning_rate": 9.731501039000817e-06,
      "logits/chosen": -2.868509531021118,
      "logits/rejected": -2.787456512451172,
      "logps/chosen": -314.9598388671875,
      "logps/rejected": -382.2249755859375,
      "loss": 0.241,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -4.628749370574951,
      "rewards/margins": 2.9322702884674072,
      "rewards/rejected": -7.5610198974609375,
      "step": 6755
    },
    {
      "epoch": 1.4642044839163868,
      "grad_norm": 20.125,
      "learning_rate": 9.725057148829381e-06,
      "logits/chosen": -2.955052137374878,
      "logits/rejected": -3.014068126678467,
      "logps/chosen": -315.62750244140625,
      "logps/rejected": -367.60980224609375,
      "loss": 0.1892,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.705874919891357,
      "rewards/margins": 3.100148916244507,
      "rewards/rejected": -7.806024074554443,
      "step": 6760
    },
    {
      "epoch": 1.465287555507419,
      "grad_norm": 16.625,
      "learning_rate": 9.718612313985298e-06,
      "logits/chosen": -3.065420150756836,
      "logits/rejected": -3.01059627532959,
      "logps/chosen": -325.8017272949219,
      "logps/rejected": -382.58880615234375,
      "loss": 0.2723,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.564990043640137,
      "rewards/margins": 3.045269727706909,
      "rewards/rejected": -8.610259056091309,
      "step": 6765
    },
    {
      "epoch": 1.4663706270984513,
      "grad_norm": 21.25,
      "learning_rate": 9.712166543657598e-06,
      "logits/chosen": -3.246455669403076,
      "logits/rejected": -3.0660722255706787,
      "logps/chosen": -322.1203918457031,
      "logps/rejected": -389.78125,
      "loss": 0.2588,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.763317584991455,
      "rewards/margins": 3.0122294425964355,
      "rewards/rejected": -8.775546073913574,
      "step": 6770
    },
    {
      "epoch": 1.4674536986894835,
      "grad_norm": 30.0,
      "learning_rate": 9.705719847036643e-06,
      "logits/chosen": -3.034111261367798,
      "logits/rejected": -3.040653705596924,
      "logps/chosen": -325.1871337890625,
      "logps/rejected": -369.71728515625,
      "loss": 0.2805,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.190677642822266,
      "rewards/margins": 2.7748517990112305,
      "rewards/rejected": -7.965529441833496,
      "step": 6775
    },
    {
      "epoch": 1.4685367702805157,
      "grad_norm": 13.375,
      "learning_rate": 9.699272233314128e-06,
      "logits/chosen": -3.032627582550049,
      "logits/rejected": -3.0095417499542236,
      "logps/chosen": -344.13262939453125,
      "logps/rejected": -397.716796875,
      "loss": 0.2938,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.626452922821045,
      "rewards/margins": 2.8365132808685303,
      "rewards/rejected": -8.462966918945312,
      "step": 6780
    },
    {
      "epoch": 1.4696198418715478,
      "grad_norm": 26.5,
      "learning_rate": 9.692823711683043e-06,
      "logits/chosen": -3.1474318504333496,
      "logits/rejected": -3.0318570137023926,
      "logps/chosen": -341.16015625,
      "logps/rejected": -393.54791259765625,
      "loss": 0.3005,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.509292125701904,
      "rewards/margins": 2.81170916557312,
      "rewards/rejected": -8.321001052856445,
      "step": 6785
    },
    {
      "epoch": 1.47070291346258,
      "grad_norm": 13.1875,
      "learning_rate": 9.686374291337678e-06,
      "logits/chosen": -3.148159980773926,
      "logits/rejected": -3.0798497200012207,
      "logps/chosen": -337.9826965332031,
      "logps/rejected": -397.5670471191406,
      "loss": 0.2458,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.6766815185546875,
      "rewards/margins": 2.951230525970459,
      "rewards/rejected": -8.627912521362305,
      "step": 6790
    },
    {
      "epoch": 1.4717859850536121,
      "grad_norm": 22.75,
      "learning_rate": 9.679923981473603e-06,
      "logits/chosen": -3.1506409645080566,
      "logits/rejected": -3.029466390609741,
      "logps/chosen": -317.5527648925781,
      "logps/rejected": -364.7167663574219,
      "loss": 0.2589,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.158413410186768,
      "rewards/margins": 2.4892961978912354,
      "rewards/rejected": -7.647708892822266,
      "step": 6795
    },
    {
      "epoch": 1.4728690566446443,
      "grad_norm": 15.3125,
      "learning_rate": 9.673472791287657e-06,
      "logits/chosen": -3.0689780712127686,
      "logits/rejected": -3.1360092163085938,
      "logps/chosen": -307.9644470214844,
      "logps/rejected": -360.14471435546875,
      "loss": 0.2383,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -4.9693708419799805,
      "rewards/margins": 2.8899435997009277,
      "rewards/rejected": -7.85931396484375,
      "step": 6800
    },
    {
      "epoch": 1.4739521282356765,
      "grad_norm": 14.9375,
      "learning_rate": 9.667020729977936e-06,
      "logits/chosen": -3.067978858947754,
      "logits/rejected": -3.0086967945098877,
      "logps/chosen": -317.1405029296875,
      "logps/rejected": -381.595947265625,
      "loss": 0.2163,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.131951332092285,
      "rewards/margins": 2.9529800415039062,
      "rewards/rejected": -8.084930419921875,
      "step": 6805
    },
    {
      "epoch": 1.4750351998267086,
      "grad_norm": 16.375,
      "learning_rate": 9.660567806743773e-06,
      "logits/chosen": -3.0132412910461426,
      "logits/rejected": -2.9509496688842773,
      "logps/chosen": -338.93463134765625,
      "logps/rejected": -405.3013916015625,
      "loss": 0.2489,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.070411205291748,
      "rewards/margins": 2.953415632247925,
      "rewards/rejected": -8.02382755279541,
      "step": 6810
    },
    {
      "epoch": 1.4761182714177408,
      "grad_norm": 32.75,
      "learning_rate": 9.654114030785732e-06,
      "logits/chosen": -2.8992769718170166,
      "logits/rejected": -2.9727540016174316,
      "logps/chosen": -335.9479064941406,
      "logps/rejected": -370.36199951171875,
      "loss": 0.313,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.303946495056152,
      "rewards/margins": 2.618410587310791,
      "rewards/rejected": -7.922356605529785,
      "step": 6815
    },
    {
      "epoch": 1.477201343008773,
      "grad_norm": 12.625,
      "learning_rate": 9.647659411305598e-06,
      "logits/chosen": -3.1597156524658203,
      "logits/rejected": -3.141420364379883,
      "logps/chosen": -317.5924377441406,
      "logps/rejected": -371.35882568359375,
      "loss": 0.227,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.561173915863037,
      "rewards/margins": 2.9291369915008545,
      "rewards/rejected": -8.490311622619629,
      "step": 6820
    },
    {
      "epoch": 1.478284414599805,
      "grad_norm": 21.125,
      "learning_rate": 9.641203957506354e-06,
      "logits/chosen": -3.077293872833252,
      "logits/rejected": -3.098902702331543,
      "logps/chosen": -332.62762451171875,
      "logps/rejected": -391.6153259277344,
      "loss": 0.2423,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.600566387176514,
      "rewards/margins": 3.253203868865967,
      "rewards/rejected": -8.853769302368164,
      "step": 6825
    },
    {
      "epoch": 1.4793674861908372,
      "grad_norm": 15.875,
      "learning_rate": 9.63474767859217e-06,
      "logits/chosen": -3.0194029808044434,
      "logits/rejected": -3.0767338275909424,
      "logps/chosen": -324.7406311035156,
      "logps/rejected": -380.43438720703125,
      "loss": 0.2109,
      "rewards/accuracies": 0.9249998927116394,
      "rewards/chosen": -4.854010105133057,
      "rewards/margins": 3.27423095703125,
      "rewards/rejected": -8.128240585327148,
      "step": 6830
    },
    {
      "epoch": 1.4804505577818694,
      "grad_norm": 20.25,
      "learning_rate": 9.628290583768397e-06,
      "logits/chosen": -3.0839638710021973,
      "logits/rejected": -3.046609401702881,
      "logps/chosen": -309.7850036621094,
      "logps/rejected": -368.9315185546875,
      "loss": 0.2241,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -4.660407066345215,
      "rewards/margins": 2.9768435955047607,
      "rewards/rejected": -7.637251377105713,
      "step": 6835
    },
    {
      "epoch": 1.4815336293729016,
      "grad_norm": 22.75,
      "learning_rate": 9.62183268224155e-06,
      "logits/chosen": -3.036630630493164,
      "logits/rejected": -3.120851993560791,
      "logps/chosen": -319.8919372558594,
      "logps/rejected": -368.5615539550781,
      "loss": 0.2771,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -4.79457950592041,
      "rewards/margins": 2.606766939163208,
      "rewards/rejected": -7.401346683502197,
      "step": 6840
    },
    {
      "epoch": 1.4826167009639337,
      "grad_norm": 26.25,
      "learning_rate": 9.615373983219288e-06,
      "logits/chosen": -3.041980266571045,
      "logits/rejected": -3.1275157928466797,
      "logps/chosen": -350.4825439453125,
      "logps/rejected": -383.44293212890625,
      "loss": 0.3357,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.260771751403809,
      "rewards/margins": 2.7772316932678223,
      "rewards/rejected": -8.038003921508789,
      "step": 6845
    },
    {
      "epoch": 1.4836997725549659,
      "grad_norm": 27.0,
      "learning_rate": 9.608914495910412e-06,
      "logits/chosen": -3.1394386291503906,
      "logits/rejected": -3.1335461139678955,
      "logps/chosen": -319.4728698730469,
      "logps/rejected": -369.52337646484375,
      "loss": 0.3136,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.137831211090088,
      "rewards/margins": 2.5397965908050537,
      "rewards/rejected": -7.6776275634765625,
      "step": 6850
    },
    {
      "epoch": 1.484782844145998,
      "grad_norm": 20.25,
      "learning_rate": 9.602454229524847e-06,
      "logits/chosen": -3.0344290733337402,
      "logits/rejected": -2.906233072280884,
      "logps/chosen": -328.48480224609375,
      "logps/rejected": -398.7265319824219,
      "loss": 0.2603,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.299132823944092,
      "rewards/margins": 2.9859213829040527,
      "rewards/rejected": -8.285054206848145,
      "step": 6855
    },
    {
      "epoch": 1.4858659157370302,
      "grad_norm": 22.25,
      "learning_rate": 9.595993193273632e-06,
      "logits/chosen": -2.978639602661133,
      "logits/rejected": -3.086247682571411,
      "logps/chosen": -317.7562561035156,
      "logps/rejected": -366.4784240722656,
      "loss": 0.3286,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.1445112228393555,
      "rewards/margins": 2.7727150917053223,
      "rewards/rejected": -7.917226314544678,
      "step": 6860
    },
    {
      "epoch": 1.4869489873280624,
      "grad_norm": 26.125,
      "learning_rate": 9.58953139636889e-06,
      "logits/chosen": -3.1120333671569824,
      "logits/rejected": -3.1292357444763184,
      "logps/chosen": -333.8434143066406,
      "logps/rejected": -382.1817321777344,
      "loss": 0.324,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.265040397644043,
      "rewards/margins": 2.8490347862243652,
      "rewards/rejected": -8.114076614379883,
      "step": 6865
    },
    {
      "epoch": 1.4880320589190945,
      "grad_norm": 22.75,
      "learning_rate": 9.583068848023846e-06,
      "logits/chosen": -3.044633388519287,
      "logits/rejected": -3.0277562141418457,
      "logps/chosen": -330.0776062011719,
      "logps/rejected": -390.194580078125,
      "loss": 0.2425,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.261080741882324,
      "rewards/margins": 3.1335535049438477,
      "rewards/rejected": -8.394633293151855,
      "step": 6870
    },
    {
      "epoch": 1.4891151305101267,
      "grad_norm": 23.25,
      "learning_rate": 9.576605557452784e-06,
      "logits/chosen": -3.056817054748535,
      "logits/rejected": -3.0553228855133057,
      "logps/chosen": -322.46563720703125,
      "logps/rejected": -371.50079345703125,
      "loss": 0.2981,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -4.903461456298828,
      "rewards/margins": 2.6917576789855957,
      "rewards/rejected": -7.59522008895874,
      "step": 6875
    },
    {
      "epoch": 1.4901982021011588,
      "grad_norm": 21.75,
      "learning_rate": 9.570141533871054e-06,
      "logits/chosen": -2.945016384124756,
      "logits/rejected": -2.879382610321045,
      "logps/chosen": -317.93670654296875,
      "logps/rejected": -375.4947814941406,
      "loss": 0.2669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.740754127502441,
      "rewards/margins": 2.6145732402801514,
      "rewards/rejected": -7.355327606201172,
      "step": 6880
    },
    {
      "epoch": 1.491281273692191,
      "grad_norm": 17.875,
      "learning_rate": 9.563676786495046e-06,
      "logits/chosen": -3.0868937969207764,
      "logits/rejected": -3.016596794128418,
      "logps/chosen": -309.11663818359375,
      "logps/rejected": -370.8156433105469,
      "loss": 0.2461,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.089086532592773,
      "rewards/margins": 2.948068141937256,
      "rewards/rejected": -8.03715705871582,
      "step": 6885
    },
    {
      "epoch": 1.4923643452832231,
      "grad_norm": 25.625,
      "learning_rate": 9.557211324542182e-06,
      "logits/chosen": -3.00544810295105,
      "logits/rejected": -2.9889636039733887,
      "logps/chosen": -331.2441711425781,
      "logps/rejected": -388.3189697265625,
      "loss": 0.2707,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.451547622680664,
      "rewards/margins": 2.6426703929901123,
      "rewards/rejected": -8.094218254089355,
      "step": 6890
    },
    {
      "epoch": 1.4934474168742553,
      "grad_norm": 11.0625,
      "learning_rate": 9.550745157230909e-06,
      "logits/chosen": -3.031165361404419,
      "logits/rejected": -3.0906827449798584,
      "logps/chosen": -327.1440734863281,
      "logps/rejected": -372.572021484375,
      "loss": 0.247,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.320913314819336,
      "rewards/margins": 2.7246530055999756,
      "rewards/rejected": -8.045565605163574,
      "step": 6895
    },
    {
      "epoch": 1.4945304884652875,
      "grad_norm": 7.6875,
      "learning_rate": 9.544278293780673e-06,
      "logits/chosen": -2.9420244693756104,
      "logits/rejected": -2.998241901397705,
      "logps/chosen": -326.3425598144531,
      "logps/rejected": -378.89239501953125,
      "loss": 0.1737,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -4.77906608581543,
      "rewards/margins": 3.467921733856201,
      "rewards/rejected": -8.246988296508789,
      "step": 6900
    },
    {
      "epoch": 1.4956135600563196,
      "grad_norm": 21.0,
      "learning_rate": 9.537810743411913e-06,
      "logits/chosen": -3.1023240089416504,
      "logits/rejected": -3.0590062141418457,
      "logps/chosen": -329.0272216796875,
      "logps/rejected": -382.1690979003906,
      "loss": 0.2713,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.559637546539307,
      "rewards/margins": 2.7536444664001465,
      "rewards/rejected": -8.313283920288086,
      "step": 6905
    },
    {
      "epoch": 1.4966966316473518,
      "grad_norm": 19.875,
      "learning_rate": 9.531342515346054e-06,
      "logits/chosen": -3.1180357933044434,
      "logits/rejected": -3.1426734924316406,
      "logps/chosen": -345.2480163574219,
      "logps/rejected": -387.2304992675781,
      "loss": 0.2977,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.896130561828613,
      "rewards/margins": 2.8940634727478027,
      "rewards/rejected": -8.790193557739258,
      "step": 6910
    },
    {
      "epoch": 1.497779703238384,
      "grad_norm": 27.0,
      "learning_rate": 9.524873618805478e-06,
      "logits/chosen": -3.007397413253784,
      "logits/rejected": -2.979395627975464,
      "logps/chosen": -329.5482177734375,
      "logps/rejected": -384.6141357421875,
      "loss": 0.3064,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.625042915344238,
      "rewards/margins": 2.387424945831299,
      "rewards/rejected": -8.012468338012695,
      "step": 6915
    },
    {
      "epoch": 1.498862774829416,
      "grad_norm": 21.875,
      "learning_rate": 9.518404063013528e-06,
      "logits/chosen": -3.14449143409729,
      "logits/rejected": -3.154934883117676,
      "logps/chosen": -325.2856140136719,
      "logps/rejected": -382.66094970703125,
      "loss": 0.2262,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.532168388366699,
      "rewards/margins": 2.868722438812256,
      "rewards/rejected": -8.400890350341797,
      "step": 6920
    },
    {
      "epoch": 1.4999458464204483,
      "grad_norm": 20.25,
      "learning_rate": 9.511933857194481e-06,
      "logits/chosen": -3.113806962966919,
      "logits/rejected": -3.03432035446167,
      "logps/chosen": -339.9832458496094,
      "logps/rejected": -387.5312805175781,
      "loss": 0.2915,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.557313442230225,
      "rewards/margins": 2.702073574066162,
      "rewards/rejected": -8.25938606262207,
      "step": 6925
    },
    {
      "epoch": 1.5010289180114804,
      "grad_norm": 17.375,
      "learning_rate": 9.505463010573544e-06,
      "logits/chosen": -3.089354991912842,
      "logits/rejected": -2.999387741088867,
      "logps/chosen": -329.4388122558594,
      "logps/rejected": -386.7347717285156,
      "loss": 0.2923,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.228104114532471,
      "rewards/margins": 2.9545886516571045,
      "rewards/rejected": -8.182692527770996,
      "step": 6930
    },
    {
      "epoch": 1.5021119896025126,
      "grad_norm": 18.25,
      "learning_rate": 9.498991532376837e-06,
      "logits/chosen": -2.8995473384857178,
      "logits/rejected": -2.864755153656006,
      "logps/chosen": -339.3115234375,
      "logps/rejected": -397.8387145996094,
      "loss": 0.2231,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.416640281677246,
      "rewards/margins": 2.8434720039367676,
      "rewards/rejected": -8.260111808776855,
      "step": 6935
    },
    {
      "epoch": 1.5031950611935447,
      "grad_norm": 11.75,
      "learning_rate": 9.492519431831382e-06,
      "logits/chosen": -3.2799410820007324,
      "logits/rejected": -3.1838507652282715,
      "logps/chosen": -310.7138366699219,
      "logps/rejected": -373.5116882324219,
      "loss": 0.2547,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.320448398590088,
      "rewards/margins": 2.8370094299316406,
      "rewards/rejected": -8.15745735168457,
      "step": 6940
    },
    {
      "epoch": 1.5042781327845771,
      "grad_norm": 10.1875,
      "learning_rate": 9.486046718165086e-06,
      "logits/chosen": -3.0277180671691895,
      "logits/rejected": -2.9989376068115234,
      "logps/chosen": -312.3487243652344,
      "logps/rejected": -375.55010986328125,
      "loss": 0.2076,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.363905906677246,
      "rewards/margins": 3.0875182151794434,
      "rewards/rejected": -7.451424598693848,
      "step": 6945
    },
    {
      "epoch": 1.5053612043756093,
      "grad_norm": 37.0,
      "learning_rate": 9.479573400606734e-06,
      "logits/chosen": -3.0343313217163086,
      "logits/rejected": -3.0045814514160156,
      "logps/chosen": -332.7703552246094,
      "logps/rejected": -388.3984680175781,
      "loss": 0.2948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.912116050720215,
      "rewards/margins": 2.7163262367248535,
      "rewards/rejected": -7.628442287445068,
      "step": 6950
    },
    {
      "epoch": 1.5064442759666414,
      "grad_norm": 23.125,
      "learning_rate": 9.473099488385965e-06,
      "logits/chosen": -3.1061670780181885,
      "logits/rejected": -3.0676214694976807,
      "logps/chosen": -331.9761657714844,
      "logps/rejected": -392.05133056640625,
      "loss": 0.2853,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.323493003845215,
      "rewards/margins": 2.8560357093811035,
      "rewards/rejected": -8.179529190063477,
      "step": 6955
    },
    {
      "epoch": 1.5075273475576736,
      "grad_norm": 17.75,
      "learning_rate": 9.46662499073327e-06,
      "logits/chosen": -3.0970187187194824,
      "logits/rejected": -3.167436361312866,
      "logps/chosen": -327.4009704589844,
      "logps/rejected": -377.5740661621094,
      "loss": 0.2105,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.367177963256836,
      "rewards/margins": 2.968069553375244,
      "rewards/rejected": -8.335247993469238,
      "step": 6960
    },
    {
      "epoch": 1.5086104191487058,
      "grad_norm": 34.0,
      "learning_rate": 9.460149916879977e-06,
      "logits/chosen": -3.1007142066955566,
      "logits/rejected": -3.081282377243042,
      "logps/chosen": -339.56060791015625,
      "logps/rejected": -383.1930847167969,
      "loss": 0.3776,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.568500518798828,
      "rewards/margins": 2.4234228134155273,
      "rewards/rejected": -7.9919233322143555,
      "step": 6965
    },
    {
      "epoch": 1.509693490739738,
      "grad_norm": 21.625,
      "learning_rate": 9.453674276058232e-06,
      "logits/chosen": -3.216114044189453,
      "logits/rejected": -3.165992021560669,
      "logps/chosen": -311.38043212890625,
      "logps/rejected": -375.68280029296875,
      "loss": 0.2532,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.083646774291992,
      "rewards/margins": 2.9336748123168945,
      "rewards/rejected": -8.017321586608887,
      "step": 6970
    },
    {
      "epoch": 1.51077656233077,
      "grad_norm": 26.25,
      "learning_rate": 9.447198077500993e-06,
      "logits/chosen": -3.0564162731170654,
      "logits/rejected": -3.029979705810547,
      "logps/chosen": -327.56610107421875,
      "logps/rejected": -371.3681945800781,
      "loss": 0.2983,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.4439287185668945,
      "rewards/margins": 2.4258816242218018,
      "rewards/rejected": -7.869810581207275,
      "step": 6975
    },
    {
      "epoch": 1.5118596339218022,
      "grad_norm": 30.0,
      "learning_rate": 9.440721330442007e-06,
      "logits/chosen": -3.2486138343811035,
      "logits/rejected": -3.2256827354431152,
      "logps/chosen": -335.69158935546875,
      "logps/rejected": -377.0966796875,
      "loss": 0.2323,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.614086627960205,
      "rewards/margins": 2.791626453399658,
      "rewards/rejected": -8.405713081359863,
      "step": 6980
    },
    {
      "epoch": 1.5129427055128344,
      "grad_norm": 24.0,
      "learning_rate": 9.434244044115808e-06,
      "logits/chosen": -3.114623546600342,
      "logits/rejected": -2.9670615196228027,
      "logps/chosen": -332.9722900390625,
      "logps/rejected": -393.39453125,
      "loss": 0.2898,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.815535545349121,
      "rewards/margins": 2.5984435081481934,
      "rewards/rejected": -8.413979530334473,
      "step": 6985
    },
    {
      "epoch": 1.5140257771038665,
      "grad_norm": 33.0,
      "learning_rate": 9.427766227757702e-06,
      "logits/chosen": -3.1449851989746094,
      "logits/rejected": -3.087460517883301,
      "logps/chosen": -325.0448913574219,
      "logps/rejected": -375.4789123535156,
      "loss": 0.2735,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.434444427490234,
      "rewards/margins": 2.7191221714019775,
      "rewards/rejected": -8.153566360473633,
      "step": 6990
    },
    {
      "epoch": 1.5151088486948987,
      "grad_norm": 14.1875,
      "learning_rate": 9.421287890603741e-06,
      "logits/chosen": -3.2304866313934326,
      "logits/rejected": -3.1854803562164307,
      "logps/chosen": -310.6259460449219,
      "logps/rejected": -365.54229736328125,
      "loss": 0.2793,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.140172481536865,
      "rewards/margins": 2.675309658050537,
      "rewards/rejected": -7.815482139587402,
      "step": 6995
    },
    {
      "epoch": 1.5161919202859309,
      "grad_norm": 23.75,
      "learning_rate": 9.414809041890729e-06,
      "logits/chosen": -3.2474799156188965,
      "logits/rejected": -3.1134183406829834,
      "logps/chosen": -341.6644592285156,
      "logps/rejected": -395.262451171875,
      "loss": 0.3458,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -6.037227630615234,
      "rewards/margins": 2.2370336055755615,
      "rewards/rejected": -8.274261474609375,
      "step": 7000
    },
    {
      "epoch": 1.5161919202859309,
      "eval_logits/chosen": -3.240219831466675,
      "eval_logits/rejected": -3.1690564155578613,
      "eval_logps/chosen": -312.82293701171875,
      "eval_logps/rejected": -358.1783142089844,
      "eval_loss": 0.41442567110061646,
      "eval_rewards/accuracies": 0.8299999833106995,
      "eval_rewards/chosen": -4.931084156036377,
      "eval_rewards/margins": 2.094975233078003,
      "eval_rewards/rejected": -7.026059627532959,
      "eval_runtime": 14.6249,
      "eval_samples_per_second": 13.675,
      "eval_steps_per_second": 13.675,
      "step": 7000
    },
    {
      "epoch": 1.517274991876963,
      "grad_norm": 22.25,
      "learning_rate": 9.408329690856191e-06,
      "logits/chosen": -3.200613498687744,
      "logits/rejected": -3.034456253051758,
      "logps/chosen": -318.42279052734375,
      "logps/rejected": -380.83551025390625,
      "loss": 0.2876,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -4.880768299102783,
      "rewards/margins": 2.733311653137207,
      "rewards/rejected": -7.614079475402832,
      "step": 7005
    },
    {
      "epoch": 1.5183580634679954,
      "grad_norm": 21.25,
      "learning_rate": 9.401849846738377e-06,
      "logits/chosen": -3.112048625946045,
      "logits/rejected": -3.0756030082702637,
      "logps/chosen": -315.5652770996094,
      "logps/rejected": -392.8707580566406,
      "loss": 0.2055,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -5.111533164978027,
      "rewards/margins": 3.1608269214630127,
      "rewards/rejected": -8.272359848022461,
      "step": 7010
    },
    {
      "epoch": 1.5194411350590276,
      "grad_norm": 26.25,
      "learning_rate": 9.395369518776235e-06,
      "logits/chosen": -3.2212157249450684,
      "logits/rejected": -3.2354393005371094,
      "logps/chosen": -327.82855224609375,
      "logps/rejected": -367.9674377441406,
      "loss": 0.3019,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.441535949707031,
      "rewards/margins": 2.402900218963623,
      "rewards/rejected": -7.844435691833496,
      "step": 7015
    },
    {
      "epoch": 1.5205242066500597,
      "grad_norm": 32.0,
      "learning_rate": 9.388888716209401e-06,
      "logits/chosen": -3.000863552093506,
      "logits/rejected": -3.074511766433716,
      "logps/chosen": -337.3605651855469,
      "logps/rejected": -382.37274169921875,
      "loss": 0.2797,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.0412702560424805,
      "rewards/margins": 3.0577404499053955,
      "rewards/rejected": -8.099010467529297,
      "step": 7020
    },
    {
      "epoch": 1.5216072782410919,
      "grad_norm": 28.75,
      "learning_rate": 9.382407448278195e-06,
      "logits/chosen": -3.3187828063964844,
      "logits/rejected": -3.270582914352417,
      "logps/chosen": -317.42401123046875,
      "logps/rejected": -368.5248107910156,
      "loss": 0.3044,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.1682939529418945,
      "rewards/margins": 2.5529634952545166,
      "rewards/rejected": -7.721257209777832,
      "step": 7025
    },
    {
      "epoch": 1.522690349832124,
      "grad_norm": 16.625,
      "learning_rate": 9.37592572422359e-06,
      "logits/chosen": -3.1660282611846924,
      "logits/rejected": -3.129415512084961,
      "logps/chosen": -331.1405944824219,
      "logps/rejected": -389.53509521484375,
      "loss": 0.1952,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.345254898071289,
      "rewards/margins": 2.9582512378692627,
      "rewards/rejected": -8.303506851196289,
      "step": 7030
    },
    {
      "epoch": 1.5237734214231562,
      "grad_norm": 8.8125,
      "learning_rate": 9.369443553287218e-06,
      "logits/chosen": -3.155397653579712,
      "logits/rejected": -3.137444496154785,
      "logps/chosen": -315.5999755859375,
      "logps/rejected": -373.1589660644531,
      "loss": 0.2574,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.0529985427856445,
      "rewards/margins": 2.9845259189605713,
      "rewards/rejected": -8.03752326965332,
      "step": 7035
    },
    {
      "epoch": 1.5248564930141884,
      "grad_norm": 22.25,
      "learning_rate": 9.362960944711347e-06,
      "logits/chosen": -3.1796834468841553,
      "logits/rejected": -3.1349520683288574,
      "logps/chosen": -327.61346435546875,
      "logps/rejected": -390.2873229980469,
      "loss": 0.208,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.856851100921631,
      "rewards/margins": 3.0569169521331787,
      "rewards/rejected": -8.913768768310547,
      "step": 7040
    },
    {
      "epoch": 1.5259395646052205,
      "grad_norm": 34.0,
      "learning_rate": 9.356477907738865e-06,
      "logits/chosen": -3.198646068572998,
      "logits/rejected": -3.2319846153259277,
      "logps/chosen": -322.35809326171875,
      "logps/rejected": -371.3989562988281,
      "loss": 0.2968,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.1065473556518555,
      "rewards/margins": 2.880392551422119,
      "rewards/rejected": -7.986939907073975,
      "step": 7045
    },
    {
      "epoch": 1.5270226361962527,
      "grad_norm": 11.0625,
      "learning_rate": 9.349994451613272e-06,
      "logits/chosen": -3.0706162452697754,
      "logits/rejected": -3.084679126739502,
      "logps/chosen": -337.378662109375,
      "logps/rejected": -403.1842956542969,
      "loss": 0.2763,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.7482171058654785,
      "rewards/margins": 2.8153891563415527,
      "rewards/rejected": -8.563606262207031,
      "step": 7050
    },
    {
      "epoch": 1.5281057077872848,
      "grad_norm": 16.375,
      "learning_rate": 9.34351058557867e-06,
      "logits/chosen": -3.0121395587921143,
      "logits/rejected": -3.109776496887207,
      "logps/chosen": -376.4783020019531,
      "logps/rejected": -409.46856689453125,
      "loss": 0.2745,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.785402297973633,
      "rewards/margins": 2.846579074859619,
      "rewards/rejected": -8.631980895996094,
      "step": 7055
    },
    {
      "epoch": 1.529188779378317,
      "grad_norm": 24.75,
      "learning_rate": 9.337026318879739e-06,
      "logits/chosen": -3.155118465423584,
      "logits/rejected": -3.100264549255371,
      "logps/chosen": -354.1097106933594,
      "logps/rejected": -388.58306884765625,
      "loss": 0.2906,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.894417762756348,
      "rewards/margins": 2.3680052757263184,
      "rewards/rejected": -8.262423515319824,
      "step": 7060
    },
    {
      "epoch": 1.5302718509693491,
      "grad_norm": 19.25,
      "learning_rate": 9.330541660761737e-06,
      "logits/chosen": -3.159703493118286,
      "logits/rejected": -3.187391519546509,
      "logps/chosen": -319.8155212402344,
      "logps/rejected": -374.056396484375,
      "loss": 0.2735,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.5249786376953125,
      "rewards/margins": 2.795578718185425,
      "rewards/rejected": -8.320557594299316,
      "step": 7065
    },
    {
      "epoch": 1.5313549225603813,
      "grad_norm": 18.875,
      "learning_rate": 9.324056620470473e-06,
      "logits/chosen": -3.0657262802124023,
      "logits/rejected": -3.002149820327759,
      "logps/chosen": -319.818115234375,
      "logps/rejected": -382.17474365234375,
      "loss": 0.2714,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -4.570674896240234,
      "rewards/margins": 3.060448169708252,
      "rewards/rejected": -7.6311235427856445,
      "step": 7070
    },
    {
      "epoch": 1.5324379941514135,
      "grad_norm": 29.625,
      "learning_rate": 9.317571207252304e-06,
      "logits/chosen": -3.146531343460083,
      "logits/rejected": -3.1488113403320312,
      "logps/chosen": -326.07244873046875,
      "logps/rejected": -370.6309509277344,
      "loss": 0.3707,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.325742721557617,
      "rewards/margins": 2.5983808040618896,
      "rewards/rejected": -7.924123287200928,
      "step": 7075
    },
    {
      "epoch": 1.5335210657424456,
      "grad_norm": 24.875,
      "learning_rate": 9.311085430354122e-06,
      "logits/chosen": -3.123600721359253,
      "logits/rejected": -3.044560670852661,
      "logps/chosen": -321.19866943359375,
      "logps/rejected": -379.7318115234375,
      "loss": 0.2818,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.041367530822754,
      "rewards/margins": 2.5794215202331543,
      "rewards/rejected": -7.62078857421875,
      "step": 7080
    },
    {
      "epoch": 1.5346041373334778,
      "grad_norm": 18.625,
      "learning_rate": 9.304599299023335e-06,
      "logits/chosen": -3.179490804672241,
      "logits/rejected": -3.1722183227539062,
      "logps/chosen": -333.0731201171875,
      "logps/rejected": -391.1170654296875,
      "loss": 0.2716,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.58413553237915,
      "rewards/margins": 2.8305249214172363,
      "rewards/rejected": -8.414660453796387,
      "step": 7085
    },
    {
      "epoch": 1.53568720892451,
      "grad_norm": 16.625,
      "learning_rate": 9.298112822507855e-06,
      "logits/chosen": -3.2282447814941406,
      "logits/rejected": -3.2215352058410645,
      "logps/chosen": -332.55023193359375,
      "logps/rejected": -365.45391845703125,
      "loss": 0.2744,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.4365153312683105,
      "rewards/margins": 2.542168617248535,
      "rewards/rejected": -7.978684425354004,
      "step": 7090
    },
    {
      "epoch": 1.536770280515542,
      "grad_norm": 23.375,
      "learning_rate": 9.291626010056084e-06,
      "logits/chosen": -3.093816041946411,
      "logits/rejected": -3.0308847427368164,
      "logps/chosen": -320.464111328125,
      "logps/rejected": -369.06982421875,
      "loss": 0.2689,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.912548065185547,
      "rewards/margins": 2.6775729656219482,
      "rewards/rejected": -7.590120792388916,
      "step": 7095
    },
    {
      "epoch": 1.5378533521065743,
      "grad_norm": 27.125,
      "learning_rate": 9.285138870916909e-06,
      "logits/chosen": -2.9809165000915527,
      "logits/rejected": -3.043522834777832,
      "logps/chosen": -331.40863037109375,
      "logps/rejected": -371.8457946777344,
      "loss": 0.2679,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.028878211975098,
      "rewards/margins": 2.7388744354248047,
      "rewards/rejected": -7.767752170562744,
      "step": 7100
    },
    {
      "epoch": 1.5389364236976064,
      "grad_norm": 23.875,
      "learning_rate": 9.278651414339682e-06,
      "logits/chosen": -3.2713420391082764,
      "logits/rejected": -3.1870903968811035,
      "logps/chosen": -322.82513427734375,
      "logps/rejected": -377.9372863769531,
      "loss": 0.2695,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.520200252532959,
      "rewards/margins": 2.9585959911346436,
      "rewards/rejected": -8.47879695892334,
      "step": 7105
    },
    {
      "epoch": 1.5400194952886386,
      "grad_norm": 20.25,
      "learning_rate": 9.2721636495742e-06,
      "logits/chosen": -3.0832769870758057,
      "logits/rejected": -3.133151054382324,
      "logps/chosen": -331.19818115234375,
      "logps/rejected": -366.3039855957031,
      "loss": 0.3162,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.0793609619140625,
      "rewards/margins": 2.443704843521118,
      "rewards/rejected": -7.52306604385376,
      "step": 7110
    },
    {
      "epoch": 1.5411025668796707,
      "grad_norm": 31.375,
      "learning_rate": 9.26567558587071e-06,
      "logits/chosen": -3.1478524208068848,
      "logits/rejected": -3.1825900077819824,
      "logps/chosen": -343.081787109375,
      "logps/rejected": -392.183349609375,
      "loss": 0.2515,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.558094024658203,
      "rewards/margins": 2.9271185398101807,
      "rewards/rejected": -8.485213279724121,
      "step": 7115
    },
    {
      "epoch": 1.542185638470703,
      "grad_norm": 26.75,
      "learning_rate": 9.259187232479879e-06,
      "logits/chosen": -3.151175022125244,
      "logits/rejected": -3.0983104705810547,
      "logps/chosen": -350.2424621582031,
      "logps/rejected": -408.0606994628906,
      "loss": 0.2287,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.597232341766357,
      "rewards/margins": 3.0669915676116943,
      "rewards/rejected": -8.664224624633789,
      "step": 7120
    },
    {
      "epoch": 1.543268710061735,
      "grad_norm": 19.25,
      "learning_rate": 9.252698598652787e-06,
      "logits/chosen": -3.219470977783203,
      "logits/rejected": -3.1114516258239746,
      "logps/chosen": -335.8948669433594,
      "logps/rejected": -391.52801513671875,
      "loss": 0.2663,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.581000328063965,
      "rewards/margins": 2.650906562805176,
      "rewards/rejected": -8.23190689086914,
      "step": 7125
    },
    {
      "epoch": 1.5443517816527672,
      "grad_norm": 28.375,
      "learning_rate": 9.246209693640916e-06,
      "logits/chosen": -3.08156156539917,
      "logits/rejected": -3.014472723007202,
      "logps/chosen": -338.87799072265625,
      "logps/rejected": -393.08447265625,
      "loss": 0.3089,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -5.1875810623168945,
      "rewards/margins": 2.5835721492767334,
      "rewards/rejected": -7.771153450012207,
      "step": 7130
    },
    {
      "epoch": 1.5454348532437994,
      "grad_norm": 11.125,
      "learning_rate": 9.239720526696134e-06,
      "logits/chosen": -3.046375036239624,
      "logits/rejected": -3.1315529346466064,
      "logps/chosen": -332.89105224609375,
      "logps/rejected": -368.3755187988281,
      "loss": 0.2481,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.8155927658081055,
      "rewards/margins": 2.408116102218628,
      "rewards/rejected": -8.223708152770996,
      "step": 7135
    },
    {
      "epoch": 1.5465179248348315,
      "grad_norm": 25.0,
      "learning_rate": 9.23323110707068e-06,
      "logits/chosen": -3.1462554931640625,
      "logits/rejected": -3.1705222129821777,
      "logps/chosen": -317.3255920410156,
      "logps/rejected": -377.5257263183594,
      "loss": 0.2823,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.492401123046875,
      "rewards/margins": 2.6327245235443115,
      "rewards/rejected": -8.125126838684082,
      "step": 7140
    },
    {
      "epoch": 1.5476009964258637,
      "grad_norm": 24.625,
      "learning_rate": 9.226741444017156e-06,
      "logits/chosen": -3.1507654190063477,
      "logits/rejected": -3.130643606185913,
      "logps/chosen": -327.72564697265625,
      "logps/rejected": -376.68487548828125,
      "loss": 0.2484,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -4.989970684051514,
      "rewards/margins": 2.934082508087158,
      "rewards/rejected": -7.924053192138672,
      "step": 7145
    },
    {
      "epoch": 1.5486840680168958,
      "grad_norm": 19.125,
      "learning_rate": 9.220251546788514e-06,
      "logits/chosen": -3.1664810180664062,
      "logits/rejected": -3.079930305480957,
      "logps/chosen": -333.2294006347656,
      "logps/rejected": -384.9178771972656,
      "loss": 0.2693,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.766631603240967,
      "rewards/margins": 2.4252145290374756,
      "rewards/rejected": -8.19184684753418,
      "step": 7150
    },
    {
      "epoch": 1.549767139607928,
      "grad_norm": 22.125,
      "learning_rate": 9.213761424638033e-06,
      "logits/chosen": -3.237276792526245,
      "logits/rejected": -3.154345989227295,
      "logps/chosen": -348.4326171875,
      "logps/rejected": -395.35736083984375,
      "loss": 0.2102,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.989469528198242,
      "rewards/margins": 2.776582717895508,
      "rewards/rejected": -8.76605224609375,
      "step": 7155
    },
    {
      "epoch": 1.5508502111989602,
      "grad_norm": 22.0,
      "learning_rate": 9.207271086819317e-06,
      "logits/chosen": -3.093381404876709,
      "logits/rejected": -3.0862979888916016,
      "logps/chosen": -361.46514892578125,
      "logps/rejected": -414.2738342285156,
      "loss": 0.2655,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.279415607452393,
      "rewards/margins": 2.8306469917297363,
      "rewards/rejected": -9.110062599182129,
      "step": 7160
    },
    {
      "epoch": 1.5519332827899923,
      "grad_norm": 14.375,
      "learning_rate": 9.200780542586277e-06,
      "logits/chosen": -3.117537021636963,
      "logits/rejected": -3.1119112968444824,
      "logps/chosen": -335.0038146972656,
      "logps/rejected": -380.4847106933594,
      "loss": 0.316,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.5599446296691895,
      "rewards/margins": 2.315293788909912,
      "rewards/rejected": -7.875237464904785,
      "step": 7165
    },
    {
      "epoch": 1.5530163543810245,
      "grad_norm": 10.1875,
      "learning_rate": 9.194289801193118e-06,
      "logits/chosen": -2.9819259643554688,
      "logits/rejected": -3.038198709487915,
      "logps/chosen": -333.37841796875,
      "logps/rejected": -394.53466796875,
      "loss": 0.1837,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.402063369750977,
      "rewards/margins": 3.447713851928711,
      "rewards/rejected": -8.849777221679688,
      "step": 7170
    },
    {
      "epoch": 1.5540994259720566,
      "grad_norm": 28.0,
      "learning_rate": 9.187798871894325e-06,
      "logits/chosen": -3.0776610374450684,
      "logits/rejected": -3.135822296142578,
      "logps/chosen": -351.2132873535156,
      "logps/rejected": -388.8739318847656,
      "loss": 0.3012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.0241594314575195,
      "rewards/margins": 2.727292537689209,
      "rewards/rejected": -8.75145149230957,
      "step": 7175
    },
    {
      "epoch": 1.5551824975630888,
      "grad_norm": 11.375,
      "learning_rate": 9.181307763944652e-06,
      "logits/chosen": -3.324568271636963,
      "logits/rejected": -3.232135057449341,
      "logps/chosen": -343.6488037109375,
      "logps/rejected": -404.09979248046875,
      "loss": 0.2039,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.159006118774414,
      "rewards/margins": 3.1531991958618164,
      "rewards/rejected": -9.312204360961914,
      "step": 7180
    },
    {
      "epoch": 1.556265569154121,
      "grad_norm": 16.75,
      "learning_rate": 9.17481648659911e-06,
      "logits/chosen": -3.2154743671417236,
      "logits/rejected": -3.2116966247558594,
      "logps/chosen": -344.13543701171875,
      "logps/rejected": -404.68768310546875,
      "loss": 0.276,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -5.775395393371582,
      "rewards/margins": 3.198154926300049,
      "rewards/rejected": -8.973550796508789,
      "step": 7185
    },
    {
      "epoch": 1.5573486407451531,
      "grad_norm": 22.875,
      "learning_rate": 9.168325049112944e-06,
      "logits/chosen": -3.095475196838379,
      "logits/rejected": -3.312950849533081,
      "logps/chosen": -361.4244384765625,
      "logps/rejected": -398.70147705078125,
      "loss": 0.2328,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -6.579756259918213,
      "rewards/margins": 2.9224050045013428,
      "rewards/rejected": -9.502161026000977,
      "step": 7190
    },
    {
      "epoch": 1.5584317123361853,
      "grad_norm": 28.5,
      "learning_rate": 9.16183346074164e-06,
      "logits/chosen": -3.2568862438201904,
      "logits/rejected": -3.246135711669922,
      "logps/chosen": -335.12109375,
      "logps/rejected": -392.33135986328125,
      "loss": 0.2822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.971587181091309,
      "rewards/margins": 2.672891139984131,
      "rewards/rejected": -8.644477844238281,
      "step": 7195
    },
    {
      "epoch": 1.5595147839272174,
      "grad_norm": 16.125,
      "learning_rate": 9.155341730740884e-06,
      "logits/chosen": -3.180969715118408,
      "logits/rejected": -3.173091411590576,
      "logps/chosen": -326.48577880859375,
      "logps/rejected": -388.9509582519531,
      "loss": 0.2738,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.476136207580566,
      "rewards/margins": 3.161792278289795,
      "rewards/rejected": -8.637927055358887,
      "step": 7200
    },
    {
      "epoch": 1.5605978555182498,
      "grad_norm": 20.5,
      "learning_rate": 9.148849868366577e-06,
      "logits/chosen": -3.1250851154327393,
      "logits/rejected": -3.151798725128174,
      "logps/chosen": -329.99639892578125,
      "logps/rejected": -378.5079650878906,
      "loss": 0.2736,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.139610767364502,
      "rewards/margins": 2.707911729812622,
      "rewards/rejected": -8.847522735595703,
      "step": 7205
    },
    {
      "epoch": 1.561680927109282,
      "grad_norm": 16.375,
      "learning_rate": 9.142357882874799e-06,
      "logits/chosen": -3.2678284645080566,
      "logits/rejected": -3.2504525184631348,
      "logps/chosen": -338.6990966796875,
      "logps/rejected": -383.9197082519531,
      "loss": 0.2911,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.08066463470459,
      "rewards/margins": 2.6353156566619873,
      "rewards/rejected": -8.71597957611084,
      "step": 7210
    },
    {
      "epoch": 1.5627639987003141,
      "grad_norm": 33.25,
      "learning_rate": 9.135865783521816e-06,
      "logits/chosen": -3.2770233154296875,
      "logits/rejected": -3.198991298675537,
      "logps/chosen": -320.40130615234375,
      "logps/rejected": -387.40692138671875,
      "loss": 0.2899,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.536183834075928,
      "rewards/margins": 2.7606663703918457,
      "rewards/rejected": -8.296850204467773,
      "step": 7215
    },
    {
      "epoch": 1.5638470702913463,
      "grad_norm": 23.125,
      "learning_rate": 9.129373579564043e-06,
      "logits/chosen": -3.084336280822754,
      "logits/rejected": -3.2109744548797607,
      "logps/chosen": -316.57122802734375,
      "logps/rejected": -355.0462341308594,
      "loss": 0.2453,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.066257476806641,
      "rewards/margins": 2.7005701065063477,
      "rewards/rejected": -7.7668280601501465,
      "step": 7220
    },
    {
      "epoch": 1.5649301418823784,
      "grad_norm": 23.75,
      "learning_rate": 9.122881280258059e-06,
      "logits/chosen": -3.282500743865967,
      "logits/rejected": -3.1587679386138916,
      "logps/chosen": -336.5733642578125,
      "logps/rejected": -398.2720031738281,
      "loss": 0.2329,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.467576026916504,
      "rewards/margins": 3.241603136062622,
      "rewards/rejected": -8.709179878234863,
      "step": 7225
    },
    {
      "epoch": 1.5660132134734106,
      "grad_norm": 32.25,
      "learning_rate": 9.116388894860563e-06,
      "logits/chosen": -3.1553618907928467,
      "logits/rejected": -3.162926435470581,
      "logps/chosen": -334.8946228027344,
      "logps/rejected": -389.33624267578125,
      "loss": 0.2872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.675250053405762,
      "rewards/margins": 2.976215362548828,
      "rewards/rejected": -8.651464462280273,
      "step": 7230
    },
    {
      "epoch": 1.5670962850644428,
      "grad_norm": 22.75,
      "learning_rate": 9.109896432628391e-06,
      "logits/chosen": -3.155515193939209,
      "logits/rejected": -3.097283124923706,
      "logps/chosen": -329.89617919921875,
      "logps/rejected": -393.724609375,
      "loss": 0.2409,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.36275053024292,
      "rewards/margins": 3.030717372894287,
      "rewards/rejected": -8.393467903137207,
      "step": 7235
    },
    {
      "epoch": 1.568179356655475,
      "grad_norm": 18.875,
      "learning_rate": 9.10340390281848e-06,
      "logits/chosen": -3.1163315773010254,
      "logits/rejected": -3.17134165763855,
      "logps/chosen": -338.2666015625,
      "logps/rejected": -388.17205810546875,
      "loss": 0.2704,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.648530960083008,
      "rewards/margins": 2.830596446990967,
      "rewards/rejected": -8.479127883911133,
      "step": 7240
    },
    {
      "epoch": 1.569262428246507,
      "grad_norm": 16.875,
      "learning_rate": 9.096911314687866e-06,
      "logits/chosen": -2.987701892852783,
      "logits/rejected": -2.8872125148773193,
      "logps/chosen": -341.9576416015625,
      "logps/rejected": -403.52374267578125,
      "loss": 0.219,
      "rewards/accuracies": 0.9249998927116394,
      "rewards/chosen": -4.86241340637207,
      "rewards/margins": 3.1234140396118164,
      "rewards/rejected": -7.985827445983887,
      "step": 7245
    },
    {
      "epoch": 1.5703454998375392,
      "grad_norm": 21.75,
      "learning_rate": 9.09041867749367e-06,
      "logits/chosen": -3.223541259765625,
      "logits/rejected": -3.1639645099639893,
      "logps/chosen": -346.94183349609375,
      "logps/rejected": -410.3182067871094,
      "loss": 0.2713,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.382127285003662,
      "rewards/margins": 2.8205645084381104,
      "rewards/rejected": -9.202692031860352,
      "step": 7250
    },
    {
      "epoch": 1.5714285714285714,
      "grad_norm": 25.625,
      "learning_rate": 9.08392600049308e-06,
      "logits/chosen": -3.182525634765625,
      "logits/rejected": -3.3427422046661377,
      "logps/chosen": -336.73583984375,
      "logps/rejected": -399.39105224609375,
      "loss": 0.2867,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.048880100250244,
      "rewards/margins": 2.991844654083252,
      "rewards/rejected": -9.04072380065918,
      "step": 7255
    },
    {
      "epoch": 1.5725116430196036,
      "grad_norm": 25.625,
      "learning_rate": 9.077433292943339e-06,
      "logits/chosen": -3.132329225540161,
      "logits/rejected": -3.1778652667999268,
      "logps/chosen": -325.6788635253906,
      "logps/rejected": -374.24810791015625,
      "loss": 0.3362,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.474362373352051,
      "rewards/margins": 2.41854190826416,
      "rewards/rejected": -7.892904758453369,
      "step": 7260
    },
    {
      "epoch": 1.5735947146106357,
      "grad_norm": 20.5,
      "learning_rate": 9.070940564101743e-06,
      "logits/chosen": -3.093074083328247,
      "logits/rejected": -3.098426342010498,
      "logps/chosen": -331.56976318359375,
      "logps/rejected": -388.68206787109375,
      "loss": 0.2425,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.291831970214844,
      "rewards/margins": 3.165404796600342,
      "rewards/rejected": -8.457237243652344,
      "step": 7265
    },
    {
      "epoch": 1.574677786201668,
      "grad_norm": 31.875,
      "learning_rate": 9.064447823225605e-06,
      "logits/chosen": -3.2080607414245605,
      "logits/rejected": -3.1149096488952637,
      "logps/chosen": -314.63348388671875,
      "logps/rejected": -363.13031005859375,
      "loss": 0.2916,
      "rewards/accuracies": 0.841666579246521,
      "rewards/chosen": -5.369781494140625,
      "rewards/margins": 2.357081890106201,
      "rewards/rejected": -7.72686243057251,
      "step": 7270
    },
    {
      "epoch": 1.5757608577927003,
      "grad_norm": 27.875,
      "learning_rate": 9.057955079572268e-06,
      "logits/chosen": -3.1145455837249756,
      "logits/rejected": -3.175471782684326,
      "logps/chosen": -343.2930603027344,
      "logps/rejected": -386.50909423828125,
      "loss": 0.3417,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.842190742492676,
      "rewards/margins": 2.7031941413879395,
      "rewards/rejected": -8.545385360717773,
      "step": 7275
    },
    {
      "epoch": 1.5768439293837324,
      "grad_norm": 17.0,
      "learning_rate": 9.051462342399063e-06,
      "logits/chosen": -2.9288244247436523,
      "logits/rejected": -3.003889799118042,
      "logps/chosen": -343.10125732421875,
      "logps/rejected": -381.0269470214844,
      "loss": 0.2306,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.209383487701416,
      "rewards/margins": 2.669111490249634,
      "rewards/rejected": -7.8784942626953125,
      "step": 7280
    },
    {
      "epoch": 1.5779270009747646,
      "grad_norm": 20.5,
      "learning_rate": 9.044969620963332e-06,
      "logits/chosen": -3.184770107269287,
      "logits/rejected": -3.15016508102417,
      "logps/chosen": -328.24822998046875,
      "logps/rejected": -387.50067138671875,
      "loss": 0.2747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.757481098175049,
      "rewards/margins": 2.7384109497070312,
      "rewards/rejected": -8.495891571044922,
      "step": 7285
    },
    {
      "epoch": 1.5790100725657967,
      "grad_norm": 20.5,
      "learning_rate": 9.038476924522378e-06,
      "logits/chosen": -3.065070390701294,
      "logits/rejected": -3.010077714920044,
      "logps/chosen": -346.4923400878906,
      "logps/rejected": -389.78338623046875,
      "loss": 0.2856,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.695969104766846,
      "rewards/margins": 2.518632411956787,
      "rewards/rejected": -8.214601516723633,
      "step": 7290
    },
    {
      "epoch": 1.5800931441568289,
      "grad_norm": 24.625,
      "learning_rate": 9.031984262333482e-06,
      "logits/chosen": -3.098944902420044,
      "logits/rejected": -3.0679967403411865,
      "logps/chosen": -344.8092346191406,
      "logps/rejected": -406.4029541015625,
      "loss": 0.2246,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -5.887304306030273,
      "rewards/margins": 2.8319759368896484,
      "rewards/rejected": -8.719280242919922,
      "step": 7295
    },
    {
      "epoch": 1.581176215747861,
      "grad_norm": 23.5,
      "learning_rate": 9.025491643653857e-06,
      "logits/chosen": -3.344529628753662,
      "logits/rejected": -3.221583843231201,
      "logps/chosen": -309.11639404296875,
      "logps/rejected": -370.278076171875,
      "loss": 0.2375,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.54559850692749,
      "rewards/margins": 2.8472628593444824,
      "rewards/rejected": -8.392862319946289,
      "step": 7300
    },
    {
      "epoch": 1.5822592873388932,
      "grad_norm": 14.4375,
      "learning_rate": 9.018999077740677e-06,
      "logits/chosen": -3.126685619354248,
      "logits/rejected": -3.1116268634796143,
      "logps/chosen": -353.23046875,
      "logps/rejected": -414.37054443359375,
      "loss": 0.2497,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.087141036987305,
      "rewards/margins": 3.1787381172180176,
      "rewards/rejected": -9.265878677368164,
      "step": 7305
    },
    {
      "epoch": 1.5833423589299254,
      "grad_norm": 22.375,
      "learning_rate": 9.012506573851022e-06,
      "logits/chosen": -3.1274209022521973,
      "logits/rejected": -3.057802200317383,
      "logps/chosen": -342.0838928222656,
      "logps/rejected": -414.049560546875,
      "loss": 0.2364,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.369519233703613,
      "rewards/margins": 3.4841530323028564,
      "rewards/rejected": -8.853672981262207,
      "step": 7310
    },
    {
      "epoch": 1.5844254305209575,
      "grad_norm": 17.625,
      "learning_rate": 9.006014141241895e-06,
      "logits/chosen": -3.2305667400360107,
      "logits/rejected": -3.088601589202881,
      "logps/chosen": -326.46905517578125,
      "logps/rejected": -401.97662353515625,
      "loss": 0.2356,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.525664806365967,
      "rewards/margins": 3.352860689163208,
      "rewards/rejected": -8.87852668762207,
      "step": 7315
    },
    {
      "epoch": 1.5855085021119897,
      "grad_norm": 12.5625,
      "learning_rate": 8.999521789170189e-06,
      "logits/chosen": -3.201580762863159,
      "logits/rejected": -3.107903480529785,
      "logps/chosen": -343.2080383300781,
      "logps/rejected": -393.21990966796875,
      "loss": 0.2407,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.928816795349121,
      "rewards/margins": 3.051546335220337,
      "rewards/rejected": -8.980362892150879,
      "step": 7320
    },
    {
      "epoch": 1.5865915737030218,
      "grad_norm": 22.625,
      "learning_rate": 8.993029526892692e-06,
      "logits/chosen": -3.0582547187805176,
      "logits/rejected": -3.127194881439209,
      "logps/chosen": -342.2777404785156,
      "logps/rejected": -397.2563781738281,
      "loss": 0.2514,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.5637664794921875,
      "rewards/margins": 3.0752978324890137,
      "rewards/rejected": -8.639063835144043,
      "step": 7325
    },
    {
      "epoch": 1.587674645294054,
      "grad_norm": 22.0,
      "learning_rate": 8.986537363666053e-06,
      "logits/chosen": -3.189532518386841,
      "logits/rejected": -3.1647515296936035,
      "logps/chosen": -348.50628662109375,
      "logps/rejected": -404.6119384765625,
      "loss": 0.2144,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.690191268920898,
      "rewards/margins": 3.3319034576416016,
      "rewards/rejected": -9.022092819213867,
      "step": 7330
    },
    {
      "epoch": 1.5887577168850862,
      "grad_norm": 18.375,
      "learning_rate": 8.98004530874679e-06,
      "logits/chosen": -3.188483715057373,
      "logits/rejected": -3.1066930294036865,
      "logps/chosen": -341.5953369140625,
      "logps/rejected": -402.008544921875,
      "loss": 0.2685,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.1467132568359375,
      "rewards/margins": 2.802535057067871,
      "rewards/rejected": -8.949247360229492,
      "step": 7335
    },
    {
      "epoch": 1.5898407884761183,
      "grad_norm": 28.625,
      "learning_rate": 8.97355337139126e-06,
      "logits/chosen": -3.1860194206237793,
      "logits/rejected": -3.3381245136260986,
      "logps/chosen": -351.83074951171875,
      "logps/rejected": -394.5777893066406,
      "loss": 0.2581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.387605667114258,
      "rewards/margins": 2.7167136669158936,
      "rewards/rejected": -9.104320526123047,
      "step": 7340
    },
    {
      "epoch": 1.5909238600671505,
      "grad_norm": 9.1875,
      "learning_rate": 8.967061560855654e-06,
      "logits/chosen": -3.2117104530334473,
      "logits/rejected": -3.2493293285369873,
      "logps/chosen": -355.6941833496094,
      "logps/rejected": -426.19171142578125,
      "loss": 0.1887,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.145462512969971,
      "rewards/margins": 3.4538276195526123,
      "rewards/rejected": -9.59929084777832,
      "step": 7345
    },
    {
      "epoch": 1.5920069316581826,
      "grad_norm": 19.25,
      "learning_rate": 8.960569886395983e-06,
      "logits/chosen": -3.2385764122009277,
      "logits/rejected": -3.2788288593292236,
      "logps/chosen": -350.33599853515625,
      "logps/rejected": -393.58587646484375,
      "loss": 0.2969,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.5408806800842285,
      "rewards/margins": 2.489666223526001,
      "rewards/rejected": -9.030546188354492,
      "step": 7350
    },
    {
      "epoch": 1.5930900032492148,
      "grad_norm": 18.5,
      "learning_rate": 8.954078357268061e-06,
      "logits/chosen": -3.101475477218628,
      "logits/rejected": -3.0791263580322266,
      "logps/chosen": -354.9466247558594,
      "logps/rejected": -398.69830322265625,
      "loss": 0.2714,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.082449913024902,
      "rewards/margins": 2.7042126655578613,
      "rewards/rejected": -8.786663055419922,
      "step": 7355
    },
    {
      "epoch": 1.594173074840247,
      "grad_norm": 13.4375,
      "learning_rate": 8.9475869827275e-06,
      "logits/chosen": -3.1955647468566895,
      "logits/rejected": -3.1670868396759033,
      "logps/chosen": -337.99127197265625,
      "logps/rejected": -398.53338623046875,
      "loss": 0.3454,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.164637565612793,
      "rewards/margins": 2.709900140762329,
      "rewards/rejected": -8.874539375305176,
      "step": 7360
    },
    {
      "epoch": 1.595256146431279,
      "grad_norm": 31.5,
      "learning_rate": 8.941095772029689e-06,
      "logits/chosen": -3.2993264198303223,
      "logits/rejected": -3.2054481506347656,
      "logps/chosen": -352.71826171875,
      "logps/rejected": -402.976806640625,
      "loss": 0.2616,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.460489749908447,
      "rewards/margins": 2.736056089401245,
      "rewards/rejected": -9.196545600891113,
      "step": 7365
    },
    {
      "epoch": 1.5963392180223113,
      "grad_norm": 20.0,
      "learning_rate": 8.93460473442978e-06,
      "logits/chosen": -3.2520527839660645,
      "logits/rejected": -3.260105609893799,
      "logps/chosen": -361.86529541015625,
      "logps/rejected": -398.85064697265625,
      "loss": 0.2982,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.998593330383301,
      "rewards/margins": 2.6108827590942383,
      "rewards/rejected": -9.609476089477539,
      "step": 7370
    },
    {
      "epoch": 1.5974222896133434,
      "grad_norm": 29.875,
      "learning_rate": 8.928113879182687e-06,
      "logits/chosen": -3.0813064575195312,
      "logits/rejected": -3.0591490268707275,
      "logps/chosen": -348.7625732421875,
      "logps/rejected": -398.8260803222656,
      "loss": 0.2406,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.989401817321777,
      "rewards/margins": 2.823481559753418,
      "rewards/rejected": -8.812884330749512,
      "step": 7375
    },
    {
      "epoch": 1.5985053612043756,
      "grad_norm": 31.625,
      "learning_rate": 8.92162321554305e-06,
      "logits/chosen": -3.174172878265381,
      "logits/rejected": -3.2637248039245605,
      "logps/chosen": -338.4259033203125,
      "logps/rejected": -382.30535888671875,
      "loss": 0.2367,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.424201488494873,
      "rewards/margins": 3.0214829444885254,
      "rewards/rejected": -8.445684432983398,
      "step": 7380
    },
    {
      "epoch": 1.5995884327954077,
      "grad_norm": 26.875,
      "learning_rate": 8.915132752765253e-06,
      "logits/chosen": -3.1462321281433105,
      "logits/rejected": -3.1988625526428223,
      "logps/chosen": -356.0948181152344,
      "logps/rejected": -397.1586608886719,
      "loss": 0.2899,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.4254961013793945,
      "rewards/margins": 2.585278034210205,
      "rewards/rejected": -9.010774612426758,
      "step": 7385
    },
    {
      "epoch": 1.60067150438644,
      "grad_norm": 24.25,
      "learning_rate": 8.90864250010338e-06,
      "logits/chosen": -3.223494291305542,
      "logits/rejected": -3.206655502319336,
      "logps/chosen": -336.4815979003906,
      "logps/rejected": -384.75262451171875,
      "loss": 0.2467,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.692349433898926,
      "rewards/margins": 2.8490636348724365,
      "rewards/rejected": -8.541413307189941,
      "step": 7390
    },
    {
      "epoch": 1.601754575977472,
      "grad_norm": 30.125,
      "learning_rate": 8.902152466811217e-06,
      "logits/chosen": -3.2335457801818848,
      "logits/rejected": -3.239440441131592,
      "logps/chosen": -366.40692138671875,
      "logps/rejected": -413.24639892578125,
      "loss": 0.3399,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -6.83834981918335,
      "rewards/margins": 2.5104947090148926,
      "rewards/rejected": -9.348845481872559,
      "step": 7395
    },
    {
      "epoch": 1.6028376475685042,
      "grad_norm": 12.375,
      "learning_rate": 8.895662662142246e-06,
      "logits/chosen": -3.1739306449890137,
      "logits/rejected": -3.2162437438964844,
      "logps/chosen": -354.8019714355469,
      "logps/rejected": -412.65057373046875,
      "loss": 0.2006,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.1840620040893555,
      "rewards/margins": 2.960634231567383,
      "rewards/rejected": -9.144696235656738,
      "step": 7400
    },
    {
      "epoch": 1.6039207191595364,
      "grad_norm": 15.625,
      "learning_rate": 8.889173095349619e-06,
      "logits/chosen": -3.101254940032959,
      "logits/rejected": -3.1201605796813965,
      "logps/chosen": -343.15411376953125,
      "logps/rejected": -402.9792175292969,
      "loss": 0.2137,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -5.446257591247559,
      "rewards/margins": 3.2898056507110596,
      "rewards/rejected": -8.736063957214355,
      "step": 7405
    },
    {
      "epoch": 1.6050037907505685,
      "grad_norm": 23.75,
      "learning_rate": 8.882683775686141e-06,
      "logits/chosen": -3.286433696746826,
      "logits/rejected": -3.213118314743042,
      "logps/chosen": -330.2290344238281,
      "logps/rejected": -386.27130126953125,
      "loss": 0.2622,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.111742973327637,
      "rewards/margins": 2.9333388805389404,
      "rewards/rejected": -8.045080184936523,
      "step": 7410
    },
    {
      "epoch": 1.6060868623416007,
      "grad_norm": 16.625,
      "learning_rate": 8.876194712404274e-06,
      "logits/chosen": -3.1605310440063477,
      "logits/rejected": -3.167250871658325,
      "logps/chosen": -349.9772033691406,
      "logps/rejected": -404.82977294921875,
      "loss": 0.2323,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.945608615875244,
      "rewards/margins": 2.8681552410125732,
      "rewards/rejected": -8.813764572143555,
      "step": 7415
    },
    {
      "epoch": 1.6071699339326329,
      "grad_norm": 26.0,
      "learning_rate": 8.869705914756112e-06,
      "logits/chosen": -3.2742226123809814,
      "logits/rejected": -3.281130313873291,
      "logps/chosen": -347.4670104980469,
      "logps/rejected": -389.45660400390625,
      "loss": 0.342,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.266068458557129,
      "rewards/margins": 2.8554441928863525,
      "rewards/rejected": -9.121514320373535,
      "step": 7420
    },
    {
      "epoch": 1.608253005523665,
      "grad_norm": 18.625,
      "learning_rate": 8.863217391993368e-06,
      "logits/chosen": -3.2139370441436768,
      "logits/rejected": -3.188356876373291,
      "logps/chosen": -333.25775146484375,
      "logps/rejected": -389.935302734375,
      "loss": 0.2527,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.678569793701172,
      "rewards/margins": 2.9923062324523926,
      "rewards/rejected": -8.670875549316406,
      "step": 7425
    },
    {
      "epoch": 1.6093360771146972,
      "grad_norm": 18.75,
      "learning_rate": 8.856729153367367e-06,
      "logits/chosen": -3.1449368000030518,
      "logits/rejected": -3.1476962566375732,
      "logps/chosen": -335.5579833984375,
      "logps/rejected": -386.47186279296875,
      "loss": 0.2238,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.513119697570801,
      "rewards/margins": 2.9329991340637207,
      "rewards/rejected": -8.44611930847168,
      "step": 7430
    },
    {
      "epoch": 1.6104191487057293,
      "grad_norm": 22.0,
      "learning_rate": 8.850241208129023e-06,
      "logits/chosen": -3.0342559814453125,
      "logits/rejected": -3.0234999656677246,
      "logps/chosen": -340.0980224609375,
      "logps/rejected": -392.82574462890625,
      "loss": 0.2433,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.564701557159424,
      "rewards/margins": 2.7949776649475098,
      "rewards/rejected": -8.359679222106934,
      "step": 7435
    },
    {
      "epoch": 1.6115022202967615,
      "grad_norm": 21.875,
      "learning_rate": 8.843753565528838e-06,
      "logits/chosen": -3.3382811546325684,
      "logits/rejected": -3.286038875579834,
      "logps/chosen": -320.7479553222656,
      "logps/rejected": -363.03900146484375,
      "loss": 0.2908,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.6523942947387695,
      "rewards/margins": 2.4483370780944824,
      "rewards/rejected": -8.10073184967041,
      "step": 7440
    },
    {
      "epoch": 1.6125852918877936,
      "grad_norm": 21.375,
      "learning_rate": 8.837266234816882e-06,
      "logits/chosen": -3.1533732414245605,
      "logits/rejected": -3.2032132148742676,
      "logps/chosen": -355.71484375,
      "logps/rejected": -399.49468994140625,
      "loss": 0.2987,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.998322010040283,
      "rewards/margins": 2.750792980194092,
      "rewards/rejected": -8.749114990234375,
      "step": 7445
    },
    {
      "epoch": 1.6136683634788258,
      "grad_norm": 21.0,
      "learning_rate": 8.830779225242772e-06,
      "logits/chosen": -3.131357431411743,
      "logits/rejected": -3.10942006111145,
      "logps/chosen": -333.17071533203125,
      "logps/rejected": -382.275146484375,
      "loss": 0.2494,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.307631492614746,
      "rewards/margins": 2.8117194175720215,
      "rewards/rejected": -8.11935043334961,
      "step": 7450
    },
    {
      "epoch": 1.614751435069858,
      "grad_norm": 25.0,
      "learning_rate": 8.824292546055677e-06,
      "logits/chosen": -3.1096084117889404,
      "logits/rejected": -3.2667078971862793,
      "logps/chosen": -331.7309875488281,
      "logps/rejected": -369.59356689453125,
      "loss": 0.3516,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.037100791931152,
      "rewards/margins": 2.2317092418670654,
      "rewards/rejected": -8.26880931854248,
      "step": 7455
    },
    {
      "epoch": 1.6158345066608901,
      "grad_norm": 14.5,
      "learning_rate": 8.817806206504293e-06,
      "logits/chosen": -3.055266857147217,
      "logits/rejected": -3.1348989009857178,
      "logps/chosen": -343.0975036621094,
      "logps/rejected": -394.963134765625,
      "loss": 0.214,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -5.793118000030518,
      "rewards/margins": 2.966879367828369,
      "rewards/rejected": -8.759997367858887,
      "step": 7460
    },
    {
      "epoch": 1.6169175782519225,
      "grad_norm": 17.0,
      "learning_rate": 8.811320215836825e-06,
      "logits/chosen": -3.104365110397339,
      "logits/rejected": -3.0914173126220703,
      "logps/chosen": -326.1053161621094,
      "logps/rejected": -386.61822509765625,
      "loss": 0.2687,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.575974941253662,
      "rewards/margins": 2.7492542266845703,
      "rewards/rejected": -8.325228691101074,
      "step": 7465
    },
    {
      "epoch": 1.6180006498429547,
      "grad_norm": 27.125,
      "learning_rate": 8.80483458330099e-06,
      "logits/chosen": -3.1688013076782227,
      "logits/rejected": -3.2244255542755127,
      "logps/chosen": -330.3938293457031,
      "logps/rejected": -382.0798645019531,
      "loss": 0.2708,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.152255058288574,
      "rewards/margins": 2.987567663192749,
      "rewards/rejected": -8.139822959899902,
      "step": 7470
    },
    {
      "epoch": 1.6190837214339868,
      "grad_norm": 11.3125,
      "learning_rate": 8.798349318143985e-06,
      "logits/chosen": -3.121403932571411,
      "logits/rejected": -2.9765725135803223,
      "logps/chosen": -326.88848876953125,
      "logps/rejected": -387.26123046875,
      "loss": 0.28,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -4.94266939163208,
      "rewards/margins": 2.7104313373565674,
      "rewards/rejected": -7.65310001373291,
      "step": 7475
    },
    {
      "epoch": 1.620166793025019,
      "grad_norm": 25.875,
      "learning_rate": 8.791864429612493e-06,
      "logits/chosen": -3.1896796226501465,
      "logits/rejected": -3.2268600463867188,
      "logps/chosen": -331.646484375,
      "logps/rejected": -385.2218017578125,
      "loss": 0.277,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.624708652496338,
      "rewards/margins": 3.0100209712982178,
      "rewards/rejected": -8.634729385375977,
      "step": 7480
    },
    {
      "epoch": 1.6212498646160511,
      "grad_norm": 10.75,
      "learning_rate": 8.785379926952654e-06,
      "logits/chosen": -3.2910170555114746,
      "logits/rejected": -3.1759183406829834,
      "logps/chosen": -331.63018798828125,
      "logps/rejected": -397.29498291015625,
      "loss": 0.1914,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.692002773284912,
      "rewards/margins": 3.3317553997039795,
      "rewards/rejected": -9.023758888244629,
      "step": 7485
    },
    {
      "epoch": 1.6223329362070833,
      "grad_norm": 13.5,
      "learning_rate": 8.778895819410054e-06,
      "logits/chosen": -3.097079038619995,
      "logits/rejected": -3.1162827014923096,
      "logps/chosen": -335.18487548828125,
      "logps/rejected": -386.95172119140625,
      "loss": 0.267,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.413605213165283,
      "rewards/margins": 2.9538116455078125,
      "rewards/rejected": -8.367417335510254,
      "step": 7490
    },
    {
      "epoch": 1.6234160077981155,
      "grad_norm": 23.625,
      "learning_rate": 8.772412116229728e-06,
      "logits/chosen": -3.209641933441162,
      "logits/rejected": -3.085801839828491,
      "logps/chosen": -348.84765625,
      "logps/rejected": -410.14208984375,
      "loss": 0.2091,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.621572971343994,
      "rewards/margins": 2.9860432147979736,
      "rewards/rejected": -8.60761547088623,
      "step": 7495
    },
    {
      "epoch": 1.6244990793891476,
      "grad_norm": 25.875,
      "learning_rate": 8.765928826656121e-06,
      "logits/chosen": -3.168360471725464,
      "logits/rejected": -3.0885837078094482,
      "logps/chosen": -342.59356689453125,
      "logps/rejected": -398.3309020996094,
      "loss": 0.2114,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.569211483001709,
      "rewards/margins": 2.9129252433776855,
      "rewards/rejected": -8.482137680053711,
      "step": 7500
    },
    {
      "epoch": 1.6244990793891476,
      "eval_logits/chosen": -3.3348965644836426,
      "eval_logits/rejected": -3.2649354934692383,
      "eval_logps/chosen": -328.1272277832031,
      "eval_logps/rejected": -376.4630432128906,
      "eval_loss": 0.3966388702392578,
      "eval_rewards/accuracies": 0.824999988079071,
      "eval_rewards/chosen": -5.696298599243164,
      "eval_rewards/margins": 2.24399733543396,
      "eval_rewards/rejected": -7.940296173095703,
      "eval_runtime": 14.5936,
      "eval_samples_per_second": 13.705,
      "eval_steps_per_second": 13.705,
      "step": 7500
    },
    {
      "epoch": 1.6255821509801798,
      "grad_norm": 22.0,
      "learning_rate": 8.759445959933096e-06,
      "logits/chosen": -3.1493000984191895,
      "logits/rejected": -3.1301684379577637,
      "logps/chosen": -354.94659423828125,
      "logps/rejected": -407.7454528808594,
      "loss": 0.2674,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.240975379943848,
      "rewards/margins": 2.924854278564453,
      "rewards/rejected": -9.1658296585083,
      "step": 7505
    },
    {
      "epoch": 1.626665222571212,
      "grad_norm": 21.75,
      "learning_rate": 8.75296352530391e-06,
      "logits/chosen": -3.352530002593994,
      "logits/rejected": -3.3176512718200684,
      "logps/chosen": -349.30279541015625,
      "logps/rejected": -398.62640380859375,
      "loss": 0.2236,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.353281021118164,
      "rewards/margins": 2.8709139823913574,
      "rewards/rejected": -9.224194526672363,
      "step": 7510
    },
    {
      "epoch": 1.627748294162244,
      "grad_norm": 17.375,
      "learning_rate": 8.746481532011211e-06,
      "logits/chosen": -3.2870330810546875,
      "logits/rejected": -3.3633217811584473,
      "logps/chosen": -339.8268127441406,
      "logps/rejected": -377.9615783691406,
      "loss": 0.269,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -6.131731986999512,
      "rewards/margins": 2.541926860809326,
      "rewards/rejected": -8.67365837097168,
      "step": 7515
    },
    {
      "epoch": 1.6288313657532763,
      "grad_norm": 21.5,
      "learning_rate": 8.739999989297005e-06,
      "logits/chosen": -3.282248020172119,
      "logits/rejected": -3.2851626873016357,
      "logps/chosen": -352.33636474609375,
      "logps/rejected": -402.34161376953125,
      "loss": 0.2445,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.873376846313477,
      "rewards/margins": 3.059209108352661,
      "rewards/rejected": -8.932585716247559,
      "step": 7520
    },
    {
      "epoch": 1.6299144373443084,
      "grad_norm": 19.75,
      "learning_rate": 8.733518906402669e-06,
      "logits/chosen": -3.1225366592407227,
      "logits/rejected": -3.1595640182495117,
      "logps/chosen": -346.83880615234375,
      "logps/rejected": -393.5505065917969,
      "loss": 0.24,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.847884178161621,
      "rewards/margins": 2.7460217475891113,
      "rewards/rejected": -8.593905448913574,
      "step": 7525
    },
    {
      "epoch": 1.6309975089353408,
      "grad_norm": 19.875,
      "learning_rate": 8.72703829256891e-06,
      "logits/chosen": -3.2148067951202393,
      "logits/rejected": -3.1660947799682617,
      "logps/chosen": -361.98626708984375,
      "logps/rejected": -413.90960693359375,
      "loss": 0.2609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.3443403244018555,
      "rewards/margins": 2.728811740875244,
      "rewards/rejected": -9.073151588439941,
      "step": 7530
    },
    {
      "epoch": 1.632080580526373,
      "grad_norm": 37.0,
      "learning_rate": 8.720558157035783e-06,
      "logits/chosen": -3.2835872173309326,
      "logits/rejected": -3.3409886360168457,
      "logps/chosen": -356.2604064941406,
      "logps/rejected": -399.6597900390625,
      "loss": 0.2905,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.151839733123779,
      "rewards/margins": 3.078937530517578,
      "rewards/rejected": -9.230777740478516,
      "step": 7535
    },
    {
      "epoch": 1.633163652117405,
      "grad_norm": 21.375,
      "learning_rate": 8.714078509042646e-06,
      "logits/chosen": -3.3196723461151123,
      "logits/rejected": -3.180068254470825,
      "logps/chosen": -351.47003173828125,
      "logps/rejected": -409.4732971191406,
      "loss": 0.2589,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.406506538391113,
      "rewards/margins": 2.657386064529419,
      "rewards/rejected": -9.063891410827637,
      "step": 7540
    },
    {
      "epoch": 1.6342467237084373,
      "grad_norm": 27.875,
      "learning_rate": 8.707599357828166e-06,
      "logits/chosen": -3.1308093070983887,
      "logits/rejected": -3.1083908081054688,
      "logps/chosen": -333.94696044921875,
      "logps/rejected": -378.79461669921875,
      "loss": 0.3208,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.537535190582275,
      "rewards/margins": 2.491121768951416,
      "rewards/rejected": -8.028657913208008,
      "step": 7545
    },
    {
      "epoch": 1.6353297952994694,
      "grad_norm": 19.75,
      "learning_rate": 8.701120712630311e-06,
      "logits/chosen": -3.21419095993042,
      "logits/rejected": -3.134399890899658,
      "logps/chosen": -345.0562438964844,
      "logps/rejected": -396.74066162109375,
      "loss": 0.2542,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.9196600914001465,
      "rewards/margins": 2.848421335220337,
      "rewards/rejected": -8.768081665039062,
      "step": 7550
    },
    {
      "epoch": 1.6364128668905016,
      "grad_norm": 24.375,
      "learning_rate": 8.694642582686319e-06,
      "logits/chosen": -3.053743600845337,
      "logits/rejected": -3.098557472229004,
      "logps/chosen": -338.1480712890625,
      "logps/rejected": -378.8416442871094,
      "loss": 0.2853,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.462432861328125,
      "rewards/margins": 2.5650577545166016,
      "rewards/rejected": -8.027490615844727,
      "step": 7555
    },
    {
      "epoch": 1.6374959384815337,
      "grad_norm": 13.6875,
      "learning_rate": 8.68816497723269e-06,
      "logits/chosen": -3.2384495735168457,
      "logits/rejected": -3.2198500633239746,
      "logps/chosen": -329.4116516113281,
      "logps/rejected": -387.4097900390625,
      "loss": 0.213,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.872766971588135,
      "rewards/margins": 2.8978989124298096,
      "rewards/rejected": -8.770665168762207,
      "step": 7560
    },
    {
      "epoch": 1.638579010072566,
      "grad_norm": 11.25,
      "learning_rate": 8.681687905505183e-06,
      "logits/chosen": -3.152956008911133,
      "logits/rejected": -3.1303601264953613,
      "logps/chosen": -333.58544921875,
      "logps/rejected": -385.5212707519531,
      "loss": 0.2115,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.52278470993042,
      "rewards/margins": 3.1235244274139404,
      "rewards/rejected": -8.646309852600098,
      "step": 7565
    },
    {
      "epoch": 1.639662081663598,
      "grad_norm": 23.5,
      "learning_rate": 8.675211376738798e-06,
      "logits/chosen": -3.2461891174316406,
      "logits/rejected": -3.0915915966033936,
      "logps/chosen": -333.9050598144531,
      "logps/rejected": -388.32269287109375,
      "loss": 0.2473,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.543787479400635,
      "rewards/margins": 2.827558994293213,
      "rewards/rejected": -8.371346473693848,
      "step": 7570
    },
    {
      "epoch": 1.6407451532546302,
      "grad_norm": 30.375,
      "learning_rate": 8.668735400167747e-06,
      "logits/chosen": -3.1717617511749268,
      "logits/rejected": -3.050631046295166,
      "logps/chosen": -333.7986755371094,
      "logps/rejected": -400.29632568359375,
      "loss": 0.2597,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.686055660247803,
      "rewards/margins": 3.0523641109466553,
      "rewards/rejected": -8.738419532775879,
      "step": 7575
    },
    {
      "epoch": 1.6418282248456624,
      "grad_norm": 27.625,
      "learning_rate": 8.662259985025473e-06,
      "logits/chosen": -3.1494765281677246,
      "logits/rejected": -3.1212735176086426,
      "logps/chosen": -366.8988952636719,
      "logps/rejected": -415.490966796875,
      "loss": 0.3007,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.537384033203125,
      "rewards/margins": 2.724135160446167,
      "rewards/rejected": -9.261520385742188,
      "step": 7580
    },
    {
      "epoch": 1.6429112964366945,
      "grad_norm": 19.25,
      "learning_rate": 8.655785140544608e-06,
      "logits/chosen": -3.1489691734313965,
      "logits/rejected": -3.168165683746338,
      "logps/chosen": -353.7528381347656,
      "logps/rejected": -405.766357421875,
      "loss": 0.3041,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.817262172698975,
      "rewards/margins": 2.6297574043273926,
      "rewards/rejected": -9.447019577026367,
      "step": 7585
    },
    {
      "epoch": 1.6439943680277267,
      "grad_norm": 34.25,
      "learning_rate": 8.649310875956972e-06,
      "logits/chosen": -3.1064462661743164,
      "logits/rejected": -3.1826577186584473,
      "logps/chosen": -347.4754333496094,
      "logps/rejected": -390.21734619140625,
      "loss": 0.2335,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.804431915283203,
      "rewards/margins": 3.070131778717041,
      "rewards/rejected": -8.874563217163086,
      "step": 7590
    },
    {
      "epoch": 1.6450774396187589,
      "grad_norm": 16.25,
      "learning_rate": 8.642837200493555e-06,
      "logits/chosen": -3.3098092079162598,
      "logits/rejected": -3.2692527770996094,
      "logps/chosen": -339.75445556640625,
      "logps/rejected": -399.4328918457031,
      "loss": 0.2293,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.978736877441406,
      "rewards/margins": 3.214130401611328,
      "rewards/rejected": -9.19286823272705,
      "step": 7595
    },
    {
      "epoch": 1.646160511209791,
      "grad_norm": 27.375,
      "learning_rate": 8.636364123384517e-06,
      "logits/chosen": -3.2360165119171143,
      "logits/rejected": -3.340480327606201,
      "logps/chosen": -338.483642578125,
      "logps/rejected": -377.67755126953125,
      "loss": 0.3368,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.124659061431885,
      "rewards/margins": 2.7478818893432617,
      "rewards/rejected": -8.872541427612305,
      "step": 7600
    },
    {
      "epoch": 1.6472435828008232,
      "grad_norm": 21.25,
      "learning_rate": 8.629891653859153e-06,
      "logits/chosen": -3.0713984966278076,
      "logits/rejected": -3.0824971199035645,
      "logps/chosen": -326.51220703125,
      "logps/rejected": -384.30987548828125,
      "loss": 0.2092,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.220745086669922,
      "rewards/margins": 2.876854419708252,
      "rewards/rejected": -8.097599983215332,
      "step": 7605
    },
    {
      "epoch": 1.6483266543918553,
      "grad_norm": 23.75,
      "learning_rate": 8.6234198011459e-06,
      "logits/chosen": -3.132930278778076,
      "logits/rejected": -3.0950958728790283,
      "logps/chosen": -355.66961669921875,
      "logps/rejected": -416.14141845703125,
      "loss": 0.2511,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.831258773803711,
      "rewards/margins": 3.077399730682373,
      "rewards/rejected": -8.908658981323242,
      "step": 7610
    },
    {
      "epoch": 1.6494097259828875,
      "grad_norm": 26.5,
      "learning_rate": 8.616948574472307e-06,
      "logits/chosen": -3.0442841053009033,
      "logits/rejected": -3.0511221885681152,
      "logps/chosen": -372.9717712402344,
      "logps/rejected": -430.70751953125,
      "loss": 0.2721,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -6.016740798950195,
      "rewards/margins": 2.8477160930633545,
      "rewards/rejected": -8.864458084106445,
      "step": 7615
    },
    {
      "epoch": 1.6504927975739196,
      "grad_norm": 19.875,
      "learning_rate": 8.610477983065041e-06,
      "logits/chosen": -3.336956739425659,
      "logits/rejected": -3.249951124191284,
      "logps/chosen": -324.8174133300781,
      "logps/rejected": -394.0953674316406,
      "loss": 0.2316,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.003142356872559,
      "rewards/margins": 2.994767665863037,
      "rewards/rejected": -8.99791145324707,
      "step": 7620
    },
    {
      "epoch": 1.6515758691649518,
      "grad_norm": 25.375,
      "learning_rate": 8.604008036149861e-06,
      "logits/chosen": -3.1852970123291016,
      "logits/rejected": -3.1330933570861816,
      "logps/chosen": -339.45849609375,
      "logps/rejected": -398.9294738769531,
      "loss": 0.1978,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -5.351485729217529,
      "rewards/margins": 3.3370258808135986,
      "rewards/rejected": -8.688512802124023,
      "step": 7625
    },
    {
      "epoch": 1.652658940755984,
      "grad_norm": 39.75,
      "learning_rate": 8.597538742951598e-06,
      "logits/chosen": -3.088818073272705,
      "logits/rejected": -3.1517651081085205,
      "logps/chosen": -359.447998046875,
      "logps/rejected": -415.9283752441406,
      "loss": 0.2812,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.423519134521484,
      "rewards/margins": 2.8427529335021973,
      "rewards/rejected": -9.266271591186523,
      "step": 7630
    },
    {
      "epoch": 1.6537420123470161,
      "grad_norm": 26.375,
      "learning_rate": 8.59107011269416e-06,
      "logits/chosen": -3.1771016120910645,
      "logits/rejected": -3.2118899822235107,
      "logps/chosen": -345.99932861328125,
      "logps/rejected": -400.66925048828125,
      "loss": 0.2117,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -5.608952522277832,
      "rewards/margins": 3.4428253173828125,
      "rewards/rejected": -9.051777839660645,
      "step": 7635
    },
    {
      "epoch": 1.6548250839380483,
      "grad_norm": 25.125,
      "learning_rate": 8.584602154600509e-06,
      "logits/chosen": -3.1436707973480225,
      "logits/rejected": -3.0212481021881104,
      "logps/chosen": -346.3398742675781,
      "logps/rejected": -400.9181213378906,
      "loss": 0.2899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.312403678894043,
      "rewards/margins": 2.932797908782959,
      "rewards/rejected": -9.245201110839844,
      "step": 7640
    },
    {
      "epoch": 1.6559081555290804,
      "grad_norm": 7.71875,
      "learning_rate": 8.578134877892644e-06,
      "logits/chosen": -3.172119379043579,
      "logits/rejected": -3.134897470474243,
      "logps/chosen": -333.6301574707031,
      "logps/rejected": -387.173828125,
      "loss": 0.2442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.5949506759643555,
      "rewards/margins": 3.011176586151123,
      "rewards/rejected": -8.60612678527832,
      "step": 7645
    },
    {
      "epoch": 1.6569912271201126,
      "grad_norm": 23.25,
      "learning_rate": 8.571668291791595e-06,
      "logits/chosen": -3.2520954608917236,
      "logits/rejected": -3.2351298332214355,
      "logps/chosen": -326.61956787109375,
      "logps/rejected": -375.4128723144531,
      "loss": 0.2807,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.4561614990234375,
      "rewards/margins": 3.045022487640381,
      "rewards/rejected": -8.501184463500977,
      "step": 7650
    },
    {
      "epoch": 1.6580742987111448,
      "grad_norm": 26.375,
      "learning_rate": 8.565202405517409e-06,
      "logits/chosen": -3.140822172164917,
      "logits/rejected": -3.218686580657959,
      "logps/chosen": -356.69281005859375,
      "logps/rejected": -412.34918212890625,
      "loss": 0.2616,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.253183364868164,
      "rewards/margins": 3.045537233352661,
      "rewards/rejected": -9.298720359802246,
      "step": 7655
    },
    {
      "epoch": 1.659157370302177,
      "grad_norm": 24.875,
      "learning_rate": 8.558737228289131e-06,
      "logits/chosen": -3.060307025909424,
      "logits/rejected": -3.0443520545959473,
      "logps/chosen": -372.447998046875,
      "logps/rejected": -427.6022033691406,
      "loss": 0.2602,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.370694160461426,
      "rewards/margins": 2.8102450370788574,
      "rewards/rejected": -9.180939674377441,
      "step": 7660
    },
    {
      "epoch": 1.660240441893209,
      "grad_norm": 27.25,
      "learning_rate": 8.5522727693248e-06,
      "logits/chosen": -3.107677459716797,
      "logits/rejected": -3.152036190032959,
      "logps/chosen": -355.2466125488281,
      "logps/rejected": -408.8744812011719,
      "loss": 0.2139,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.17763614654541,
      "rewards/margins": 3.1587235927581787,
      "rewards/rejected": -9.336359977722168,
      "step": 7665
    },
    {
      "epoch": 1.6613235134842412,
      "grad_norm": 15.875,
      "learning_rate": 8.545809037841429e-06,
      "logits/chosen": -3.0850296020507812,
      "logits/rejected": -3.2914328575134277,
      "logps/chosen": -335.2914733886719,
      "logps/rejected": -392.6787109375,
      "loss": 0.248,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.888883113861084,
      "rewards/margins": 2.9799680709838867,
      "rewards/rejected": -8.868850708007812,
      "step": 7670
    },
    {
      "epoch": 1.6624065850752734,
      "grad_norm": 19.125,
      "learning_rate": 8.539346043054992e-06,
      "logits/chosen": -3.1139791011810303,
      "logits/rejected": -3.181580066680908,
      "logps/chosen": -313.35272216796875,
      "logps/rejected": -365.4864807128906,
      "loss": 0.2755,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.110234260559082,
      "rewards/margins": 2.9675512313842773,
      "rewards/rejected": -8.077784538269043,
      "step": 7675
    },
    {
      "epoch": 1.6634896566663055,
      "grad_norm": 25.0,
      "learning_rate": 8.532883794180415e-06,
      "logits/chosen": -3.1973817348480225,
      "logits/rejected": -3.216925859451294,
      "logps/chosen": -343.67742919921875,
      "logps/rejected": -392.8218994140625,
      "loss": 0.2662,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.921957969665527,
      "rewards/margins": 2.8443443775177,
      "rewards/rejected": -8.766302108764648,
      "step": 7680
    },
    {
      "epoch": 1.6645727282573377,
      "grad_norm": 26.75,
      "learning_rate": 8.526422300431557e-06,
      "logits/chosen": -3.2612075805664062,
      "logits/rejected": -3.2802894115448,
      "logps/chosen": -339.0722961425781,
      "logps/rejected": -398.816650390625,
      "loss": 0.2767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.150081634521484,
      "rewards/margins": 3.0260300636291504,
      "rewards/rejected": -9.176111221313477,
      "step": 7685
    },
    {
      "epoch": 1.6656557998483699,
      "grad_norm": 16.375,
      "learning_rate": 8.519961571021207e-06,
      "logits/chosen": -3.3034377098083496,
      "logits/rejected": -3.370349168777466,
      "logps/chosen": -334.46405029296875,
      "logps/rejected": -376.0068664550781,
      "loss": 0.2437,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.912297248840332,
      "rewards/margins": 2.820070266723633,
      "rewards/rejected": -8.732367515563965,
      "step": 7690
    },
    {
      "epoch": 1.666738871439402,
      "grad_norm": 30.25,
      "learning_rate": 8.513501615161054e-06,
      "logits/chosen": -3.2470755577087402,
      "logits/rejected": -3.1186718940734863,
      "logps/chosen": -336.1717224121094,
      "logps/rejected": -389.7174072265625,
      "loss": 0.2412,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.626702308654785,
      "rewards/margins": 2.9276394844055176,
      "rewards/rejected": -8.554343223571777,
      "step": 7695
    },
    {
      "epoch": 1.6678219430304342,
      "grad_norm": 14.625,
      "learning_rate": 8.507042442061698e-06,
      "logits/chosen": -3.429154872894287,
      "logits/rejected": -3.2374274730682373,
      "logps/chosen": -312.3702697753906,
      "logps/rejected": -387.8059387207031,
      "loss": 0.2594,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.65181827545166,
      "rewards/margins": 2.9652202129364014,
      "rewards/rejected": -8.617039680480957,
      "step": 7700
    },
    {
      "epoch": 1.6689050146214663,
      "grad_norm": 17.5,
      "learning_rate": 8.500584060932608e-06,
      "logits/chosen": -3.137375831604004,
      "logits/rejected": -3.0648694038391113,
      "logps/chosen": -345.3934326171875,
      "logps/rejected": -411.08984375,
      "loss": 0.2324,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.6641340255737305,
      "rewards/margins": 3.223134994506836,
      "rewards/rejected": -8.887269973754883,
      "step": 7705
    },
    {
      "epoch": 1.6699880862124985,
      "grad_norm": 7.40625,
      "learning_rate": 8.494126480982136e-06,
      "logits/chosen": -3.095233201980591,
      "logits/rejected": -3.0613350868225098,
      "logps/chosen": -352.94989013671875,
      "logps/rejected": -410.210693359375,
      "loss": 0.2418,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.149703025817871,
      "rewards/margins": 3.0849545001983643,
      "rewards/rejected": -9.234657287597656,
      "step": 7710
    },
    {
      "epoch": 1.6710711578035307,
      "grad_norm": 19.75,
      "learning_rate": 8.48766971141748e-06,
      "logits/chosen": -3.1064937114715576,
      "logits/rejected": -3.0697576999664307,
      "logps/chosen": -336.81134033203125,
      "logps/rejected": -394.5721130371094,
      "loss": 0.2434,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.501482009887695,
      "rewards/margins": 2.668320894241333,
      "rewards/rejected": -8.169803619384766,
      "step": 7715
    },
    {
      "epoch": 1.6721542293945628,
      "grad_norm": 19.0,
      "learning_rate": 8.481213761444696e-06,
      "logits/chosen": -3.327718734741211,
      "logits/rejected": -3.2219367027282715,
      "logps/chosen": -331.93267822265625,
      "logps/rejected": -399.7538757324219,
      "loss": 0.1974,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -5.855520725250244,
      "rewards/margins": 3.3586018085479736,
      "rewards/rejected": -9.214120864868164,
      "step": 7720
    },
    {
      "epoch": 1.6732373009855952,
      "grad_norm": 18.625,
      "learning_rate": 8.474758640268662e-06,
      "logits/chosen": -3.2528793811798096,
      "logits/rejected": -3.3269882202148438,
      "logps/chosen": -337.77777099609375,
      "logps/rejected": -389.4310607910156,
      "loss": 0.2655,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.638972759246826,
      "rewards/margins": 3.0975799560546875,
      "rewards/rejected": -8.736554145812988,
      "step": 7725
    },
    {
      "epoch": 1.6743203725766274,
      "grad_norm": 18.875,
      "learning_rate": 8.468304357093074e-06,
      "logits/chosen": -3.1533284187316895,
      "logits/rejected": -3.1378207206726074,
      "logps/chosen": -333.61834716796875,
      "logps/rejected": -393.0112609863281,
      "loss": 0.2134,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.65184211730957,
      "rewards/margins": 3.1243534088134766,
      "rewards/rejected": -8.776195526123047,
      "step": 7730
    },
    {
      "epoch": 1.6754034441676595,
      "grad_norm": 25.875,
      "learning_rate": 8.461850921120438e-06,
      "logits/chosen": -3.288428544998169,
      "logits/rejected": -3.107126235961914,
      "logps/chosen": -333.66595458984375,
      "logps/rejected": -410.4688415527344,
      "loss": 0.2913,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.152081489562988,
      "rewards/margins": 2.8538589477539062,
      "rewards/rejected": -9.005940437316895,
      "step": 7735
    },
    {
      "epoch": 1.6764865157586917,
      "grad_norm": 27.75,
      "learning_rate": 8.455398341552053e-06,
      "logits/chosen": -3.266279697418213,
      "logits/rejected": -3.211076259613037,
      "logps/chosen": -336.7007751464844,
      "logps/rejected": -400.2386474609375,
      "loss": 0.2744,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.050455570220947,
      "rewards/margins": 2.769324541091919,
      "rewards/rejected": -8.819781303405762,
      "step": 7740
    },
    {
      "epoch": 1.6775695873497238,
      "grad_norm": 24.75,
      "learning_rate": 8.448946627587992e-06,
      "logits/chosen": -3.3501415252685547,
      "logits/rejected": -3.3204410076141357,
      "logps/chosen": -342.8399658203125,
      "logps/rejected": -393.25494384765625,
      "loss": 0.2947,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.530185699462891,
      "rewards/margins": 2.6322970390319824,
      "rewards/rejected": -9.162483215332031,
      "step": 7745
    },
    {
      "epoch": 1.678652658940756,
      "grad_norm": 22.625,
      "learning_rate": 8.442495788427092e-06,
      "logits/chosen": -3.4528000354766846,
      "logits/rejected": -3.3902409076690674,
      "logps/chosen": -333.9834289550781,
      "logps/rejected": -397.95758056640625,
      "loss": 0.2129,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -6.454791069030762,
      "rewards/margins": 3.333214521408081,
      "rewards/rejected": -9.788005828857422,
      "step": 7750
    },
    {
      "epoch": 1.6797357305317882,
      "grad_norm": 19.875,
      "learning_rate": 8.43604583326695e-06,
      "logits/chosen": -3.2928779125213623,
      "logits/rejected": -3.2911746501922607,
      "logps/chosen": -334.6468811035156,
      "logps/rejected": -388.22161865234375,
      "loss": 0.2124,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.906661033630371,
      "rewards/margins": 3.4863548278808594,
      "rewards/rejected": -9.39301586151123,
      "step": 7755
    },
    {
      "epoch": 1.6808188021228203,
      "grad_norm": 31.625,
      "learning_rate": 8.429596771303901e-06,
      "logits/chosen": -3.304589033126831,
      "logits/rejected": -3.2367138862609863,
      "logps/chosen": -331.3647766113281,
      "logps/rejected": -399.50933837890625,
      "loss": 0.2257,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.873335838317871,
      "rewards/margins": 3.4382667541503906,
      "rewards/rejected": -9.311602592468262,
      "step": 7760
    },
    {
      "epoch": 1.6819018737138525,
      "grad_norm": 17.0,
      "learning_rate": 8.423148611732999e-06,
      "logits/chosen": -3.3322250843048096,
      "logits/rejected": -3.215902328491211,
      "logps/chosen": -341.8564453125,
      "logps/rejected": -398.5255126953125,
      "loss": 0.2145,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.757988452911377,
      "rewards/margins": 3.282278060913086,
      "rewards/rejected": -9.040266990661621,
      "step": 7765
    },
    {
      "epoch": 1.6829849453048846,
      "grad_norm": 20.875,
      "learning_rate": 8.416701363748023e-06,
      "logits/chosen": -3.399932384490967,
      "logits/rejected": -3.388361692428589,
      "logps/chosen": -328.4076232910156,
      "logps/rejected": -385.35577392578125,
      "loss": 0.2892,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -5.8092546463012695,
      "rewards/margins": 2.9996771812438965,
      "rewards/rejected": -8.808932304382324,
      "step": 7770
    },
    {
      "epoch": 1.6840680168959168,
      "grad_norm": 14.6875,
      "learning_rate": 8.410255036541443e-06,
      "logits/chosen": -3.3251335620880127,
      "logits/rejected": -3.3367602825164795,
      "logps/chosen": -318.5076599121094,
      "logps/rejected": -384.09381103515625,
      "loss": 0.1634,
      "rewards/accuracies": 0.9416666030883789,
      "rewards/chosen": -5.456808567047119,
      "rewards/margins": 3.523090362548828,
      "rewards/rejected": -8.979898452758789,
      "step": 7775
    },
    {
      "epoch": 1.685151088486949,
      "grad_norm": 20.5,
      "learning_rate": 8.40380963930442e-06,
      "logits/chosen": -3.1994452476501465,
      "logits/rejected": -3.306533098220825,
      "logps/chosen": -355.33233642578125,
      "logps/rejected": -397.09259033203125,
      "loss": 0.2884,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.310423851013184,
      "rewards/margins": 3.0100324153900146,
      "rewards/rejected": -9.320457458496094,
      "step": 7780
    },
    {
      "epoch": 1.686234160077981,
      "grad_norm": 27.25,
      "learning_rate": 8.397365181226786e-06,
      "logits/chosen": -3.2544779777526855,
      "logits/rejected": -3.2112247943878174,
      "logps/chosen": -330.6983642578125,
      "logps/rejected": -382.9519348144531,
      "loss": 0.2596,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.0521063804626465,
      "rewards/margins": 2.7241733074188232,
      "rewards/rejected": -8.77627944946289,
      "step": 7785
    },
    {
      "epoch": 1.6873172316690135,
      "grad_norm": 30.5,
      "learning_rate": 8.390921671497041e-06,
      "logits/chosen": -3.2086081504821777,
      "logits/rejected": -3.133607864379883,
      "logps/chosen": -356.3215026855469,
      "logps/rejected": -410.87347412109375,
      "loss": 0.2913,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.342009544372559,
      "rewards/margins": 2.5996084213256836,
      "rewards/rejected": -8.941617965698242,
      "step": 7790
    },
    {
      "epoch": 1.6884003032600456,
      "grad_norm": 18.625,
      "learning_rate": 8.384479119302324e-06,
      "logits/chosen": -3.2256782054901123,
      "logits/rejected": -3.137066602706909,
      "logps/chosen": -354.3873596191406,
      "logps/rejected": -430.94696044921875,
      "loss": 0.1949,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.245771884918213,
      "rewards/margins": 3.2919182777404785,
      "rewards/rejected": -9.537691116333008,
      "step": 7795
    },
    {
      "epoch": 1.6894833748510778,
      "grad_norm": 21.625,
      "learning_rate": 8.378037533828415e-06,
      "logits/chosen": -3.306926727294922,
      "logits/rejected": -3.2953312397003174,
      "logps/chosen": -324.46551513671875,
      "logps/rejected": -390.4478454589844,
      "loss": 0.235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.075008392333984,
      "rewards/margins": 3.064425230026245,
      "rewards/rejected": -9.139433860778809,
      "step": 7800
    },
    {
      "epoch": 1.69056644644211,
      "grad_norm": 20.625,
      "learning_rate": 8.371596924259713e-06,
      "logits/chosen": -3.1896064281463623,
      "logits/rejected": -3.090049982070923,
      "logps/chosen": -339.1053161621094,
      "logps/rejected": -414.83868408203125,
      "loss": 0.2125,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.811116695404053,
      "rewards/margins": 3.4289443492889404,
      "rewards/rejected": -9.240060806274414,
      "step": 7805
    },
    {
      "epoch": 1.6916495180331421,
      "grad_norm": 28.625,
      "learning_rate": 8.365157299779229e-06,
      "logits/chosen": -3.1155004501342773,
      "logits/rejected": -3.134432315826416,
      "logps/chosen": -349.80322265625,
      "logps/rejected": -408.47869873046875,
      "loss": 0.2758,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.1170196533203125,
      "rewards/margins": 3.3323731422424316,
      "rewards/rejected": -9.449392318725586,
      "step": 7810
    },
    {
      "epoch": 1.6927325896241743,
      "grad_norm": 20.25,
      "learning_rate": 8.35871866956856e-06,
      "logits/chosen": -3.173478603363037,
      "logits/rejected": -3.2023258209228516,
      "logps/chosen": -365.82696533203125,
      "logps/rejected": -416.95806884765625,
      "loss": 0.2327,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.435656547546387,
      "rewards/margins": 2.9254324436187744,
      "rewards/rejected": -9.361089706420898,
      "step": 7815
    },
    {
      "epoch": 1.6938156612152064,
      "grad_norm": 27.25,
      "learning_rate": 8.352281042807901e-06,
      "logits/chosen": -3.3422226905822754,
      "logits/rejected": -3.3470351696014404,
      "logps/chosen": -333.7770080566406,
      "logps/rejected": -395.27276611328125,
      "loss": 0.2232,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.15850305557251,
      "rewards/margins": 3.322652816772461,
      "rewards/rejected": -9.481155395507812,
      "step": 7820
    },
    {
      "epoch": 1.6948987328062386,
      "grad_norm": 11.1875,
      "learning_rate": 8.345844428676e-06,
      "logits/chosen": -3.4854958057403564,
      "logits/rejected": -3.438394069671631,
      "logps/chosen": -333.5716247558594,
      "logps/rejected": -397.05084228515625,
      "loss": 0.2231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.282759189605713,
      "rewards/margins": 2.9280364513397217,
      "rewards/rejected": -9.210795402526855,
      "step": 7825
    },
    {
      "epoch": 1.6959818043972708,
      "grad_norm": 21.75,
      "learning_rate": 8.33940883635017e-06,
      "logits/chosen": -3.2248363494873047,
      "logits/rejected": -3.2316341400146484,
      "logps/chosen": -337.78045654296875,
      "logps/rejected": -400.1634521484375,
      "loss": 0.2295,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.9021897315979,
      "rewards/margins": 3.0320229530334473,
      "rewards/rejected": -8.934211730957031,
      "step": 7830
    },
    {
      "epoch": 1.697064875988303,
      "grad_norm": 17.5,
      "learning_rate": 8.332974275006267e-06,
      "logits/chosen": -3.273409366607666,
      "logits/rejected": -3.2789950370788574,
      "logps/chosen": -344.17413330078125,
      "logps/rejected": -414.2052307128906,
      "loss": 0.2279,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.520042419433594,
      "rewards/margins": 3.1982059478759766,
      "rewards/rejected": -9.718247413635254,
      "step": 7835
    },
    {
      "epoch": 1.698147947579335,
      "grad_norm": 19.375,
      "learning_rate": 8.326540753818676e-06,
      "logits/chosen": -3.2786102294921875,
      "logits/rejected": -3.143738269805908,
      "logps/chosen": -344.7550964355469,
      "logps/rejected": -395.533447265625,
      "loss": 0.2912,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.0244245529174805,
      "rewards/margins": 2.6838581562042236,
      "rewards/rejected": -8.708282470703125,
      "step": 7840
    },
    {
      "epoch": 1.6992310191703672,
      "grad_norm": 24.75,
      "learning_rate": 8.320108281960296e-06,
      "logits/chosen": -3.2644340991973877,
      "logits/rejected": -3.198594093322754,
      "logps/chosen": -346.10302734375,
      "logps/rejected": -403.03558349609375,
      "loss": 0.3159,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.756441593170166,
      "rewards/margins": 3.1341872215270996,
      "rewards/rejected": -8.890629768371582,
      "step": 7845
    },
    {
      "epoch": 1.7003140907613994,
      "grad_norm": 23.875,
      "learning_rate": 8.313676868602535e-06,
      "logits/chosen": -3.3951003551483154,
      "logits/rejected": -3.379676342010498,
      "logps/chosen": -336.5987854003906,
      "logps/rejected": -389.28546142578125,
      "loss": 0.2885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.04199743270874,
      "rewards/margins": 2.783555507659912,
      "rewards/rejected": -8.825551986694336,
      "step": 7850
    },
    {
      "epoch": 1.7013971623524315,
      "grad_norm": 21.75,
      "learning_rate": 8.307246522915288e-06,
      "logits/chosen": -3.3080837726593018,
      "logits/rejected": -3.3708584308624268,
      "logps/chosen": -354.72564697265625,
      "logps/rejected": -399.70098876953125,
      "loss": 0.3079,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.423037528991699,
      "rewards/margins": 3.001690149307251,
      "rewards/rejected": -9.424728393554688,
      "step": 7855
    },
    {
      "epoch": 1.7024802339434637,
      "grad_norm": 21.375,
      "learning_rate": 8.300817254066931e-06,
      "logits/chosen": -3.2457664012908936,
      "logits/rejected": -3.2131104469299316,
      "logps/chosen": -362.5649719238281,
      "logps/rejected": -423.04150390625,
      "loss": 0.1715,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.442339897155762,
      "rewards/margins": 3.4505462646484375,
      "rewards/rejected": -9.892885208129883,
      "step": 7860
    },
    {
      "epoch": 1.7035633055344959,
      "grad_norm": 24.5,
      "learning_rate": 8.2943890712243e-06,
      "logits/chosen": -3.164849042892456,
      "logits/rejected": -3.176689863204956,
      "logps/chosen": -351.3653869628906,
      "logps/rejected": -403.16107177734375,
      "loss": 0.3011,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.507473945617676,
      "rewards/margins": 2.76486873626709,
      "rewards/rejected": -9.272342681884766,
      "step": 7865
    },
    {
      "epoch": 1.704646377125528,
      "grad_norm": 15.125,
      "learning_rate": 8.287961983552686e-06,
      "logits/chosen": -3.2671241760253906,
      "logits/rejected": -3.280458450317383,
      "logps/chosen": -346.2356872558594,
      "logps/rejected": -386.6893005371094,
      "loss": 0.3327,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -6.151129722595215,
      "rewards/margins": 2.754103660583496,
      "rewards/rejected": -8.905233383178711,
      "step": 7870
    },
    {
      "epoch": 1.7057294487165602,
      "grad_norm": 15.9375,
      "learning_rate": 8.281536000215814e-06,
      "logits/chosen": -3.3180317878723145,
      "logits/rejected": -3.2301642894744873,
      "logps/chosen": -340.6478576660156,
      "logps/rejected": -411.7662048339844,
      "loss": 0.2053,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.29735803604126,
      "rewards/margins": 3.6358237266540527,
      "rewards/rejected": -9.933181762695312,
      "step": 7875
    },
    {
      "epoch": 1.7068125203075923,
      "grad_norm": 30.25,
      "learning_rate": 8.275111130375846e-06,
      "logits/chosen": -3.222682476043701,
      "logits/rejected": -3.2772216796875,
      "logps/chosen": -345.0823059082031,
      "logps/rejected": -401.0930480957031,
      "loss": 0.2443,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.293870449066162,
      "rewards/margins": 3.2488834857940674,
      "rewards/rejected": -9.542754173278809,
      "step": 7880
    },
    {
      "epoch": 1.7078955918986245,
      "grad_norm": 21.5,
      "learning_rate": 8.268687383193344e-06,
      "logits/chosen": -3.256251096725464,
      "logits/rejected": -3.1591737270355225,
      "logps/chosen": -352.57220458984375,
      "logps/rejected": -418.079833984375,
      "loss": 0.2104,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.379638671875,
      "rewards/margins": 3.188875675201416,
      "rewards/rejected": -9.568513870239258,
      "step": 7885
    },
    {
      "epoch": 1.7089786634896567,
      "grad_norm": 28.75,
      "learning_rate": 8.262264767827274e-06,
      "logits/chosen": -3.232551097869873,
      "logits/rejected": -3.2006516456604004,
      "logps/chosen": -343.0650329589844,
      "logps/rejected": -416.9515686035156,
      "loss": 0.2215,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.146053314208984,
      "rewards/margins": 3.135481834411621,
      "rewards/rejected": -9.281534194946289,
      "step": 7890
    },
    {
      "epoch": 1.7100617350806888,
      "grad_norm": 24.0,
      "learning_rate": 8.255843293434989e-06,
      "logits/chosen": -3.3675010204315186,
      "logits/rejected": -3.3516221046447754,
      "logps/chosen": -332.3602294921875,
      "logps/rejected": -385.25909423828125,
      "loss": 0.2407,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.005386829376221,
      "rewards/margins": 2.964277744293213,
      "rewards/rejected": -8.969663619995117,
      "step": 7895
    },
    {
      "epoch": 1.711144806671721,
      "grad_norm": 23.25,
      "learning_rate": 8.249422969172211e-06,
      "logits/chosen": -3.3904380798339844,
      "logits/rejected": -3.2866573333740234,
      "logps/chosen": -357.2455749511719,
      "logps/rejected": -413.8421936035156,
      "loss": 0.2685,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.760790824890137,
      "rewards/margins": 3.0757312774658203,
      "rewards/rejected": -9.836523056030273,
      "step": 7900
    },
    {
      "epoch": 1.7122278782627531,
      "grad_norm": 25.125,
      "learning_rate": 8.24300380419303e-06,
      "logits/chosen": -3.2332167625427246,
      "logits/rejected": -3.2559425830841064,
      "logps/chosen": -343.2796936035156,
      "logps/rejected": -396.7730712890625,
      "loss": 0.321,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.290593147277832,
      "rewards/margins": 2.836195468902588,
      "rewards/rejected": -9.126788139343262,
      "step": 7905
    },
    {
      "epoch": 1.7133109498537853,
      "grad_norm": 21.625,
      "learning_rate": 8.236585807649875e-06,
      "logits/chosen": -3.1095893383026123,
      "logits/rejected": -3.0634357929229736,
      "logps/chosen": -364.1743469238281,
      "logps/rejected": -412.41717529296875,
      "loss": 0.2532,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.508750915527344,
      "rewards/margins": 2.7306339740753174,
      "rewards/rejected": -9.239385604858398,
      "step": 7910
    },
    {
      "epoch": 1.7143940214448175,
      "grad_norm": 25.375,
      "learning_rate": 8.230168988693513e-06,
      "logits/chosen": -3.245288372039795,
      "logits/rejected": -3.2338314056396484,
      "logps/chosen": -347.81707763671875,
      "logps/rejected": -400.6295166015625,
      "loss": 0.2277,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.9669599533081055,
      "rewards/margins": 3.1841373443603516,
      "rewards/rejected": -9.151097297668457,
      "step": 7915
    },
    {
      "epoch": 1.7154770930358496,
      "grad_norm": 24.375,
      "learning_rate": 8.223753356473034e-06,
      "logits/chosen": -3.237393856048584,
      "logits/rejected": -3.2437355518341064,
      "logps/chosen": -349.50006103515625,
      "logps/rejected": -411.874755859375,
      "loss": 0.2286,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -6.159745216369629,
      "rewards/margins": 3.252872943878174,
      "rewards/rejected": -9.412617683410645,
      "step": 7920
    },
    {
      "epoch": 1.7165601646268818,
      "grad_norm": 22.5,
      "learning_rate": 8.217338920135832e-06,
      "logits/chosen": -3.171527147293091,
      "logits/rejected": -3.1821205615997314,
      "logps/chosen": -346.3213806152344,
      "logps/rejected": -407.0174865722656,
      "loss": 0.282,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.250563144683838,
      "rewards/margins": 2.9239699840545654,
      "rewards/rejected": -9.174532890319824,
      "step": 7925
    },
    {
      "epoch": 1.717643236217914,
      "grad_norm": 18.75,
      "learning_rate": 8.210925688827597e-06,
      "logits/chosen": -3.2480056285858154,
      "logits/rejected": -3.1335957050323486,
      "logps/chosen": -346.9445495605469,
      "logps/rejected": -411.5123596191406,
      "loss": 0.2346,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.05561637878418,
      "rewards/margins": 3.1792635917663574,
      "rewards/rejected": -9.234880447387695,
      "step": 7930
    },
    {
      "epoch": 1.718726307808946,
      "grad_norm": 34.25,
      "learning_rate": 8.204513671692304e-06,
      "logits/chosen": -3.3731276988983154,
      "logits/rejected": -3.3499393463134766,
      "logps/chosen": -351.79718017578125,
      "logps/rejected": -416.75408935546875,
      "loss": 0.2382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.908949375152588,
      "rewards/margins": 3.1571202278137207,
      "rewards/rejected": -10.066070556640625,
      "step": 7935
    },
    {
      "epoch": 1.7198093793999782,
      "grad_norm": 14.5,
      "learning_rate": 8.198102877872191e-06,
      "logits/chosen": -3.280707836151123,
      "logits/rejected": -3.299086093902588,
      "logps/chosen": -333.072509765625,
      "logps/rejected": -406.785400390625,
      "loss": 0.2372,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.742514610290527,
      "rewards/margins": 3.027763605117798,
      "rewards/rejected": -8.770278930664062,
      "step": 7940
    },
    {
      "epoch": 1.7208924509910104,
      "grad_norm": 24.625,
      "learning_rate": 8.191693316507754e-06,
      "logits/chosen": -3.3127808570861816,
      "logits/rejected": -3.2611336708068848,
      "logps/chosen": -323.07427978515625,
      "logps/rejected": -377.6083984375,
      "loss": 0.2871,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.708763122558594,
      "rewards/margins": 2.7098779678344727,
      "rewards/rejected": -8.418639183044434,
      "step": 7945
    },
    {
      "epoch": 1.7219755225820426,
      "grad_norm": 25.75,
      "learning_rate": 8.185284996737734e-06,
      "logits/chosen": -3.26347279548645,
      "logits/rejected": -3.271294355392456,
      "logps/chosen": -362.29400634765625,
      "logps/rejected": -407.30792236328125,
      "loss": 0.2503,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.421308994293213,
      "rewards/margins": 3.0790135860443115,
      "rewards/rejected": -9.500322341918945,
      "step": 7950
    },
    {
      "epoch": 1.7230585941730747,
      "grad_norm": 13.0,
      "learning_rate": 8.178877927699103e-06,
      "logits/chosen": -3.3526854515075684,
      "logits/rejected": -3.2582554817199707,
      "logps/chosen": -341.3164978027344,
      "logps/rejected": -409.8785095214844,
      "loss": 0.1638,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.081578731536865,
      "rewards/margins": 3.567495346069336,
      "rewards/rejected": -9.649073600769043,
      "step": 7955
    },
    {
      "epoch": 1.7241416657641069,
      "grad_norm": 20.625,
      "learning_rate": 8.172472118527045e-06,
      "logits/chosen": -3.2750370502471924,
      "logits/rejected": -3.337656021118164,
      "logps/chosen": -340.23284912109375,
      "logps/rejected": -380.8800964355469,
      "loss": 0.3613,
      "rewards/accuracies": 0.8166666030883789,
      "rewards/chosen": -5.700125694274902,
      "rewards/margins": 2.7850823402404785,
      "rewards/rejected": -8.485206604003906,
      "step": 7960
    },
    {
      "epoch": 1.725224737355139,
      "grad_norm": 27.5,
      "learning_rate": 8.166067578354948e-06,
      "logits/chosen": -3.1857457160949707,
      "logits/rejected": -3.234631299972534,
      "logps/chosen": -349.88543701171875,
      "logps/rejected": -402.17510986328125,
      "loss": 0.2503,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.039498805999756,
      "rewards/margins": 2.884539842605591,
      "rewards/rejected": -8.924039840698242,
      "step": 7965
    },
    {
      "epoch": 1.7263078089461712,
      "grad_norm": 20.25,
      "learning_rate": 8.159664316314397e-06,
      "logits/chosen": -3.25628399848938,
      "logits/rejected": -3.270890474319458,
      "logps/chosen": -354.18804931640625,
      "logps/rejected": -409.0033874511719,
      "loss": 0.2594,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.903025150299072,
      "rewards/margins": 3.069683313369751,
      "rewards/rejected": -8.972708702087402,
      "step": 7970
    },
    {
      "epoch": 1.7273908805372034,
      "grad_norm": 25.125,
      "learning_rate": 8.153262341535148e-06,
      "logits/chosen": -3.223767042160034,
      "logits/rejected": -3.1979053020477295,
      "logps/chosen": -342.32696533203125,
      "logps/rejected": -396.96160888671875,
      "loss": 0.2194,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.9579267501831055,
      "rewards/margins": 2.933785915374756,
      "rewards/rejected": -8.891712188720703,
      "step": 7975
    },
    {
      "epoch": 1.7284739521282355,
      "grad_norm": 22.625,
      "learning_rate": 8.14686166314512e-06,
      "logits/chosen": -3.1524627208709717,
      "logits/rejected": -3.0446057319641113,
      "logps/chosen": -349.35205078125,
      "logps/rejected": -421.101318359375,
      "loss": 0.2384,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.313918590545654,
      "rewards/margins": 3.2432122230529785,
      "rewards/rejected": -9.557130813598633,
      "step": 7980
    },
    {
      "epoch": 1.729557023719268,
      "grad_norm": 25.25,
      "learning_rate": 8.14046229027039e-06,
      "logits/chosen": -3.1657779216766357,
      "logits/rejected": -3.2588562965393066,
      "logps/chosen": -354.2939147949219,
      "logps/rejected": -406.9386901855469,
      "loss": 0.2526,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.923929691314697,
      "rewards/margins": 3.2345967292785645,
      "rewards/rejected": -9.158526420593262,
      "step": 7985
    },
    {
      "epoch": 1.7306400953103,
      "grad_norm": 25.375,
      "learning_rate": 8.134064232035176e-06,
      "logits/chosen": -3.3356997966766357,
      "logits/rejected": -3.3251616954803467,
      "logps/chosen": -353.91217041015625,
      "logps/rejected": -398.69390869140625,
      "loss": 0.2675,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.576992988586426,
      "rewards/margins": 2.468947172164917,
      "rewards/rejected": -9.045939445495605,
      "step": 7990
    },
    {
      "epoch": 1.7317231669013322,
      "grad_norm": 12.0625,
      "learning_rate": 8.12766749756181e-06,
      "logits/chosen": -3.200958728790283,
      "logits/rejected": -3.135249614715576,
      "logps/chosen": -351.0456848144531,
      "logps/rejected": -407.6814880371094,
      "loss": 0.2001,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.990227699279785,
      "rewards/margins": 3.294189929962158,
      "rewards/rejected": -9.284418106079102,
      "step": 7995
    },
    {
      "epoch": 1.7328062384923644,
      "grad_norm": 23.375,
      "learning_rate": 8.121272095970747e-06,
      "logits/chosen": -3.339323043823242,
      "logits/rejected": -3.2099642753601074,
      "logps/chosen": -346.4736022949219,
      "logps/rejected": -407.7232360839844,
      "loss": 0.2764,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -6.543285369873047,
      "rewards/margins": 2.671057939529419,
      "rewards/rejected": -9.21434211730957,
      "step": 8000
    },
    {
      "epoch": 1.7328062384923644,
      "eval_logits/chosen": -3.3372952938079834,
      "eval_logits/rejected": -3.272723436355591,
      "eval_logps/chosen": -333.5038757324219,
      "eval_logps/rejected": -383.208984375,
      "eval_loss": 0.3864555060863495,
      "eval_rewards/accuracies": 0.8149999976158142,
      "eval_rewards/chosen": -5.965130805969238,
      "eval_rewards/margins": 2.3124611377716064,
      "eval_rewards/rejected": -8.277591705322266,
      "eval_runtime": 14.4342,
      "eval_samples_per_second": 13.856,
      "eval_steps_per_second": 13.856,
      "step": 8000
    },
    {
      "epoch": 1.7338893100833965,
      "grad_norm": 24.875,
      "learning_rate": 8.114878036380535e-06,
      "logits/chosen": -3.291736125946045,
      "logits/rejected": -3.3205349445343018,
      "logps/chosen": -351.87481689453125,
      "logps/rejected": -411.96820068359375,
      "loss": 0.211,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.740364074707031,
      "rewards/margins": 3.221195936203003,
      "rewards/rejected": -9.96156120300293,
      "step": 8005
    },
    {
      "epoch": 1.7349723816744287,
      "grad_norm": 19.625,
      "learning_rate": 8.10848532790781e-06,
      "logits/chosen": -3.2520174980163574,
      "logits/rejected": -3.218716859817505,
      "logps/chosen": -340.00909423828125,
      "logps/rejected": -400.0207824707031,
      "loss": 0.1914,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.244508266448975,
      "rewards/margins": 3.1167571544647217,
      "rewards/rejected": -9.361265182495117,
      "step": 8010
    },
    {
      "epoch": 1.7360554532654608,
      "grad_norm": 16.125,
      "learning_rate": 8.10209397966729e-06,
      "logits/chosen": -3.062471866607666,
      "logits/rejected": -3.094879150390625,
      "logps/chosen": -348.77301025390625,
      "logps/rejected": -407.36737060546875,
      "loss": 0.2179,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.8965582847595215,
      "rewards/margins": 3.3296027183532715,
      "rewards/rejected": -9.226161003112793,
      "step": 8015
    },
    {
      "epoch": 1.737138524856493,
      "grad_norm": 26.625,
      "learning_rate": 8.095704000771739e-06,
      "logits/chosen": -3.319715976715088,
      "logits/rejected": -3.3518147468566895,
      "logps/chosen": -334.9211120605469,
      "logps/rejected": -391.70111083984375,
      "loss": 0.229,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.941099643707275,
      "rewards/margins": 2.891627311706543,
      "rewards/rejected": -8.83272647857666,
      "step": 8020
    },
    {
      "epoch": 1.7382215964475252,
      "grad_norm": 19.125,
      "learning_rate": 8.089315400331976e-06,
      "logits/chosen": -3.289872407913208,
      "logits/rejected": -3.2248165607452393,
      "logps/chosen": -341.98565673828125,
      "logps/rejected": -404.3893127441406,
      "loss": 0.2275,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.971475124359131,
      "rewards/margins": 3.264786958694458,
      "rewards/rejected": -9.236261367797852,
      "step": 8025
    },
    {
      "epoch": 1.7393046680385573,
      "grad_norm": 21.5,
      "learning_rate": 8.082928187456862e-06,
      "logits/chosen": -3.297043561935425,
      "logits/rejected": -3.3236217498779297,
      "logps/chosen": -368.41326904296875,
      "logps/rejected": -445.36090087890625,
      "loss": 0.2291,
      "rewards/accuracies": 0.9416666030883789,
      "rewards/chosen": -6.751275539398193,
      "rewards/margins": 3.433659315109253,
      "rewards/rejected": -10.184934616088867,
      "step": 8030
    },
    {
      "epoch": 1.7403877396295895,
      "grad_norm": 24.25,
      "learning_rate": 8.076542371253265e-06,
      "logits/chosen": -3.275219678878784,
      "logits/rejected": -3.3267502784729004,
      "logps/chosen": -351.78472900390625,
      "logps/rejected": -405.56817626953125,
      "loss": 0.282,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.447077751159668,
      "rewards/margins": 2.9073617458343506,
      "rewards/rejected": -9.354439735412598,
      "step": 8035
    },
    {
      "epoch": 1.7414708112206216,
      "grad_norm": 26.25,
      "learning_rate": 8.070157960826072e-06,
      "logits/chosen": -3.242205858230591,
      "logits/rejected": -3.204155683517456,
      "logps/chosen": -354.28521728515625,
      "logps/rejected": -414.4486389160156,
      "loss": 0.222,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.148865699768066,
      "rewards/margins": 3.1228325366973877,
      "rewards/rejected": -9.271697998046875,
      "step": 8040
    },
    {
      "epoch": 1.7425538828116538,
      "grad_norm": 22.375,
      "learning_rate": 8.063774965278163e-06,
      "logits/chosen": -3.4005751609802246,
      "logits/rejected": -3.4478583335876465,
      "logps/chosen": -354.8149719238281,
      "logps/rejected": -414.066162109375,
      "loss": 0.244,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.87600040435791,
      "rewards/margins": 3.1273865699768066,
      "rewards/rejected": -10.003386497497559,
      "step": 8045
    },
    {
      "epoch": 1.7436369544026862,
      "grad_norm": 15.125,
      "learning_rate": 8.0573933937104e-06,
      "logits/chosen": -3.4323477745056152,
      "logits/rejected": -3.4255402088165283,
      "logps/chosen": -326.42999267578125,
      "logps/rejected": -385.66107177734375,
      "loss": 0.1903,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -5.937867164611816,
      "rewards/margins": 3.124962329864502,
      "rewards/rejected": -9.062829971313477,
      "step": 8050
    },
    {
      "epoch": 1.7447200259937183,
      "grad_norm": 14.5625,
      "learning_rate": 8.051013255221613e-06,
      "logits/chosen": -3.2992262840270996,
      "logits/rejected": -3.2823429107666016,
      "logps/chosen": -331.6766357421875,
      "logps/rejected": -402.3438720703125,
      "loss": 0.2336,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.286230087280273,
      "rewards/margins": 3.127101182937622,
      "rewards/rejected": -9.413331031799316,
      "step": 8055
    },
    {
      "epoch": 1.7458030975847505,
      "grad_norm": 22.0,
      "learning_rate": 8.044634558908592e-06,
      "logits/chosen": -3.4022796154022217,
      "logits/rejected": -3.2833893299102783,
      "logps/chosen": -343.49176025390625,
      "logps/rejected": -426.40240478515625,
      "loss": 0.2507,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.078262805938721,
      "rewards/margins": 3.32232403755188,
      "rewards/rejected": -9.40058708190918,
      "step": 8060
    },
    {
      "epoch": 1.7468861691757827,
      "grad_norm": 33.25,
      "learning_rate": 8.038257313866068e-06,
      "logits/chosen": -3.3089663982391357,
      "logits/rejected": -3.3649277687072754,
      "logps/chosen": -345.2264709472656,
      "logps/rejected": -397.3958740234375,
      "loss": 0.2749,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.423470497131348,
      "rewards/margins": 2.9702022075653076,
      "rewards/rejected": -9.393671989440918,
      "step": 8065
    },
    {
      "epoch": 1.7479692407668148,
      "grad_norm": 20.25,
      "learning_rate": 8.031881529186708e-06,
      "logits/chosen": -3.2999491691589355,
      "logits/rejected": -3.227686643600464,
      "logps/chosen": -360.0122375488281,
      "logps/rejected": -417.01959228515625,
      "loss": 0.2441,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.546011447906494,
      "rewards/margins": 3.100595474243164,
      "rewards/rejected": -9.646607398986816,
      "step": 8070
    },
    {
      "epoch": 1.749052312357847,
      "grad_norm": 16.375,
      "learning_rate": 8.02550721396109e-06,
      "logits/chosen": -3.305905818939209,
      "logits/rejected": -3.2928130626678467,
      "logps/chosen": -352.9267883300781,
      "logps/rejected": -409.2171325683594,
      "loss": 0.2464,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.473934173583984,
      "rewards/margins": 2.8538920879364014,
      "rewards/rejected": -9.327826499938965,
      "step": 8075
    },
    {
      "epoch": 1.7501353839488791,
      "grad_norm": 28.875,
      "learning_rate": 8.019134377277698e-06,
      "logits/chosen": -3.433579206466675,
      "logits/rejected": -3.4874720573425293,
      "logps/chosen": -335.08050537109375,
      "logps/rejected": -395.8656921386719,
      "loss": 0.2788,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.052579879760742,
      "rewards/margins": 3.3361458778381348,
      "rewards/rejected": -9.388725280761719,
      "step": 8080
    },
    {
      "epoch": 1.7512184555399113,
      "grad_norm": 31.0,
      "learning_rate": 8.012763028222915e-06,
      "logits/chosen": -3.3500804901123047,
      "logits/rejected": -3.403470516204834,
      "logps/chosen": -353.08563232421875,
      "logps/rejected": -415.22259521484375,
      "loss": 0.2931,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.595480918884277,
      "rewards/margins": 2.953307628631592,
      "rewards/rejected": -9.548789978027344,
      "step": 8085
    },
    {
      "epoch": 1.7523015271309434,
      "grad_norm": 21.5,
      "learning_rate": 8.006393175880993e-06,
      "logits/chosen": -3.3358447551727295,
      "logits/rejected": -3.264803647994995,
      "logps/chosen": -357.3340759277344,
      "logps/rejected": -412.5440368652344,
      "loss": 0.2442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.434146881103516,
      "rewards/margins": 3.1869869232177734,
      "rewards/rejected": -9.621133804321289,
      "step": 8090
    },
    {
      "epoch": 1.7533845987219756,
      "grad_norm": 22.0,
      "learning_rate": 8.000024829334059e-06,
      "logits/chosen": -3.320702314376831,
      "logits/rejected": -3.4178929328918457,
      "logps/chosen": -362.0105285644531,
      "logps/rejected": -415.1690368652344,
      "loss": 0.2032,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.791146755218506,
      "rewards/margins": 3.5195090770721436,
      "rewards/rejected": -10.310656547546387,
      "step": 8095
    },
    {
      "epoch": 1.7544676703130078,
      "grad_norm": 24.875,
      "learning_rate": 7.99365799766208e-06,
      "logits/chosen": -3.4587693214416504,
      "logits/rejected": -3.38621187210083,
      "logps/chosen": -350.4355163574219,
      "logps/rejected": -414.31964111328125,
      "loss": 0.2817,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.11322021484375,
      "rewards/margins": 3.2891952991485596,
      "rewards/rejected": -9.402414321899414,
      "step": 8100
    },
    {
      "epoch": 1.75555074190404,
      "grad_norm": 19.625,
      "learning_rate": 7.987292689942881e-06,
      "logits/chosen": -3.244767427444458,
      "logits/rejected": -3.244572401046753,
      "logps/chosen": -334.8599853515625,
      "logps/rejected": -391.0261535644531,
      "loss": 0.3255,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.364319801330566,
      "rewards/margins": 2.5355422496795654,
      "rewards/rejected": -8.899862289428711,
      "step": 8105
    },
    {
      "epoch": 1.756633813495072,
      "grad_norm": 15.125,
      "learning_rate": 7.980928915252102e-06,
      "logits/chosen": -3.2625763416290283,
      "logits/rejected": -3.3952574729919434,
      "logps/chosen": -342.8200378417969,
      "logps/rejected": -398.7030029296875,
      "loss": 0.2267,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.811882019042969,
      "rewards/margins": 3.3528220653533936,
      "rewards/rejected": -9.164704322814941,
      "step": 8110
    },
    {
      "epoch": 1.7577168850861042,
      "grad_norm": 24.5,
      "learning_rate": 7.9745666826632e-06,
      "logits/chosen": -3.419031858444214,
      "logits/rejected": -3.3148536682128906,
      "logps/chosen": -338.86322021484375,
      "logps/rejected": -398.7893981933594,
      "loss": 0.2918,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.66196346282959,
      "rewards/margins": 3.082500457763672,
      "rewards/rejected": -9.744463920593262,
      "step": 8115
    },
    {
      "epoch": 1.7587999566771364,
      "grad_norm": 17.375,
      "learning_rate": 7.968206001247431e-06,
      "logits/chosen": -3.345994234085083,
      "logits/rejected": -3.3796303272247314,
      "logps/chosen": -350.23406982421875,
      "logps/rejected": -395.3008117675781,
      "loss": 0.2023,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.443139553070068,
      "rewards/margins": 3.0487210750579834,
      "rewards/rejected": -9.491861343383789,
      "step": 8120
    },
    {
      "epoch": 1.7598830282681686,
      "grad_norm": 18.5,
      "learning_rate": 7.961846880073847e-06,
      "logits/chosen": -3.508265256881714,
      "logits/rejected": -3.471205234527588,
      "logps/chosen": -313.8059387207031,
      "logps/rejected": -384.72528076171875,
      "loss": 0.213,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.052445411682129,
      "rewards/margins": 3.458955764770508,
      "rewards/rejected": -9.51140308380127,
      "step": 8125
    },
    {
      "epoch": 1.7609660998592007,
      "grad_norm": 29.625,
      "learning_rate": 7.955489328209267e-06,
      "logits/chosen": -3.2932400703430176,
      "logits/rejected": -3.30694580078125,
      "logps/chosen": -346.68658447265625,
      "logps/rejected": -401.1622009277344,
      "loss": 0.3013,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -5.948622703552246,
      "rewards/margins": 3.1242048740386963,
      "rewards/rejected": -9.072827339172363,
      "step": 8130
    },
    {
      "epoch": 1.7620491714502329,
      "grad_norm": 25.5,
      "learning_rate": 7.949133354718274e-06,
      "logits/chosen": -3.3488173484802246,
      "logits/rejected": -3.3334879875183105,
      "logps/chosen": -330.88397216796875,
      "logps/rejected": -392.6513977050781,
      "loss": 0.1703,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.889535903930664,
      "rewards/margins": 3.356135129928589,
      "rewards/rejected": -9.245671272277832,
      "step": 8135
    },
    {
      "epoch": 1.763132243041265,
      "grad_norm": 15.0,
      "learning_rate": 7.942778968663203e-06,
      "logits/chosen": -3.1969313621520996,
      "logits/rejected": -3.2251293659210205,
      "logps/chosen": -338.1903076171875,
      "logps/rejected": -395.2104187011719,
      "loss": 0.2391,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.5381598472595215,
      "rewards/margins": 3.032541275024414,
      "rewards/rejected": -8.57070255279541,
      "step": 8140
    },
    {
      "epoch": 1.7642153146322972,
      "grad_norm": 26.625,
      "learning_rate": 7.93642617910413e-06,
      "logits/chosen": -3.3025379180908203,
      "logits/rejected": -3.3284473419189453,
      "logps/chosen": -363.01318359375,
      "logps/rejected": -419.4964294433594,
      "loss": 0.2203,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.489510536193848,
      "rewards/margins": 3.0165927410125732,
      "rewards/rejected": -9.506103515625,
      "step": 8145
    },
    {
      "epoch": 1.7652983862233294,
      "grad_norm": 24.5,
      "learning_rate": 7.930074995098843e-06,
      "logits/chosen": -3.3884778022766113,
      "logits/rejected": -3.2636210918426514,
      "logps/chosen": -350.21075439453125,
      "logps/rejected": -413.5787048339844,
      "loss": 0.2257,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.51792049407959,
      "rewards/margins": 2.981090545654297,
      "rewards/rejected": -9.499011039733887,
      "step": 8150
    },
    {
      "epoch": 1.7663814578143615,
      "grad_norm": 27.0,
      "learning_rate": 7.92372542570285e-06,
      "logits/chosen": -3.4094841480255127,
      "logits/rejected": -3.3888416290283203,
      "logps/chosen": -351.39385986328125,
      "logps/rejected": -404.4165954589844,
      "loss": 0.2785,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.185462474822998,
      "rewards/margins": 3.139655351638794,
      "rewards/rejected": -9.325118064880371,
      "step": 8155
    },
    {
      "epoch": 1.7674645294053937,
      "grad_norm": 23.75,
      "learning_rate": 7.917377479969355e-06,
      "logits/chosen": -3.4964470863342285,
      "logits/rejected": -3.413074016571045,
      "logps/chosen": -346.6517028808594,
      "logps/rejected": -407.18035888671875,
      "loss": 0.2126,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.733954429626465,
      "rewards/margins": 3.204792022705078,
      "rewards/rejected": -9.938746452331543,
      "step": 8160
    },
    {
      "epoch": 1.7685476009964258,
      "grad_norm": 21.375,
      "learning_rate": 7.911031166949246e-06,
      "logits/chosen": -3.5386359691619873,
      "logits/rejected": -3.335465669631958,
      "logps/chosen": -343.4313049316406,
      "logps/rejected": -434.942626953125,
      "loss": 0.1648,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.340444087982178,
      "rewards/margins": 3.9130637645721436,
      "rewards/rejected": -10.253508567810059,
      "step": 8165
    },
    {
      "epoch": 1.769630672587458,
      "grad_norm": 20.125,
      "learning_rate": 7.90468649569108e-06,
      "logits/chosen": -3.534710645675659,
      "logits/rejected": -3.409766674041748,
      "logps/chosen": -345.51123046875,
      "logps/rejected": -427.1748962402344,
      "loss": 0.1659,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.254249572753906,
      "rewards/margins": 3.8225083351135254,
      "rewards/rejected": -10.076757431030273,
      "step": 8170
    },
    {
      "epoch": 1.7707137441784901,
      "grad_norm": 21.625,
      "learning_rate": 7.89834347524108e-06,
      "logits/chosen": -3.4136626720428467,
      "logits/rejected": -3.3956661224365234,
      "logps/chosen": -358.46722412109375,
      "logps/rejected": -427.2906188964844,
      "loss": 0.249,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.5609331130981445,
      "rewards/margins": 3.577470302581787,
      "rewards/rejected": -10.138402938842773,
      "step": 8175
    },
    {
      "epoch": 1.7717968157695223,
      "grad_norm": 15.5625,
      "learning_rate": 7.892002114643111e-06,
      "logits/chosen": -3.4880542755126953,
      "logits/rejected": -3.4077351093292236,
      "logps/chosen": -337.7037048339844,
      "logps/rejected": -405.52960205078125,
      "loss": 0.2468,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.200531482696533,
      "rewards/margins": 3.078298330307007,
      "rewards/rejected": -9.278829574584961,
      "step": 8180
    },
    {
      "epoch": 1.7728798873605545,
      "grad_norm": 29.125,
      "learning_rate": 7.885662422938674e-06,
      "logits/chosen": -3.400186061859131,
      "logits/rejected": -3.418732166290283,
      "logps/chosen": -363.7987976074219,
      "logps/rejected": -425.77630615234375,
      "loss": 0.2539,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.688124179840088,
      "rewards/margins": 3.4017436504364014,
      "rewards/rejected": -10.089868545532227,
      "step": 8185
    },
    {
      "epoch": 1.7739629589515866,
      "grad_norm": 20.0,
      "learning_rate": 7.879324409166887e-06,
      "logits/chosen": -3.3868165016174316,
      "logits/rejected": -3.305180311203003,
      "logps/chosen": -351.12628173828125,
      "logps/rejected": -401.13446044921875,
      "loss": 0.2326,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -6.42153787612915,
      "rewards/margins": 3.014800548553467,
      "rewards/rejected": -9.436339378356934,
      "step": 8190
    },
    {
      "epoch": 1.7750460305426188,
      "grad_norm": 22.125,
      "learning_rate": 7.872988082364475e-06,
      "logits/chosen": -3.391331434249878,
      "logits/rejected": -3.3903636932373047,
      "logps/chosen": -340.89068603515625,
      "logps/rejected": -414.91937255859375,
      "loss": 0.2658,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.823500633239746,
      "rewards/margins": 3.768113613128662,
      "rewards/rejected": -9.59161376953125,
      "step": 8195
    },
    {
      "epoch": 1.776129102133651,
      "grad_norm": 21.25,
      "learning_rate": 7.866653451565763e-06,
      "logits/chosen": -3.345299482345581,
      "logits/rejected": -3.2910428047180176,
      "logps/chosen": -341.3680419921875,
      "logps/rejected": -417.36376953125,
      "loss": 0.2546,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.310941696166992,
      "rewards/margins": 3.310528516769409,
      "rewards/rejected": -9.62147045135498,
      "step": 8200
    },
    {
      "epoch": 1.777212173724683,
      "grad_norm": 20.625,
      "learning_rate": 7.860320525802656e-06,
      "logits/chosen": -3.497066020965576,
      "logits/rejected": -3.422095537185669,
      "logps/chosen": -340.750732421875,
      "logps/rejected": -403.6180725097656,
      "loss": 0.2549,
      "rewards/accuracies": 0.8916667699813843,
      "rewards/chosen": -6.355643272399902,
      "rewards/margins": 3.2138829231262207,
      "rewards/rejected": -9.569524765014648,
      "step": 8205
    },
    {
      "epoch": 1.7782952453157153,
      "grad_norm": 27.625,
      "learning_rate": 7.853989314104621e-06,
      "logits/chosen": -3.384403944015503,
      "logits/rejected": -3.2706546783447266,
      "logps/chosen": -342.49005126953125,
      "logps/rejected": -415.6665954589844,
      "loss": 0.2398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.244143009185791,
      "rewards/margins": 3.1576027870178223,
      "rewards/rejected": -9.401745796203613,
      "step": 8210
    },
    {
      "epoch": 1.7793783169067474,
      "grad_norm": 20.875,
      "learning_rate": 7.847659825498694e-06,
      "logits/chosen": -3.4422125816345215,
      "logits/rejected": -3.383897304534912,
      "logps/chosen": -340.945068359375,
      "logps/rejected": -407.14801025390625,
      "loss": 0.2891,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.3332839012146,
      "rewards/margins": 2.9495465755462646,
      "rewards/rejected": -9.282831192016602,
      "step": 8215
    },
    {
      "epoch": 1.7804613884977796,
      "grad_norm": 27.25,
      "learning_rate": 7.841332069009443e-06,
      "logits/chosen": -3.3780124187469482,
      "logits/rejected": -3.3343169689178467,
      "logps/chosen": -349.46630859375,
      "logps/rejected": -401.6248474121094,
      "loss": 0.2286,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.519265651702881,
      "rewards/margins": 2.9403741359710693,
      "rewards/rejected": -9.459640502929688,
      "step": 8220
    },
    {
      "epoch": 1.7815444600888117,
      "grad_norm": 30.5,
      "learning_rate": 7.835006053658972e-06,
      "logits/chosen": -3.4037227630615234,
      "logits/rejected": -3.3700332641601562,
      "logps/chosen": -364.0152282714844,
      "logps/rejected": -400.90802001953125,
      "loss": 0.2407,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.51946496963501,
      "rewards/margins": 2.6814427375793457,
      "rewards/rejected": -9.200908660888672,
      "step": 8225
    },
    {
      "epoch": 1.782627531679844,
      "grad_norm": 25.25,
      "learning_rate": 7.828681788466899e-06,
      "logits/chosen": -3.376319169998169,
      "logits/rejected": -3.36098051071167,
      "logps/chosen": -347.2886657714844,
      "logps/rejected": -402.09649658203125,
      "loss": 0.2417,
      "rewards/accuracies": 0.9249998927116394,
      "rewards/chosen": -6.223296165466309,
      "rewards/margins": 3.0677359104156494,
      "rewards/rejected": -9.291030883789062,
      "step": 8230
    },
    {
      "epoch": 1.783710603270876,
      "grad_norm": 28.375,
      "learning_rate": 7.822359282450352e-06,
      "logits/chosen": -3.3151040077209473,
      "logits/rejected": -3.390700101852417,
      "logps/chosen": -352.26312255859375,
      "logps/rejected": -410.41033935546875,
      "loss": 0.2846,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.350226402282715,
      "rewards/margins": 3.11210298538208,
      "rewards/rejected": -9.462329864501953,
      "step": 8235
    },
    {
      "epoch": 1.7847936748619082,
      "grad_norm": 16.25,
      "learning_rate": 7.816038544623942e-06,
      "logits/chosen": -3.262378692626953,
      "logits/rejected": -3.2178807258605957,
      "logps/chosen": -329.0621643066406,
      "logps/rejected": -378.818603515625,
      "loss": 0.2466,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.711397171020508,
      "rewards/margins": 3.0442514419555664,
      "rewards/rejected": -8.755647659301758,
      "step": 8240
    },
    {
      "epoch": 1.7858767464529406,
      "grad_norm": 22.0,
      "learning_rate": 7.809719583999769e-06,
      "logits/chosen": -3.432832717895508,
      "logits/rejected": -3.4253971576690674,
      "logps/chosen": -340.4163513183594,
      "logps/rejected": -397.7962951660156,
      "loss": 0.3074,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -5.967554569244385,
      "rewards/margins": 3.1401314735412598,
      "rewards/rejected": -9.107686042785645,
      "step": 8245
    },
    {
      "epoch": 1.7869598180439727,
      "grad_norm": 9.3125,
      "learning_rate": 7.803402409587392e-06,
      "logits/chosen": -3.2742645740509033,
      "logits/rejected": -3.294106960296631,
      "logps/chosen": -355.2840881347656,
      "logps/rejected": -397.39617919921875,
      "loss": 0.2654,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.314670562744141,
      "rewards/margins": 3.0435614585876465,
      "rewards/rejected": -9.358232498168945,
      "step": 8250
    },
    {
      "epoch": 1.788042889635005,
      "grad_norm": 23.875,
      "learning_rate": 7.79708703039383e-06,
      "logits/chosen": -3.3281421661376953,
      "logits/rejected": -3.244288206100464,
      "logps/chosen": -344.690185546875,
      "logps/rejected": -404.2783203125,
      "loss": 0.2512,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -6.131931304931641,
      "rewards/margins": 3.0766265392303467,
      "rewards/rejected": -9.208558082580566,
      "step": 8255
    },
    {
      "epoch": 1.789125961226037,
      "grad_norm": 29.0,
      "learning_rate": 7.790773455423532e-06,
      "logits/chosen": -3.376703977584839,
      "logits/rejected": -3.2735142707824707,
      "logps/chosen": -328.6663513183594,
      "logps/rejected": -391.4866027832031,
      "loss": 0.2161,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.833152770996094,
      "rewards/margins": 3.155816078186035,
      "rewards/rejected": -8.988969802856445,
      "step": 8260
    },
    {
      "epoch": 1.7902090328170692,
      "grad_norm": 36.75,
      "learning_rate": 7.784461693678384e-06,
      "logits/chosen": -3.4167392253875732,
      "logits/rejected": -3.471005916595459,
      "logps/chosen": -348.1844482421875,
      "logps/rejected": -395.0099182128906,
      "loss": 0.2697,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.706645965576172,
      "rewards/margins": 2.8028924465179443,
      "rewards/rejected": -9.509537696838379,
      "step": 8265
    },
    {
      "epoch": 1.7912921044081014,
      "grad_norm": 17.5,
      "learning_rate": 7.778151754157679e-06,
      "logits/chosen": -3.2235400676727295,
      "logits/rejected": -3.1826021671295166,
      "logps/chosen": -356.93072509765625,
      "logps/rejected": -414.342041015625,
      "loss": 0.2455,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.628097057342529,
      "rewards/margins": 3.1535091400146484,
      "rewards/rejected": -8.781608581542969,
      "step": 8270
    },
    {
      "epoch": 1.7923751759991335,
      "grad_norm": 26.375,
      "learning_rate": 7.771843645858123e-06,
      "logits/chosen": -3.4787166118621826,
      "logits/rejected": -3.345120906829834,
      "logps/chosen": -363.9753112792969,
      "logps/rejected": -426.16650390625,
      "loss": 0.2848,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -7.064262390136719,
      "rewards/margins": 3.0069515705108643,
      "rewards/rejected": -10.071212768554688,
      "step": 8275
    },
    {
      "epoch": 1.7934582475901657,
      "grad_norm": 31.125,
      "learning_rate": 7.7655373777738e-06,
      "logits/chosen": -3.483267307281494,
      "logits/rejected": -3.357208728790283,
      "logps/chosen": -348.53607177734375,
      "logps/rejected": -419.89154052734375,
      "loss": 0.3033,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.610105991363525,
      "rewards/margins": 2.9719204902648926,
      "rewards/rejected": -9.58202838897705,
      "step": 8280
    },
    {
      "epoch": 1.7945413191811979,
      "grad_norm": 34.75,
      "learning_rate": 7.759232958896172e-06,
      "logits/chosen": -3.404839038848877,
      "logits/rejected": -3.352254867553711,
      "logps/chosen": -338.17962646484375,
      "logps/rejected": -402.00091552734375,
      "loss": 0.234,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.479266166687012,
      "rewards/margins": 3.0551564693450928,
      "rewards/rejected": -9.534423828125,
      "step": 8285
    },
    {
      "epoch": 1.79562439077223,
      "grad_norm": 31.625,
      "learning_rate": 7.75293039821407e-06,
      "logits/chosen": -3.396970748901367,
      "logits/rejected": -3.410198926925659,
      "logps/chosen": -337.3106689453125,
      "logps/rejected": -406.06427001953125,
      "loss": 0.2291,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -6.050718307495117,
      "rewards/margins": 3.349233627319336,
      "rewards/rejected": -9.39995288848877,
      "step": 8290
    },
    {
      "epoch": 1.7967074623632622,
      "grad_norm": 16.375,
      "learning_rate": 7.746629704713673e-06,
      "logits/chosen": -3.522439956665039,
      "logits/rejected": -3.4521565437316895,
      "logps/chosen": -336.3605041503906,
      "logps/rejected": -408.8424377441406,
      "loss": 0.1973,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.463712215423584,
      "rewards/margins": 3.5623295307159424,
      "rewards/rejected": -10.026041030883789,
      "step": 8295
    },
    {
      "epoch": 1.7977905339542943,
      "grad_norm": 29.5,
      "learning_rate": 7.740330887378495e-06,
      "logits/chosen": -3.3594391345977783,
      "logits/rejected": -3.2504348754882812,
      "logps/chosen": -363.59136962890625,
      "logps/rejected": -427.0015563964844,
      "loss": 0.2435,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.338497161865234,
      "rewards/margins": 3.1855406761169434,
      "rewards/rejected": -9.524038314819336,
      "step": 8300
    },
    {
      "epoch": 1.7988736055453265,
      "grad_norm": 29.875,
      "learning_rate": 7.73403395518938e-06,
      "logits/chosen": -3.3509483337402344,
      "logits/rejected": -3.4768447875976562,
      "logps/chosen": -350.7064208984375,
      "logps/rejected": -399.46710205078125,
      "loss": 0.2932,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.965395450592041,
      "rewards/margins": 2.796736240386963,
      "rewards/rejected": -9.762131690979004,
      "step": 8305
    },
    {
      "epoch": 1.7999566771363589,
      "grad_norm": 22.75,
      "learning_rate": 7.727738917124476e-06,
      "logits/chosen": -3.6126601696014404,
      "logits/rejected": -3.526273250579834,
      "logps/chosen": -341.59234619140625,
      "logps/rejected": -406.94488525390625,
      "loss": 0.2767,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.523411750793457,
      "rewards/margins": 3.0395853519439697,
      "rewards/rejected": -9.562997817993164,
      "step": 8310
    },
    {
      "epoch": 1.801039748727391,
      "grad_norm": 31.625,
      "learning_rate": 7.72144578215924e-06,
      "logits/chosen": -3.4116053581237793,
      "logits/rejected": -3.391467332839966,
      "logps/chosen": -345.8619079589844,
      "logps/rejected": -402.7410583496094,
      "loss": 0.2312,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.056614875793457,
      "rewards/margins": 3.4809093475341797,
      "rewards/rejected": -9.537524223327637,
      "step": 8315
    },
    {
      "epoch": 1.8021228203184232,
      "grad_norm": 18.625,
      "learning_rate": 7.71515455926641e-06,
      "logits/chosen": -3.501974105834961,
      "logits/rejected": -3.2970707416534424,
      "logps/chosen": -353.42852783203125,
      "logps/rejected": -430.89678955078125,
      "loss": 0.181,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -6.722494602203369,
      "rewards/margins": 3.432305097579956,
      "rewards/rejected": -10.154801368713379,
      "step": 8320
    },
    {
      "epoch": 1.8032058919094553,
      "grad_norm": 27.75,
      "learning_rate": 7.708865257416e-06,
      "logits/chosen": -3.5350124835968018,
      "logits/rejected": -3.4731571674346924,
      "logps/chosen": -342.0189208984375,
      "logps/rejected": -406.1677551269531,
      "loss": 0.282,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.543013572692871,
      "rewards/margins": 2.9958183765411377,
      "rewards/rejected": -9.538832664489746,
      "step": 8325
    },
    {
      "epoch": 1.8042889635004875,
      "grad_norm": 28.875,
      "learning_rate": 7.702577885575281e-06,
      "logits/chosen": -3.42565655708313,
      "logits/rejected": -3.3497252464294434,
      "logps/chosen": -344.79010009765625,
      "logps/rejected": -393.7426452636719,
      "loss": 0.2542,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.413697719573975,
      "rewards/margins": 2.7924695014953613,
      "rewards/rejected": -9.20616626739502,
      "step": 8330
    },
    {
      "epoch": 1.8053720350915197,
      "grad_norm": 39.75,
      "learning_rate": 7.696292452708779e-06,
      "logits/chosen": -3.393890857696533,
      "logits/rejected": -3.411257266998291,
      "logps/chosen": -349.90252685546875,
      "logps/rejected": -399.2911071777344,
      "loss": 0.3296,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.874824523925781,
      "rewards/margins": 2.973849058151245,
      "rewards/rejected": -8.848672866821289,
      "step": 8335
    },
    {
      "epoch": 1.8064551066825518,
      "grad_norm": 33.75,
      "learning_rate": 7.690008967778249e-06,
      "logits/chosen": -3.4750266075134277,
      "logits/rejected": -3.4421229362487793,
      "logps/chosen": -342.3661804199219,
      "logps/rejected": -404.789794921875,
      "loss": 0.2852,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.147346019744873,
      "rewards/margins": 3.207547426223755,
      "rewards/rejected": -9.35489273071289,
      "step": 8340
    },
    {
      "epoch": 1.807538178273584,
      "grad_norm": 16.875,
      "learning_rate": 7.683727439742672e-06,
      "logits/chosen": -3.31427264213562,
      "logits/rejected": -3.2438836097717285,
      "logps/chosen": -370.69512939453125,
      "logps/rejected": -432.9740295410156,
      "loss": 0.2052,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.323065757751465,
      "rewards/margins": 3.2834830284118652,
      "rewards/rejected": -9.606550216674805,
      "step": 8345
    },
    {
      "epoch": 1.8086212498646161,
      "grad_norm": 22.125,
      "learning_rate": 7.677447877558238e-06,
      "logits/chosen": -3.385066509246826,
      "logits/rejected": -3.398239850997925,
      "logps/chosen": -356.3119201660156,
      "logps/rejected": -418.94671630859375,
      "loss": 0.1885,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -6.311424732208252,
      "rewards/margins": 3.610029935836792,
      "rewards/rejected": -9.921454429626465,
      "step": 8350
    },
    {
      "epoch": 1.8097043214556483,
      "grad_norm": 26.0,
      "learning_rate": 7.671170290178331e-06,
      "logits/chosen": -3.440699338912964,
      "logits/rejected": -3.4773077964782715,
      "logps/chosen": -342.14276123046875,
      "logps/rejected": -395.88177490234375,
      "loss": 0.2798,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.496354103088379,
      "rewards/margins": 2.985257625579834,
      "rewards/rejected": -9.481612205505371,
      "step": 8355
    },
    {
      "epoch": 1.8107873930466805,
      "grad_norm": 18.0,
      "learning_rate": 7.664894686553526e-06,
      "logits/chosen": -3.499682903289795,
      "logits/rejected": -3.512174129486084,
      "logps/chosen": -354.07598876953125,
      "logps/rejected": -414.08270263671875,
      "loss": 0.2702,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.888572692871094,
      "rewards/margins": 3.303147792816162,
      "rewards/rejected": -10.191720962524414,
      "step": 8360
    },
    {
      "epoch": 1.8118704646377126,
      "grad_norm": 23.0,
      "learning_rate": 7.658621075631569e-06,
      "logits/chosen": -3.4456112384796143,
      "logits/rejected": -3.4714722633361816,
      "logps/chosen": -352.4776306152344,
      "logps/rejected": -404.8619079589844,
      "loss": 0.1886,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -6.577867031097412,
      "rewards/margins": 3.1819725036621094,
      "rewards/rejected": -9.75984001159668,
      "step": 8365
    },
    {
      "epoch": 1.8129535362287448,
      "grad_norm": 17.625,
      "learning_rate": 7.652349466357356e-06,
      "logits/chosen": -3.3470444679260254,
      "logits/rejected": -3.3476345539093018,
      "logps/chosen": -348.7019958496094,
      "logps/rejected": -409.674072265625,
      "loss": 0.2129,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.03096866607666,
      "rewards/margins": 3.256945848464966,
      "rewards/rejected": -9.287914276123047,
      "step": 8370
    },
    {
      "epoch": 1.814036607819777,
      "grad_norm": 18.875,
      "learning_rate": 7.646079867672937e-06,
      "logits/chosen": -3.4595038890838623,
      "logits/rejected": -3.523921489715576,
      "logps/chosen": -341.2619934082031,
      "logps/rejected": -406.4261779785156,
      "loss": 0.1848,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -6.164153099060059,
      "rewards/margins": 3.5986580848693848,
      "rewards/rejected": -9.762811660766602,
      "step": 8375
    },
    {
      "epoch": 1.815119679410809,
      "grad_norm": 17.625,
      "learning_rate": 7.639812288517493e-06,
      "logits/chosen": -3.4587063789367676,
      "logits/rejected": -3.499951124191284,
      "logps/chosen": -362.8094482421875,
      "logps/rejected": -421.2620544433594,
      "loss": 0.2273,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -7.1602582931518555,
      "rewards/margins": 3.3365883827209473,
      "rewards/rejected": -10.496847152709961,
      "step": 8380
    },
    {
      "epoch": 1.8162027510018413,
      "grad_norm": 17.75,
      "learning_rate": 7.633546737827325e-06,
      "logits/chosen": -3.5455849170684814,
      "logits/rejected": -3.4301846027374268,
      "logps/chosen": -358.8755798339844,
      "logps/rejected": -433.3658142089844,
      "loss": 0.231,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.79339075088501,
      "rewards/margins": 3.6094558238983154,
      "rewards/rejected": -10.402846336364746,
      "step": 8385
    },
    {
      "epoch": 1.8172858225928734,
      "grad_norm": 14.125,
      "learning_rate": 7.627283224535844e-06,
      "logits/chosen": -3.4454808235168457,
      "logits/rejected": -3.428114414215088,
      "logps/chosen": -345.0173034667969,
      "logps/rejected": -407.9368896484375,
      "loss": 0.2231,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.011581897735596,
      "rewards/margins": 3.155622959136963,
      "rewards/rejected": -9.167204856872559,
      "step": 8390
    },
    {
      "epoch": 1.8183688941839056,
      "grad_norm": 28.0,
      "learning_rate": 7.621021757573554e-06,
      "logits/chosen": -3.413142681121826,
      "logits/rejected": -3.425377368927002,
      "logps/chosen": -347.28179931640625,
      "logps/rejected": -412.6964416503906,
      "loss": 0.2167,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.587004661560059,
      "rewards/margins": 3.441331148147583,
      "rewards/rejected": -10.028335571289062,
      "step": 8395
    },
    {
      "epoch": 1.8194519657749377,
      "grad_norm": 21.125,
      "learning_rate": 7.614762345868042e-06,
      "logits/chosen": -3.4172942638397217,
      "logits/rejected": -3.4514527320861816,
      "logps/chosen": -364.3475341796875,
      "logps/rejected": -423.57525634765625,
      "loss": 0.2719,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.923779487609863,
      "rewards/margins": 3.177563190460205,
      "rewards/rejected": -10.10134220123291,
      "step": 8400
    },
    {
      "epoch": 1.8205350373659699,
      "grad_norm": 23.75,
      "learning_rate": 7.608504998343969e-06,
      "logits/chosen": -3.4325804710388184,
      "logits/rejected": -3.4746623039245605,
      "logps/chosen": -365.75653076171875,
      "logps/rejected": -425.0745544433594,
      "loss": 0.2659,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.9976301193237305,
      "rewards/margins": 3.2118313312530518,
      "rewards/rejected": -10.20946216583252,
      "step": 8405
    },
    {
      "epoch": 1.821618108957002,
      "grad_norm": 27.375,
      "learning_rate": 7.602249723923039e-06,
      "logits/chosen": -3.5013504028320312,
      "logits/rejected": -3.607421398162842,
      "logps/chosen": -352.52728271484375,
      "logps/rejected": -396.1763610839844,
      "loss": 0.2421,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -7.084596157073975,
      "rewards/margins": 3.105402708053589,
      "rewards/rejected": -10.189998626708984,
      "step": 8410
    },
    {
      "epoch": 1.8227011805480342,
      "grad_norm": 22.125,
      "learning_rate": 7.5959965315240195e-06,
      "logits/chosen": -3.4366955757141113,
      "logits/rejected": -3.424175262451172,
      "logps/chosen": -350.6023254394531,
      "logps/rejected": -401.33221435546875,
      "loss": 0.2675,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.9726128578186035,
      "rewards/margins": 2.9875478744506836,
      "rewards/rejected": -9.960161209106445,
      "step": 8415
    },
    {
      "epoch": 1.8237842521390664,
      "grad_norm": 27.375,
      "learning_rate": 7.589745430062691e-06,
      "logits/chosen": -3.4592032432556152,
      "logits/rejected": -3.5597941875457764,
      "logps/chosen": -361.19464111328125,
      "logps/rejected": -412.1800842285156,
      "loss": 0.1827,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.795918941497803,
      "rewards/margins": 3.4021124839782715,
      "rewards/rejected": -10.198031425476074,
      "step": 8420
    },
    {
      "epoch": 1.8248673237300985,
      "grad_norm": 35.5,
      "learning_rate": 7.583496428451869e-06,
      "logits/chosen": -3.438019275665283,
      "logits/rejected": -3.3285603523254395,
      "logps/chosen": -348.3456115722656,
      "logps/rejected": -434.3353576660156,
      "loss": 0.2633,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.261181831359863,
      "rewards/margins": 3.6098854541778564,
      "rewards/rejected": -9.87106704711914,
      "step": 8425
    },
    {
      "epoch": 1.8259503953211307,
      "grad_norm": 25.375,
      "learning_rate": 7.57724953560136e-06,
      "logits/chosen": -3.4094672203063965,
      "logits/rejected": -3.3820483684539795,
      "logps/chosen": -341.3477478027344,
      "logps/rejected": -408.323974609375,
      "loss": 0.2804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.243919372558594,
      "rewards/margins": 3.151536464691162,
      "rewards/rejected": -9.395456314086914,
      "step": 8430
    },
    {
      "epoch": 1.8270334669121628,
      "grad_norm": 24.25,
      "learning_rate": 7.57100476041798e-06,
      "logits/chosen": -3.4014618396759033,
      "logits/rejected": -3.310850143432617,
      "logps/chosen": -346.63970947265625,
      "logps/rejected": -413.76910400390625,
      "loss": 0.2628,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.296353340148926,
      "rewards/margins": 3.170633554458618,
      "rewards/rejected": -9.466986656188965,
      "step": 8435
    },
    {
      "epoch": 1.828116538503195,
      "grad_norm": 19.875,
      "learning_rate": 7.564762111805509e-06,
      "logits/chosen": -3.4162662029266357,
      "logits/rejected": -3.3934922218322754,
      "logps/chosen": -355.3464050292969,
      "logps/rejected": -418.7093200683594,
      "loss": 0.2218,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.248719215393066,
      "rewards/margins": 3.5250401496887207,
      "rewards/rejected": -9.773759841918945,
      "step": 8440
    },
    {
      "epoch": 1.8291996100942272,
      "grad_norm": 37.0,
      "learning_rate": 7.558521598664708e-06,
      "logits/chosen": -3.399352550506592,
      "logits/rejected": -3.5094082355499268,
      "logps/chosen": -354.6970520019531,
      "logps/rejected": -403.4288635253906,
      "loss": 0.2791,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.334079742431641,
      "rewards/margins": 3.292987108230591,
      "rewards/rejected": -9.627066612243652,
      "step": 8445
    },
    {
      "epoch": 1.8302826816852593,
      "grad_norm": 27.875,
      "learning_rate": 7.552283229893284e-06,
      "logits/chosen": -3.528564453125,
      "logits/rejected": -3.4142982959747314,
      "logps/chosen": -346.66729736328125,
      "logps/rejected": -414.2886657714844,
      "loss": 0.3877,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -6.795240879058838,
      "rewards/margins": 3.023608446121216,
      "rewards/rejected": -9.818849563598633,
      "step": 8450
    },
    {
      "epoch": 1.8313657532762915,
      "grad_norm": 30.125,
      "learning_rate": 7.546047014385893e-06,
      "logits/chosen": -3.4562745094299316,
      "logits/rejected": -3.410799503326416,
      "logps/chosen": -362.85198974609375,
      "logps/rejected": -412.09197998046875,
      "loss": 0.3132,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.537581443786621,
      "rewards/margins": 3.1238110065460205,
      "rewards/rejected": -9.661392211914062,
      "step": 8455
    },
    {
      "epoch": 1.8324488248673236,
      "grad_norm": 22.625,
      "learning_rate": 7.5398129610341194e-06,
      "logits/chosen": -3.4160029888153076,
      "logits/rejected": -3.300602674484253,
      "logps/chosen": -344.4552917480469,
      "logps/rejected": -415.8011169433594,
      "loss": 0.2456,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.2076735496521,
      "rewards/margins": 3.3447933197021484,
      "rewards/rejected": -9.552467346191406,
      "step": 8460
    },
    {
      "epoch": 1.8335318964583558,
      "grad_norm": 22.5,
      "learning_rate": 7.533581078726462e-06,
      "logits/chosen": -3.4193615913391113,
      "logits/rejected": -3.3393635749816895,
      "logps/chosen": -334.24700927734375,
      "logps/rejected": -400.6553955078125,
      "loss": 0.252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.209761142730713,
      "rewards/margins": 3.1564011573791504,
      "rewards/rejected": -9.36616325378418,
      "step": 8465
    },
    {
      "epoch": 1.834614968049388,
      "grad_norm": 26.75,
      "learning_rate": 7.527351376348326e-06,
      "logits/chosen": -3.398554563522339,
      "logits/rejected": -3.406140089035034,
      "logps/chosen": -346.3675842285156,
      "logps/rejected": -394.46368408203125,
      "loss": 0.2785,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.07343053817749,
      "rewards/margins": 2.8451759815216064,
      "rewards/rejected": -8.918606758117676,
      "step": 8470
    },
    {
      "epoch": 1.83569803964042,
      "grad_norm": 21.875,
      "learning_rate": 7.52112386278201e-06,
      "logits/chosen": -3.274111270904541,
      "logits/rejected": -3.260653257369995,
      "logps/chosen": -365.2570495605469,
      "logps/rejected": -433.2776794433594,
      "loss": 0.2213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.521506309509277,
      "rewards/margins": 3.498063325881958,
      "rewards/rejected": -10.019569396972656,
      "step": 8475
    },
    {
      "epoch": 1.8367811112314523,
      "grad_norm": 19.5,
      "learning_rate": 7.514898546906691e-06,
      "logits/chosen": -3.4594993591308594,
      "logits/rejected": -3.431791305541992,
      "logps/chosen": -334.54083251953125,
      "logps/rejected": -408.0412292480469,
      "loss": 0.1778,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.1429033279418945,
      "rewards/margins": 3.6513524055480957,
      "rewards/rejected": -9.794256210327148,
      "step": 8480
    },
    {
      "epoch": 1.8378641828224844,
      "grad_norm": 21.375,
      "learning_rate": 7.508675437598409e-06,
      "logits/chosen": -3.452190399169922,
      "logits/rejected": -3.45709228515625,
      "logps/chosen": -355.8913269042969,
      "logps/rejected": -391.1919250488281,
      "loss": 0.3384,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.397363662719727,
      "rewards/margins": 2.454059362411499,
      "rewards/rejected": -8.851424217224121,
      "step": 8485
    },
    {
      "epoch": 1.8389472544135166,
      "grad_norm": 27.0,
      "learning_rate": 7.502454543730063e-06,
      "logits/chosen": -3.336272716522217,
      "logits/rejected": -3.332129955291748,
      "logps/chosen": -344.2724914550781,
      "logps/rejected": -401.4328308105469,
      "loss": 0.25,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.722356796264648,
      "rewards/margins": 3.0976128578186035,
      "rewards/rejected": -8.819969177246094,
      "step": 8490
    },
    {
      "epoch": 1.8400303260045487,
      "grad_norm": 24.625,
      "learning_rate": 7.496235874171389e-06,
      "logits/chosen": -3.2664215564727783,
      "logits/rejected": -3.284480571746826,
      "logps/chosen": -334.7022705078125,
      "logps/rejected": -407.25408935546875,
      "loss": 0.212,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.913434982299805,
      "rewards/margins": 3.404186248779297,
      "rewards/rejected": -9.317621231079102,
      "step": 8495
    },
    {
      "epoch": 1.841113397595581,
      "grad_norm": 12.5625,
      "learning_rate": 7.4900194377889565e-06,
      "logits/chosen": -3.419332981109619,
      "logits/rejected": -3.3038337230682373,
      "logps/chosen": -318.5957336425781,
      "logps/rejected": -389.79461669921875,
      "loss": 0.2665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.309103488922119,
      "rewards/margins": 3.1356990337371826,
      "rewards/rejected": -8.444803237915039,
      "step": 8500
    },
    {
      "epoch": 1.841113397595581,
      "eval_logits/chosen": -3.4283788204193115,
      "eval_logits/rejected": -3.367488384246826,
      "eval_logps/chosen": -329.9504699707031,
      "eval_logps/rejected": -382.17242431640625,
      "eval_loss": 0.372687429189682,
      "eval_rewards/accuracies": 0.8050000071525574,
      "eval_rewards/chosen": -5.787458896636963,
      "eval_rewards/margins": 2.4383039474487305,
      "eval_rewards/rejected": -8.225763320922852,
      "eval_runtime": 14.4047,
      "eval_samples_per_second": 13.884,
      "eval_steps_per_second": 13.884,
      "step": 8500
    },
    {
      "epoch": 1.8421964691866133,
      "grad_norm": 33.5,
      "learning_rate": 7.483805243446143e-06,
      "logits/chosen": -3.3655242919921875,
      "logits/rejected": -3.327535629272461,
      "logps/chosen": -351.17059326171875,
      "logps/rejected": -399.9230041503906,
      "loss": 0.3355,
      "rewards/accuracies": 0.8500000834465027,
      "rewards/chosen": -6.253751277923584,
      "rewards/margins": 2.6357460021972656,
      "rewards/rejected": -8.889497756958008,
      "step": 8505
    },
    {
      "epoch": 1.8432795407776454,
      "grad_norm": 18.25,
      "learning_rate": 7.477593300003141e-06,
      "logits/chosen": -3.384983539581299,
      "logits/rejected": -3.457653522491455,
      "logps/chosen": -356.9963073730469,
      "logps/rejected": -416.3392639160156,
      "loss": 0.2252,
      "rewards/accuracies": 0.9416667222976685,
      "rewards/chosen": -6.702123165130615,
      "rewards/margins": 3.3418288230895996,
      "rewards/rejected": -10.043951988220215,
      "step": 8510
    },
    {
      "epoch": 1.8443626123686776,
      "grad_norm": 20.75,
      "learning_rate": 7.471383616316923e-06,
      "logits/chosen": -3.160984516143799,
      "logits/rejected": -3.2346484661102295,
      "logps/chosen": -343.22064208984375,
      "logps/rejected": -401.15985107421875,
      "loss": 0.2051,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": -5.578729629516602,
      "rewards/margins": 3.1845250129699707,
      "rewards/rejected": -8.763254165649414,
      "step": 8515
    },
    {
      "epoch": 1.8454456839597098,
      "grad_norm": 22.75,
      "learning_rate": 7.465176201241242e-06,
      "logits/chosen": -3.2611351013183594,
      "logits/rejected": -3.2482357025146484,
      "logps/chosen": -328.3022155761719,
      "logps/rejected": -387.12420654296875,
      "loss": 0.2433,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.647547721862793,
      "rewards/margins": 2.9851174354553223,
      "rewards/rejected": -8.632665634155273,
      "step": 8520
    },
    {
      "epoch": 1.846528755550742,
      "grad_norm": 28.875,
      "learning_rate": 7.458971063626621e-06,
      "logits/chosen": -3.312666654586792,
      "logits/rejected": -3.285332441329956,
      "logps/chosen": -349.6825866699219,
      "logps/rejected": -415.1434020996094,
      "loss": 0.2751,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -5.954230308532715,
      "rewards/margins": 3.1126608848571777,
      "rewards/rejected": -9.06689167022705,
      "step": 8525
    },
    {
      "epoch": 1.847611827141774,
      "grad_norm": 26.375,
      "learning_rate": 7.452768212320334e-06,
      "logits/chosen": -3.3913722038269043,
      "logits/rejected": -3.341240644454956,
      "logps/chosen": -343.1981201171875,
      "logps/rejected": -403.54998779296875,
      "loss": 0.174,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": -6.499428749084473,
      "rewards/margins": 3.2752785682678223,
      "rewards/rejected": -9.774707794189453,
      "step": 8530
    },
    {
      "epoch": 1.8486948987328062,
      "grad_norm": 23.375,
      "learning_rate": 7.446567656166387e-06,
      "logits/chosen": -3.291470766067505,
      "logits/rejected": -3.256563186645508,
      "logps/chosen": -349.83447265625,
      "logps/rejected": -413.26318359375,
      "loss": 0.2607,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -5.81522274017334,
      "rewards/margins": 3.2318758964538574,
      "rewards/rejected": -9.047097206115723,
      "step": 8535
    },
    {
      "epoch": 1.8497779703238384,
      "grad_norm": 29.5,
      "learning_rate": 7.440369404005528e-06,
      "logits/chosen": -3.192824125289917,
      "logits/rejected": -3.1535656452178955,
      "logps/chosen": -358.31842041015625,
      "logps/rejected": -405.1932678222656,
      "loss": 0.2722,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.099414825439453,
      "rewards/margins": 2.743259906768799,
      "rewards/rejected": -8.842674255371094,
      "step": 8540
    },
    {
      "epoch": 1.8508610419148706,
      "grad_norm": 16.375,
      "learning_rate": 7.434173464675209e-06,
      "logits/chosen": -3.4115543365478516,
      "logits/rejected": -3.3328144550323486,
      "logps/chosen": -309.17059326171875,
      "logps/rejected": -392.8601989746094,
      "loss": 0.2046,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.426000118255615,
      "rewards/margins": 3.5711236000061035,
      "rewards/rejected": -8.997122764587402,
      "step": 8545
    },
    {
      "epoch": 1.8519441135059027,
      "grad_norm": 25.5,
      "learning_rate": 7.4279798470095885e-06,
      "logits/chosen": -3.251798629760742,
      "logits/rejected": -3.34538197517395,
      "logps/chosen": -328.4315185546875,
      "logps/rejected": -382.08856201171875,
      "loss": 0.2527,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.616824626922607,
      "rewards/margins": 3.2807655334472656,
      "rewards/rejected": -8.897590637207031,
      "step": 8550
    },
    {
      "epoch": 1.8530271850969349,
      "grad_norm": 19.625,
      "learning_rate": 7.421788559839513e-06,
      "logits/chosen": -3.377178192138672,
      "logits/rejected": -3.35271954536438,
      "logps/chosen": -358.07427978515625,
      "logps/rejected": -406.70635986328125,
      "loss": 0.1773,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -6.054493427276611,
      "rewards/margins": 3.363922595977783,
      "rewards/rejected": -9.418416023254395,
      "step": 8555
    },
    {
      "epoch": 1.854110256687967,
      "grad_norm": 24.75,
      "learning_rate": 7.41559961199251e-06,
      "logits/chosen": -3.368866443634033,
      "logits/rejected": -3.2759056091308594,
      "logps/chosen": -353.2752380371094,
      "logps/rejected": -424.2177734375,
      "loss": 0.2411,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.069211483001709,
      "rewards/margins": 2.997023820877075,
      "rewards/rejected": -9.066235542297363,
      "step": 8560
    },
    {
      "epoch": 1.8551933282789992,
      "grad_norm": 23.625,
      "learning_rate": 7.409413012292767e-06,
      "logits/chosen": -3.3884849548339844,
      "logits/rejected": -3.411658525466919,
      "logps/chosen": -330.5668640136719,
      "logps/rejected": -387.55767822265625,
      "loss": 0.2396,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.550462245941162,
      "rewards/margins": 3.649538516998291,
      "rewards/rejected": -9.200000762939453,
      "step": 8565
    },
    {
      "epoch": 1.8562763998700316,
      "grad_norm": 22.25,
      "learning_rate": 7.403228769561126e-06,
      "logits/chosen": -3.3519062995910645,
      "logits/rejected": -3.339223861694336,
      "logps/chosen": -354.9851989746094,
      "logps/rejected": -420.2920837402344,
      "loss": 0.214,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.64099645614624,
      "rewards/margins": 3.326810836791992,
      "rewards/rejected": -9.96780776977539,
      "step": 8570
    },
    {
      "epoch": 1.8573594714610637,
      "grad_norm": 15.5,
      "learning_rate": 7.3970468926150644e-06,
      "logits/chosen": -3.299257755279541,
      "logits/rejected": -3.3092968463897705,
      "logps/chosen": -349.63299560546875,
      "logps/rejected": -411.86834716796875,
      "loss": 0.2002,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.67000675201416,
      "rewards/margins": 3.561779737472534,
      "rewards/rejected": -9.231786727905273,
      "step": 8575
    },
    {
      "epoch": 1.8584425430520959,
      "grad_norm": 13.3125,
      "learning_rate": 7.390867390268693e-06,
      "logits/chosen": -3.4215450286865234,
      "logits/rejected": -3.4492926597595215,
      "logps/chosen": -349.3744201660156,
      "logps/rejected": -416.31646728515625,
      "loss": 0.2066,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.67172384262085,
      "rewards/margins": 3.308295488357544,
      "rewards/rejected": -9.980019569396973,
      "step": 8580
    },
    {
      "epoch": 1.859525614643128,
      "grad_norm": 17.0,
      "learning_rate": 7.384690271332737e-06,
      "logits/chosen": -3.405907392501831,
      "logits/rejected": -3.345986843109131,
      "logps/chosen": -334.7738952636719,
      "logps/rejected": -405.4939880371094,
      "loss": 0.2215,
      "rewards/accuracies": 0.9249998927116394,
      "rewards/chosen": -6.125035285949707,
      "rewards/margins": 3.273836851119995,
      "rewards/rejected": -9.398872375488281,
      "step": 8585
    },
    {
      "epoch": 1.8606086862341602,
      "grad_norm": 14.4375,
      "learning_rate": 7.378515544614514e-06,
      "logits/chosen": -3.2382335662841797,
      "logits/rejected": -3.255354642868042,
      "logps/chosen": -373.0169677734375,
      "logps/rejected": -417.9453125,
      "loss": 0.2995,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.670307159423828,
      "rewards/margins": 2.6871166229248047,
      "rewards/rejected": -9.357423782348633,
      "step": 8590
    },
    {
      "epoch": 1.8616917578251924,
      "grad_norm": 30.125,
      "learning_rate": 7.37234321891794e-06,
      "logits/chosen": -3.4510536193847656,
      "logits/rejected": -3.435206174850464,
      "logps/chosen": -343.9111328125,
      "logps/rejected": -406.78973388671875,
      "loss": 0.2186,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.775975704193115,
      "rewards/margins": 3.3861002922058105,
      "rewards/rejected": -10.162075996398926,
      "step": 8595
    },
    {
      "epoch": 1.8627748294162245,
      "grad_norm": 19.125,
      "learning_rate": 7.366173303043506e-06,
      "logits/chosen": -3.4066131114959717,
      "logits/rejected": -3.300393581390381,
      "logps/chosen": -338.8089904785156,
      "logps/rejected": -398.514404296875,
      "loss": 0.2562,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.323124885559082,
      "rewards/margins": 3.0595273971557617,
      "rewards/rejected": -9.382651329040527,
      "step": 8600
    },
    {
      "epoch": 1.8638579010072567,
      "grad_norm": 22.5,
      "learning_rate": 7.360005805788264e-06,
      "logits/chosen": -3.226029634475708,
      "logits/rejected": -3.2514007091522217,
      "logps/chosen": -363.84478759765625,
      "logps/rejected": -418.47607421875,
      "loss": 0.2523,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -5.764934062957764,
      "rewards/margins": 3.072857141494751,
      "rewards/rejected": -8.837791442871094,
      "step": 8605
    },
    {
      "epoch": 1.8649409725982888,
      "grad_norm": 35.5,
      "learning_rate": 7.35384073594582e-06,
      "logits/chosen": -3.2689952850341797,
      "logits/rejected": -3.341552734375,
      "logps/chosen": -358.86724853515625,
      "logps/rejected": -407.82086181640625,
      "loss": 0.2649,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.100183010101318,
      "rewards/margins": 2.9232892990112305,
      "rewards/rejected": -9.02347183227539,
      "step": 8610
    },
    {
      "epoch": 1.866024044189321,
      "grad_norm": 18.5,
      "learning_rate": 7.347678102306314e-06,
      "logits/chosen": -3.23673939704895,
      "logits/rejected": -3.166318416595459,
      "logps/chosen": -358.3095703125,
      "logps/rejected": -427.2977600097656,
      "loss": 0.2074,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.051246643066406,
      "rewards/margins": 3.5376224517822266,
      "rewards/rejected": -9.588869094848633,
      "step": 8615
    },
    {
      "epoch": 1.8671071157803532,
      "grad_norm": 21.5,
      "learning_rate": 7.341517913656423e-06,
      "logits/chosen": -3.4030661582946777,
      "logits/rejected": -3.4228103160858154,
      "logps/chosen": -359.97039794921875,
      "logps/rejected": -428.4637756347656,
      "loss": 0.2132,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.984030723571777,
      "rewards/margins": 3.1482338905334473,
      "rewards/rejected": -10.132265090942383,
      "step": 8620
    },
    {
      "epoch": 1.8681901873713853,
      "grad_norm": 18.75,
      "learning_rate": 7.335360178779326e-06,
      "logits/chosen": -3.3001036643981934,
      "logits/rejected": -3.356973171234131,
      "logps/chosen": -343.0672302246094,
      "logps/rejected": -404.0873107910156,
      "loss": 0.2351,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.489577293395996,
      "rewards/margins": 3.3207411766052246,
      "rewards/rejected": -9.810319900512695,
      "step": 8625
    },
    {
      "epoch": 1.8692732589624175,
      "grad_norm": 22.375,
      "learning_rate": 7.329204906454714e-06,
      "logits/chosen": -3.270475387573242,
      "logits/rejected": -3.264427900314331,
      "logps/chosen": -345.18304443359375,
      "logps/rejected": -395.85302734375,
      "loss": 0.2782,
      "rewards/accuracies": 0.8500000834465027,
      "rewards/chosen": -5.798720836639404,
      "rewards/margins": 3.188896894454956,
      "rewards/rejected": -8.987618446350098,
      "step": 8630
    },
    {
      "epoch": 1.8703563305534496,
      "grad_norm": 31.875,
      "learning_rate": 7.323052105458758e-06,
      "logits/chosen": -3.520235061645508,
      "logits/rejected": -3.4391422271728516,
      "logps/chosen": -330.6936950683594,
      "logps/rejected": -382.26422119140625,
      "loss": 0.3597,
      "rewards/accuracies": 0.8166667222976685,
      "rewards/chosen": -6.478018760681152,
      "rewards/margins": 2.638272523880005,
      "rewards/rejected": -9.116291999816895,
      "step": 8635
    },
    {
      "epoch": 1.8714394021444818,
      "grad_norm": 29.0,
      "learning_rate": 7.316901784564111e-06,
      "logits/chosen": -3.2915573120117188,
      "logits/rejected": -3.2156548500061035,
      "logps/chosen": -355.40313720703125,
      "logps/rejected": -425.07733154296875,
      "loss": 0.2463,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.67458438873291,
      "rewards/margins": 3.630007266998291,
      "rewards/rejected": -10.30459213256836,
      "step": 8640
    },
    {
      "epoch": 1.872522473735514,
      "grad_norm": 24.25,
      "learning_rate": 7.31075395253989e-06,
      "logits/chosen": -3.3388397693634033,
      "logits/rejected": -3.2923035621643066,
      "logps/chosen": -344.8168029785156,
      "logps/rejected": -408.442138671875,
      "loss": 0.2321,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.238162040710449,
      "rewards/margins": 3.2427773475646973,
      "rewards/rejected": -9.480939865112305,
      "step": 8645
    },
    {
      "epoch": 1.873605545326546,
      "grad_norm": 31.375,
      "learning_rate": 7.3046086181516545e-06,
      "logits/chosen": -3.3484439849853516,
      "logits/rejected": -3.34442400932312,
      "logps/chosen": -338.3299560546875,
      "logps/rejected": -403.41680908203125,
      "loss": 0.2506,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.1253981590271,
      "rewards/margins": 3.3937950134277344,
      "rewards/rejected": -9.519193649291992,
      "step": 8650
    },
    {
      "epoch": 1.8746886169175783,
      "grad_norm": 21.375,
      "learning_rate": 7.2984657901614185e-06,
      "logits/chosen": -3.3763623237609863,
      "logits/rejected": -3.2324485778808594,
      "logps/chosen": -337.80633544921875,
      "logps/rejected": -412.7088317871094,
      "loss": 0.2235,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -5.796786785125732,
      "rewards/margins": 3.4266319274902344,
      "rewards/rejected": -9.223418235778809,
      "step": 8655
    },
    {
      "epoch": 1.8757716885086104,
      "grad_norm": 14.5625,
      "learning_rate": 7.292325477327612e-06,
      "logits/chosen": -3.3705451488494873,
      "logits/rejected": -3.3510963916778564,
      "logps/chosen": -346.36724853515625,
      "logps/rejected": -415.35980224609375,
      "loss": 0.2117,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.920327663421631,
      "rewards/margins": 3.331989288330078,
      "rewards/rejected": -9.252317428588867,
      "step": 8660
    },
    {
      "epoch": 1.8768547600996426,
      "grad_norm": 18.0,
      "learning_rate": 7.286187688405075e-06,
      "logits/chosen": -3.3810341358184814,
      "logits/rejected": -3.4211490154266357,
      "logps/chosen": -356.5395812988281,
      "logps/rejected": -434.8553771972656,
      "loss": 0.1688,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -6.5734758377075195,
      "rewards/margins": 3.8102707862854004,
      "rewards/rejected": -10.383747100830078,
      "step": 8665
    },
    {
      "epoch": 1.8779378316906747,
      "grad_norm": 20.25,
      "learning_rate": 7.2800524321450605e-06,
      "logits/chosen": -3.454333543777466,
      "logits/rejected": -3.3922793865203857,
      "logps/chosen": -348.1794738769531,
      "logps/rejected": -408.35662841796875,
      "loss": 0.2008,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.516558647155762,
      "rewards/margins": 3.386030912399292,
      "rewards/rejected": -9.902588844299316,
      "step": 8670
    },
    {
      "epoch": 1.879020903281707,
      "grad_norm": 26.375,
      "learning_rate": 7.273919717295201e-06,
      "logits/chosen": -3.4041075706481934,
      "logits/rejected": -3.3979105949401855,
      "logps/chosen": -321.9855041503906,
      "logps/rejected": -386.8062438964844,
      "loss": 0.2374,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -5.68876838684082,
      "rewards/margins": 3.3163561820983887,
      "rewards/rejected": -9.005125045776367,
      "step": 8675
    },
    {
      "epoch": 1.880103974872739,
      "grad_norm": 22.375,
      "learning_rate": 7.267789552599514e-06,
      "logits/chosen": -3.3668079376220703,
      "logits/rejected": -3.275531768798828,
      "logps/chosen": -335.0922546386719,
      "logps/rejected": -409.291015625,
      "loss": 0.2175,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.085090637207031,
      "rewards/margins": 3.391735792160034,
      "rewards/rejected": -9.476827621459961,
      "step": 8680
    },
    {
      "epoch": 1.8811870464637712,
      "grad_norm": 23.25,
      "learning_rate": 7.2616619467983685e-06,
      "logits/chosen": -3.3972487449645996,
      "logits/rejected": -3.30119252204895,
      "logps/chosen": -335.7479248046875,
      "logps/rejected": -410.6044006347656,
      "loss": 0.2271,
      "rewards/accuracies": 0.9166666269302368,
      "rewards/chosen": -6.0614399909973145,
      "rewards/margins": 3.2549757957458496,
      "rewards/rejected": -9.316415786743164,
      "step": 8685
    },
    {
      "epoch": 1.8822701180548034,
      "grad_norm": 20.25,
      "learning_rate": 7.255536908628498e-06,
      "logits/chosen": -3.3639004230499268,
      "logits/rejected": -3.2545089721679688,
      "logps/chosen": -364.81829833984375,
      "logps/rejected": -429.8443908691406,
      "loss": 0.2928,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -6.808465480804443,
      "rewards/margins": 2.628030776977539,
      "rewards/rejected": -9.436495780944824,
      "step": 8690
    },
    {
      "epoch": 1.8833531896458355,
      "grad_norm": 21.375,
      "learning_rate": 7.249414446822967e-06,
      "logits/chosen": -3.4282913208007812,
      "logits/rejected": -3.4313805103302,
      "logps/chosen": -357.87359619140625,
      "logps/rejected": -404.58587646484375,
      "loss": 0.2597,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.983065605163574,
      "rewards/margins": 3.274754762649536,
      "rewards/rejected": -9.257821083068848,
      "step": 8695
    },
    {
      "epoch": 1.8844362612368677,
      "grad_norm": 12.375,
      "learning_rate": 7.2432945701111716e-06,
      "logits/chosen": -3.262350082397461,
      "logits/rejected": -3.2291622161865234,
      "logps/chosen": -357.1697998046875,
      "logps/rejected": -417.2353515625,
      "loss": 0.2619,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -5.810829162597656,
      "rewards/margins": 3.269512891769409,
      "rewards/rejected": -9.080342292785645,
      "step": 8700
    },
    {
      "epoch": 1.8855193328278999,
      "grad_norm": 20.0,
      "learning_rate": 7.2371772872188185e-06,
      "logits/chosen": -3.3833718299865723,
      "logits/rejected": -3.286325454711914,
      "logps/chosen": -343.74517822265625,
      "logps/rejected": -407.7583312988281,
      "loss": 0.2029,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.102963447570801,
      "rewards/margins": 3.4825336933135986,
      "rewards/rejected": -9.585495948791504,
      "step": 8705
    },
    {
      "epoch": 1.886602404418932,
      "grad_norm": 29.375,
      "learning_rate": 7.2310626068679184e-06,
      "logits/chosen": -3.160942792892456,
      "logits/rejected": -3.230520248413086,
      "logps/chosen": -344.7365417480469,
      "logps/rejected": -396.45562744140625,
      "loss": 0.2974,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": -5.586522102355957,
      "rewards/margins": 3.2040741443634033,
      "rewards/rejected": -8.790596961975098,
      "step": 8710
    },
    {
      "epoch": 1.8876854760099642,
      "grad_norm": 24.625,
      "learning_rate": 7.224950537776767e-06,
      "logits/chosen": -3.460270643234253,
      "logits/rejected": -3.361942768096924,
      "logps/chosen": -342.00848388671875,
      "logps/rejected": -416.237060546875,
      "loss": 0.2278,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.116549015045166,
      "rewards/margins": 3.559352159500122,
      "rewards/rejected": -9.675901412963867,
      "step": 8715
    },
    {
      "epoch": 1.8887685476009963,
      "grad_norm": 22.75,
      "learning_rate": 7.218841088659944e-06,
      "logits/chosen": -3.2822957038879395,
      "logits/rejected": -3.1693410873413086,
      "logps/chosen": -342.24090576171875,
      "logps/rejected": -402.0406494140625,
      "loss": 0.2704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.260922908782959,
      "rewards/margins": 3.1163039207458496,
      "rewards/rejected": -9.377227783203125,
      "step": 8720
    },
    {
      "epoch": 1.8898516191920285,
      "grad_norm": 29.75,
      "learning_rate": 7.2127342682282845e-06,
      "logits/chosen": -3.263798475265503,
      "logits/rejected": -3.1349475383758545,
      "logps/chosen": -336.42156982421875,
      "logps/rejected": -391.9694519042969,
      "loss": 0.2617,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -5.669588088989258,
      "rewards/margins": 2.947465419769287,
      "rewards/rejected": -8.617053031921387,
      "step": 8725
    },
    {
      "epoch": 1.8909346907830606,
      "grad_norm": 16.75,
      "learning_rate": 7.206630085188887e-06,
      "logits/chosen": -3.1857616901397705,
      "logits/rejected": -3.191707134246826,
      "logps/chosen": -341.25286865234375,
      "logps/rejected": -408.10723876953125,
      "loss": 0.2225,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.319600582122803,
      "rewards/margins": 3.611260175704956,
      "rewards/rejected": -8.93086051940918,
      "step": 8730
    },
    {
      "epoch": 1.8920177623740928,
      "grad_norm": 28.0,
      "learning_rate": 7.200528548245079e-06,
      "logits/chosen": -3.2516396045684814,
      "logits/rejected": -3.290627956390381,
      "logps/chosen": -350.45758056640625,
      "logps/rejected": -399.7490234375,
      "loss": 0.2736,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.066018581390381,
      "rewards/margins": 3.0109877586364746,
      "rewards/rejected": -9.077006340026855,
      "step": 8735
    },
    {
      "epoch": 1.893100833965125,
      "grad_norm": 25.25,
      "learning_rate": 7.194429666096424e-06,
      "logits/chosen": -3.2009143829345703,
      "logits/rejected": -3.336660385131836,
      "logps/chosen": -368.97882080078125,
      "logps/rejected": -402.2476501464844,
      "loss": 0.2441,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.459216117858887,
      "rewards/margins": 3.100982427597046,
      "rewards/rejected": -9.560199737548828,
      "step": 8740
    },
    {
      "epoch": 1.8941839055561571,
      "grad_norm": 37.0,
      "learning_rate": 7.18833344743869e-06,
      "logits/chosen": -3.318235397338867,
      "logits/rejected": -3.267129421234131,
      "logps/chosen": -343.0904235839844,
      "logps/rejected": -392.9460144042969,
      "loss": 0.2775,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.577570915222168,
      "rewards/margins": 2.9857776165008545,
      "rewards/rejected": -9.563348770141602,
      "step": 8745
    },
    {
      "epoch": 1.8952669771471893,
      "grad_norm": 23.0,
      "learning_rate": 7.182239900963858e-06,
      "logits/chosen": -3.296118974685669,
      "logits/rejected": -3.32627534866333,
      "logps/chosen": -334.70172119140625,
      "logps/rejected": -402.4932556152344,
      "loss": 0.2292,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.343363285064697,
      "rewards/margins": 3.0938026905059814,
      "rewards/rejected": -9.437166213989258,
      "step": 8750
    },
    {
      "epoch": 1.8963500487382214,
      "grad_norm": 26.5,
      "learning_rate": 7.176149035360092e-06,
      "logits/chosen": -3.3232674598693848,
      "logits/rejected": -3.224539279937744,
      "logps/chosen": -352.8531188964844,
      "logps/rejected": -421.2019958496094,
      "loss": 0.2146,
      "rewards/accuracies": 0.9166668057441711,
      "rewards/chosen": -6.784933567047119,
      "rewards/margins": 3.441648006439209,
      "rewards/rejected": -10.226581573486328,
      "step": 8755
    },
    {
      "epoch": 1.8974331203292536,
      "grad_norm": 26.5,
      "learning_rate": 7.170060859311738e-06,
      "logits/chosen": -3.2189319133758545,
      "logits/rejected": -3.2945170402526855,
      "logps/chosen": -338.82611083984375,
      "logps/rejected": -410.42218017578125,
      "loss": 0.1984,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.627255916595459,
      "rewards/margins": 3.5458824634552,
      "rewards/rejected": -9.173138618469238,
      "step": 8760
    },
    {
      "epoch": 1.898516191920286,
      "grad_norm": 19.625,
      "learning_rate": 7.1639753814993e-06,
      "logits/chosen": -3.2097697257995605,
      "logits/rejected": -3.1551413536071777,
      "logps/chosen": -371.05352783203125,
      "logps/rejected": -440.92010498046875,
      "loss": 0.2461,
      "rewards/accuracies": 0.908333420753479,
      "rewards/chosen": -6.546185493469238,
      "rewards/margins": 3.151034355163574,
      "rewards/rejected": -9.697218894958496,
      "step": 8765
    },
    {
      "epoch": 1.8995992635113181,
      "grad_norm": 24.125,
      "learning_rate": 7.1578926105994475e-06,
      "logits/chosen": -3.329343795776367,
      "logits/rejected": -3.3127713203430176,
      "logps/chosen": -339.8136901855469,
      "logps/rejected": -394.0303955078125,
      "loss": 0.2595,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.398226737976074,
      "rewards/margins": 2.870459794998169,
      "rewards/rejected": -9.268686294555664,
      "step": 8770
    },
    {
      "epoch": 1.9006823351023503,
      "grad_norm": 29.0,
      "learning_rate": 7.151812555284978e-06,
      "logits/chosen": -3.3066864013671875,
      "logits/rejected": -3.3381028175354004,
      "logps/chosen": -355.5675354003906,
      "logps/rejected": -394.8721008300781,
      "loss": 0.2742,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.283730983734131,
      "rewards/margins": 2.8737387657165527,
      "rewards/rejected": -9.157470703125,
      "step": 8775
    },
    {
      "epoch": 1.9017654066933825,
      "grad_norm": 34.25,
      "learning_rate": 7.1457352242248245e-06,
      "logits/chosen": -3.270624876022339,
      "logits/rejected": -3.381375789642334,
      "logps/chosen": -362.3157653808594,
      "logps/rejected": -400.5947265625,
      "loss": 0.2678,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.233858108520508,
      "rewards/margins": 3.2176616191864014,
      "rewards/rejected": -9.451519012451172,
      "step": 8780
    },
    {
      "epoch": 1.9028484782844146,
      "grad_norm": 23.5,
      "learning_rate": 7.1396606260840304e-06,
      "logits/chosen": -3.206200122833252,
      "logits/rejected": -3.254406690597534,
      "logps/chosen": -354.40045166015625,
      "logps/rejected": -409.0272521972656,
      "loss": 0.2223,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.014608383178711,
      "rewards/margins": 2.960808753967285,
      "rewards/rejected": -8.975417137145996,
      "step": 8785
    },
    {
      "epoch": 1.9039315498754468,
      "grad_norm": 34.25,
      "learning_rate": 7.13358876952375e-06,
      "logits/chosen": -3.245246171951294,
      "logits/rejected": -3.2490546703338623,
      "logps/chosen": -362.9050598144531,
      "logps/rejected": -426.8636169433594,
      "loss": 0.2429,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.712143898010254,
      "rewards/margins": 3.2670276165008545,
      "rewards/rejected": -9.979169845581055,
      "step": 8790
    },
    {
      "epoch": 1.905014621466479,
      "grad_norm": 20.75,
      "learning_rate": 7.127519663201218e-06,
      "logits/chosen": -3.331402540206909,
      "logits/rejected": -3.3712589740753174,
      "logps/chosen": -346.3379821777344,
      "logps/rejected": -412.9286193847656,
      "loss": 0.2559,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.270571231842041,
      "rewards/margins": 3.333958864212036,
      "rewards/rejected": -9.60452938079834,
      "step": 8795
    },
    {
      "epoch": 1.906097693057511,
      "grad_norm": 23.625,
      "learning_rate": 7.1214533157697595e-06,
      "logits/chosen": -3.3140320777893066,
      "logits/rejected": -3.30957293510437,
      "logps/chosen": -347.323974609375,
      "logps/rejected": -394.8592224121094,
      "loss": 0.2672,
      "rewards/accuracies": 0.8583332896232605,
      "rewards/chosen": -6.262240409851074,
      "rewards/margins": 2.928215503692627,
      "rewards/rejected": -9.190457344055176,
      "step": 8800
    },
    {
      "epoch": 1.9071807646485432,
      "grad_norm": 20.5,
      "learning_rate": 7.115389735878759e-06,
      "logits/chosen": -3.2979514598846436,
      "logits/rejected": -3.2346904277801514,
      "logps/chosen": -329.4864196777344,
      "logps/rejected": -387.16351318359375,
      "loss": 0.2856,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.207014560699463,
      "rewards/margins": 3.0801773071289062,
      "rewards/rejected": -8.287191390991211,
      "step": 8805
    },
    {
      "epoch": 1.9082638362395754,
      "grad_norm": 21.0,
      "learning_rate": 7.109328932173656e-06,
      "logits/chosen": -3.211568832397461,
      "logits/rejected": -3.380603313446045,
      "logps/chosen": -335.6992492675781,
      "logps/rejected": -381.88311767578125,
      "loss": 0.2392,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.974039077758789,
      "rewards/margins": 2.9274539947509766,
      "rewards/rejected": -8.901494026184082,
      "step": 8810
    },
    {
      "epoch": 1.9093469078306076,
      "grad_norm": 21.0,
      "learning_rate": 7.103270913295932e-06,
      "logits/chosen": -3.195197582244873,
      "logits/rejected": -3.1833508014678955,
      "logps/chosen": -364.83697509765625,
      "logps/rejected": -418.63250732421875,
      "loss": 0.229,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.234593391418457,
      "rewards/margins": 3.0600814819335938,
      "rewards/rejected": -9.29467487335205,
      "step": 8815
    },
    {
      "epoch": 1.9104299794216397,
      "grad_norm": 22.625,
      "learning_rate": 7.0972156878831e-06,
      "logits/chosen": -3.172122001647949,
      "logits/rejected": -3.246678113937378,
      "logps/chosen": -346.83221435546875,
      "logps/rejected": -403.2777404785156,
      "loss": 0.2422,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -5.498873710632324,
      "rewards/margins": 3.255131959915161,
      "rewards/rejected": -8.754005432128906,
      "step": 8820
    },
    {
      "epoch": 1.9115130510126719,
      "grad_norm": 17.75,
      "learning_rate": 7.091163264568684e-06,
      "logits/chosen": -3.3346285820007324,
      "logits/rejected": -3.333953857421875,
      "logps/chosen": -336.5292053222656,
      "logps/rejected": -397.0751953125,
      "loss": 0.2492,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.0963897705078125,
      "rewards/margins": 2.9248011112213135,
      "rewards/rejected": -9.021190643310547,
      "step": 8825
    },
    {
      "epoch": 1.9125961226037043,
      "grad_norm": 28.625,
      "learning_rate": 7.085113651982221e-06,
      "logits/chosen": -3.195631742477417,
      "logits/rejected": -3.2098522186279297,
      "logps/chosen": -335.9156494140625,
      "logps/rejected": -388.1016845703125,
      "loss": 0.2263,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.688387870788574,
      "rewards/margins": 2.8828694820404053,
      "rewards/rejected": -8.571257591247559,
      "step": 8830
    },
    {
      "epoch": 1.9136791941947364,
      "grad_norm": 25.625,
      "learning_rate": 7.079066858749233e-06,
      "logits/chosen": -3.1040730476379395,
      "logits/rejected": -3.1021275520324707,
      "logps/chosen": -357.5837097167969,
      "logps/rejected": -417.47650146484375,
      "loss": 0.2534,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.946324825286865,
      "rewards/margins": 3.4082798957824707,
      "rewards/rejected": -9.354604721069336,
      "step": 8835
    },
    {
      "epoch": 1.9147622657857686,
      "grad_norm": 17.875,
      "learning_rate": 7.07302289349123e-06,
      "logits/chosen": -3.1555416584014893,
      "logits/rejected": -3.2504191398620605,
      "logps/chosen": -343.1615295410156,
      "logps/rejected": -400.8316345214844,
      "loss": 0.2053,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.937642574310303,
      "rewards/margins": 3.333585262298584,
      "rewards/rejected": -9.271227836608887,
      "step": 8840
    },
    {
      "epoch": 1.9158453373768007,
      "grad_norm": 30.0,
      "learning_rate": 7.066981764825682e-06,
      "logits/chosen": -3.2143988609313965,
      "logits/rejected": -3.262260913848877,
      "logps/chosen": -327.9243469238281,
      "logps/rejected": -391.4708251953125,
      "loss": 0.2257,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -5.422950744628906,
      "rewards/margins": 3.4998836517333984,
      "rewards/rejected": -8.922834396362305,
      "step": 8845
    },
    {
      "epoch": 1.916928408967833,
      "grad_norm": 17.5,
      "learning_rate": 7.060943481366021e-06,
      "logits/chosen": -3.248425245285034,
      "logits/rejected": -3.2213141918182373,
      "logps/chosen": -351.53350830078125,
      "logps/rejected": -398.6817932128906,
      "loss": 0.2371,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -5.948168754577637,
      "rewards/margins": 3.094804286956787,
      "rewards/rejected": -9.042973518371582,
      "step": 8850
    },
    {
      "epoch": 1.918011480558865,
      "grad_norm": 27.75,
      "learning_rate": 7.054908051721617e-06,
      "logits/chosen": -3.199326753616333,
      "logits/rejected": -3.3558926582336426,
      "logps/chosen": -347.35784912109375,
      "logps/rejected": -395.2776794433594,
      "loss": 0.2971,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.928880214691162,
      "rewards/margins": 3.2548553943634033,
      "rewards/rejected": -9.183735847473145,
      "step": 8855
    },
    {
      "epoch": 1.9190945521498972,
      "grad_norm": 22.5,
      "learning_rate": 7.048875484497775e-06,
      "logits/chosen": -3.2928688526153564,
      "logits/rejected": -3.219524383544922,
      "logps/chosen": -346.63592529296875,
      "logps/rejected": -410.473388671875,
      "loss": 0.2025,
      "rewards/accuracies": 0.9250000715255737,
      "rewards/chosen": -6.037154197692871,
      "rewards/margins": 3.437025785446167,
      "rewards/rejected": -9.474180221557617,
      "step": 8860
    },
    {
      "epoch": 1.9201776237409294,
      "grad_norm": 16.625,
      "learning_rate": 7.042845788295715e-06,
      "logits/chosen": -3.3173060417175293,
      "logits/rejected": -3.172696352005005,
      "logps/chosen": -351.21917724609375,
      "logps/rejected": -419.3103942871094,
      "loss": 0.2051,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.6024346351623535,
      "rewards/margins": 3.2104735374450684,
      "rewards/rejected": -9.812908172607422,
      "step": 8865
    },
    {
      "epoch": 1.9212606953319615,
      "grad_norm": 37.25,
      "learning_rate": 7.036818971712567e-06,
      "logits/chosen": -3.2245147228240967,
      "logits/rejected": -3.3239617347717285,
      "logps/chosen": -351.32342529296875,
      "logps/rejected": -397.97528076171875,
      "loss": 0.2495,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.37738037109375,
      "rewards/margins": 3.035543441772461,
      "rewards/rejected": -9.412923812866211,
      "step": 8870
    },
    {
      "epoch": 1.9223437669229937,
      "grad_norm": 17.625,
      "learning_rate": 7.030795043341353e-06,
      "logits/chosen": -3.442185878753662,
      "logits/rejected": -3.3300557136535645,
      "logps/chosen": -348.36029052734375,
      "logps/rejected": -420.55145263671875,
      "loss": 0.2234,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.388441562652588,
      "rewards/margins": 3.495702028274536,
      "rewards/rejected": -9.884142875671387,
      "step": 8875
    },
    {
      "epoch": 1.9234268385140258,
      "grad_norm": 34.5,
      "learning_rate": 7.024774011770981e-06,
      "logits/chosen": -3.3168673515319824,
      "logits/rejected": -3.38191294670105,
      "logps/chosen": -355.7350158691406,
      "logps/rejected": -413.8560485839844,
      "loss": 0.2284,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.605541229248047,
      "rewards/margins": 3.3834872245788574,
      "rewards/rejected": -9.98902702331543,
      "step": 8880
    },
    {
      "epoch": 1.924509910105058,
      "grad_norm": 16.625,
      "learning_rate": 7.018755885586223e-06,
      "logits/chosen": -3.312659502029419,
      "logits/rejected": -3.1908249855041504,
      "logps/chosen": -326.57391357421875,
      "logps/rejected": -406.55657958984375,
      "loss": 0.1668,
      "rewards/accuracies": 0.9416666030883789,
      "rewards/chosen": -5.700817108154297,
      "rewards/margins": 3.455840587615967,
      "rewards/rejected": -9.156658172607422,
      "step": 8885
    },
    {
      "epoch": 1.9255929816960902,
      "grad_norm": 11.5625,
      "learning_rate": 7.0127406733677125e-06,
      "logits/chosen": -3.4414029121398926,
      "logits/rejected": -3.3787758350372314,
      "logps/chosen": -330.6535949707031,
      "logps/rejected": -409.5632629394531,
      "loss": 0.2065,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.1177287101745605,
      "rewards/margins": 3.6999707221984863,
      "rewards/rejected": -9.817699432373047,
      "step": 8890
    },
    {
      "epoch": 1.9266760532871223,
      "grad_norm": 27.5,
      "learning_rate": 7.0067283836919255e-06,
      "logits/chosen": -3.306638717651367,
      "logits/rejected": -3.3965847492218018,
      "logps/chosen": -344.72509765625,
      "logps/rejected": -396.96917724609375,
      "loss": 0.2265,
      "rewards/accuracies": 0.8999999165534973,
      "rewards/chosen": -6.611647129058838,
      "rewards/margins": 3.140653371810913,
      "rewards/rejected": -9.752301216125488,
      "step": 8895
    },
    {
      "epoch": 1.9277591248781545,
      "grad_norm": 18.625,
      "learning_rate": 7.000719025131174e-06,
      "logits/chosen": -3.3892159461975098,
      "logits/rejected": -3.3672211170196533,
      "logps/chosen": -332.5964050292969,
      "logps/rejected": -396.6457214355469,
      "loss": 0.2349,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.426122188568115,
      "rewards/margins": 3.1747944355010986,
      "rewards/rejected": -9.600915908813477,
      "step": 8900
    },
    {
      "epoch": 1.9288421964691866,
      "grad_norm": 30.375,
      "learning_rate": 6.994712606253589e-06,
      "logits/chosen": -3.4173507690429688,
      "logits/rejected": -3.273693084716797,
      "logps/chosen": -334.6750183105469,
      "logps/rejected": -397.3933410644531,
      "loss": 0.2609,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.491758823394775,
      "rewards/margins": 2.8257856369018555,
      "rewards/rejected": -9.317544937133789,
      "step": 8905
    },
    {
      "epoch": 1.9299252680602188,
      "grad_norm": 38.5,
      "learning_rate": 6.988709135623108e-06,
      "logits/chosen": -3.3192036151885986,
      "logits/rejected": -3.273937225341797,
      "logps/chosen": -351.8692626953125,
      "logps/rejected": -428.71484375,
      "loss": 0.2049,
      "rewards/accuracies": 0.9166666865348816,
      "rewards/chosen": -6.589383125305176,
      "rewards/margins": 3.437727689743042,
      "rewards/rejected": -10.027112007141113,
      "step": 8910
    },
    {
      "epoch": 1.931008339651251,
      "grad_norm": 25.5,
      "learning_rate": 6.982708621799471e-06,
      "logits/chosen": -3.4721221923828125,
      "logits/rejected": -3.481093645095825,
      "logps/chosen": -344.1595153808594,
      "logps/rejected": -412.85577392578125,
      "loss": 0.2231,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.45871114730835,
      "rewards/margins": 3.6124191284179688,
      "rewards/rejected": -10.071130752563477,
      "step": 8915
    },
    {
      "epoch": 1.9320914112422831,
      "grad_norm": 24.375,
      "learning_rate": 6.9767110733381985e-06,
      "logits/chosen": -3.3505725860595703,
      "logits/rejected": -3.3228697776794434,
      "logps/chosen": -342.72406005859375,
      "logps/rejected": -414.16546630859375,
      "loss": 0.2442,
      "rewards/accuracies": 0.8833333253860474,
      "rewards/chosen": -6.47998046875,
      "rewards/margins": 3.295336961746216,
      "rewards/rejected": -9.77531623840332,
      "step": 8920
    },
    {
      "epoch": 1.9331744828333153,
      "grad_norm": 34.75,
      "learning_rate": 6.97071649879058e-06,
      "logits/chosen": -3.3736939430236816,
      "logits/rejected": -3.4343600273132324,
      "logps/chosen": -357.81591796875,
      "logps/rejected": -420.67333984375,
      "loss": 0.2107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.626135349273682,
      "rewards/margins": 3.4236176013946533,
      "rewards/rejected": -11.049753189086914,
      "step": 8925
    },
    {
      "epoch": 1.9342575544243474,
      "grad_norm": 44.5,
      "learning_rate": 6.964724906703672e-06,
      "logits/chosen": -3.346764087677002,
      "logits/rejected": -3.3047916889190674,
      "logps/chosen": -352.11700439453125,
      "logps/rejected": -409.0065002441406,
      "loss": 0.2708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.506606101989746,
      "rewards/margins": 3.123749017715454,
      "rewards/rejected": -9.630354881286621,
      "step": 8930
    },
    {
      "epoch": 1.9353406260153796,
      "grad_norm": 24.125,
      "learning_rate": 6.958736305620269e-06,
      "logits/chosen": -3.325054168701172,
      "logits/rejected": -3.2762997150421143,
      "logps/chosen": -356.5971374511719,
      "logps/rejected": -411.6806640625,
      "loss": 0.2585,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -6.683479309082031,
      "rewards/margins": 2.8370490074157715,
      "rewards/rejected": -9.520528793334961,
      "step": 8935
    },
    {
      "epoch": 1.9364236976064118,
      "grad_norm": 13.8125,
      "learning_rate": 6.95275070407891e-06,
      "logits/chosen": -3.3009307384490967,
      "logits/rejected": -3.344128370285034,
      "logps/chosen": -353.79705810546875,
      "logps/rejected": -412.97723388671875,
      "loss": 0.2361,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.358022212982178,
      "rewards/margins": 3.233198881149292,
      "rewards/rejected": -9.59122085571289,
      "step": 8940
    },
    {
      "epoch": 1.937506769197444,
      "grad_norm": 18.5,
      "learning_rate": 6.946768110613854e-06,
      "logits/chosen": -3.2715859413146973,
      "logits/rejected": -3.284182071685791,
      "logps/chosen": -354.6552429199219,
      "logps/rejected": -407.3271484375,
      "loss": 0.2647,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.129233360290527,
      "rewards/margins": 3.050043821334839,
      "rewards/rejected": -9.179277420043945,
      "step": 8945
    },
    {
      "epoch": 1.938589840788476,
      "grad_norm": 21.875,
      "learning_rate": 6.9407885337550675e-06,
      "logits/chosen": -3.285862445831299,
      "logits/rejected": -3.319409132003784,
      "logps/chosen": -366.3502197265625,
      "logps/rejected": -418.6676330566406,
      "loss": 0.2187,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.5114617347717285,
      "rewards/margins": 3.247201442718506,
      "rewards/rejected": -9.758662223815918,
      "step": 8950
    },
    {
      "epoch": 1.9396729123795082,
      "grad_norm": 30.0,
      "learning_rate": 6.934811982028223e-06,
      "logits/chosen": -3.3508458137512207,
      "logits/rejected": -3.4104442596435547,
      "logps/chosen": -360.04058837890625,
      "logps/rejected": -416.72723388671875,
      "loss": 0.2213,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": -6.820055961608887,
      "rewards/margins": 3.4223499298095703,
      "rewards/rejected": -10.242406845092773,
      "step": 8955
    },
    {
      "epoch": 1.9407559839705404,
      "grad_norm": 40.5,
      "learning_rate": 6.928838463954675e-06,
      "logits/chosen": -3.3504302501678467,
      "logits/rejected": -3.349043369293213,
      "logps/chosen": -346.51971435546875,
      "logps/rejected": -415.3297424316406,
      "loss": 0.3304,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -6.83132266998291,
      "rewards/margins": 2.915895700454712,
      "rewards/rejected": -9.747217178344727,
      "step": 8960
    },
    {
      "epoch": 1.9418390555615725,
      "grad_norm": 18.125,
      "learning_rate": 6.922867988051451e-06,
      "logits/chosen": -3.324352264404297,
      "logits/rejected": -3.2761809825897217,
      "logps/chosen": -354.5389099121094,
      "logps/rejected": -433.58245849609375,
      "loss": 0.2281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.1815876960754395,
      "rewards/margins": 3.74017071723938,
      "rewards/rejected": -9.921757698059082,
      "step": 8965
    },
    {
      "epoch": 1.9429221271526047,
      "grad_norm": 18.75,
      "learning_rate": 6.916900562831245e-06,
      "logits/chosen": -3.283080577850342,
      "logits/rejected": -3.2412383556365967,
      "logps/chosen": -346.3719482421875,
      "logps/rejected": -407.9366455078125,
      "loss": 0.281,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.669057369232178,
      "rewards/margins": 2.8907761573791504,
      "rewards/rejected": -9.559833526611328,
      "step": 8970
    },
    {
      "epoch": 1.9440051987436369,
      "grad_norm": 29.5,
      "learning_rate": 6.910936196802402e-06,
      "logits/chosen": -3.3990490436553955,
      "logits/rejected": -3.3902926445007324,
      "logps/chosen": -352.2237548828125,
      "logps/rejected": -414.412841796875,
      "loss": 0.2219,
      "rewards/accuracies": 0.9083333015441895,
      "rewards/chosen": -6.516835689544678,
      "rewards/margins": 3.2132315635681152,
      "rewards/rejected": -9.730067253112793,
      "step": 8975
    },
    {
      "epoch": 1.945088270334669,
      "grad_norm": 23.0,
      "learning_rate": 6.9049748984689e-06,
      "logits/chosen": -3.371826171875,
      "logits/rejected": -3.307084560394287,
      "logps/chosen": -345.0296630859375,
      "logps/rejected": -423.3773498535156,
      "loss": 0.2183,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.673425197601318,
      "rewards/margins": 3.4804768562316895,
      "rewards/rejected": -10.153902053833008,
      "step": 8980
    },
    {
      "epoch": 1.9461713419257012,
      "grad_norm": 9.0,
      "learning_rate": 6.8990166763303465e-06,
      "logits/chosen": -3.337848663330078,
      "logits/rejected": -3.4128482341766357,
      "logps/chosen": -358.75762939453125,
      "logps/rejected": -414.0059509277344,
      "loss": 0.2481,
      "rewards/accuracies": 0.8833333849906921,
      "rewards/chosen": -6.966660499572754,
      "rewards/margins": 3.0597167015075684,
      "rewards/rejected": -10.026376724243164,
      "step": 8985
    },
    {
      "epoch": 1.9472544135167333,
      "grad_norm": 35.75,
      "learning_rate": 6.8930615388819664e-06,
      "logits/chosen": -3.3936378955841064,
      "logits/rejected": -3.4413647651672363,
      "logps/chosen": -354.0198669433594,
      "logps/rejected": -396.90911865234375,
      "loss": 0.2734,
      "rewards/accuracies": 0.89166659116745,
      "rewards/chosen": -6.416226863861084,
      "rewards/margins": 2.5892763137817383,
      "rewards/rejected": -9.005502700805664,
      "step": 8990
    },
    {
      "epoch": 1.9483374851077655,
      "grad_norm": 16.375,
      "learning_rate": 6.887109494614578e-06,
      "logits/chosen": -3.422372341156006,
      "logits/rejected": -3.2778000831604004,
      "logps/chosen": -347.83013916015625,
      "logps/rejected": -430.87835693359375,
      "loss": 0.1908,
      "rewards/accuracies": 0.8916667103767395,
      "rewards/chosen": -6.8195390701293945,
      "rewards/margins": 3.6762893199920654,
      "rewards/rejected": -10.495827674865723,
      "step": 8995
    },
    {
      "epoch": 1.9494205566987977,
      "grad_norm": 27.0,
      "learning_rate": 6.881160552014596e-06,
      "logits/chosen": -3.257035732269287,
      "logits/rejected": -3.2726969718933105,
      "logps/chosen": -365.0843505859375,
      "logps/rejected": -418.9554138183594,
      "loss": 0.2914,
      "rewards/accuracies": 0.8416666984558105,
      "rewards/chosen": -6.6492767333984375,
      "rewards/margins": 3.1824049949645996,
      "rewards/rejected": -9.831680297851562,
      "step": 9000
    },
    {
      "epoch": 1.9494205566987977,
      "eval_logits/chosen": -3.4187393188476562,
      "eval_logits/rejected": -3.357347011566162,
      "eval_logps/chosen": -339.1562805175781,
      "eval_logps/rejected": -395.1826171875,
      "eval_loss": 0.3545292615890503,
      "eval_rewards/accuracies": 0.8199999928474426,
      "eval_rewards/chosen": -6.247750282287598,
      "eval_rewards/margins": 2.628523349761963,
      "eval_rewards/rejected": -8.876274108886719,
      "eval_runtime": 14.4185,
      "eval_samples_per_second": 13.871,
      "eval_steps_per_second": 13.871,
      "step": 9000
    }
  ],
  "logging_steps": 5,
  "max_steps": 13848,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 12,
  "trial_name": null,
  "trial_params": null
}