{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 101,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "entropy": 2.566085159778595,
      "epoch": 0.009950248756218905,
      "grad_norm": 1.008041501045227,
      "learning_rate": 0.0005,
      "logits/chosen": -3.4524951520300227,
      "logits/rejected": -3.4626118857625254,
      "logps/chosen": -147.66649627685547,
      "logps/rejected": -153.5500717163086,
      "loss": 0.6931471824645996,
      "mean_token_accuracy": 0.40839148312807083,
      "num_tokens": 3138.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "entropy": 1.959685891866684,
      "epoch": 0.01990049751243781,
      "grad_norm": 1.1677398681640625,
      "learning_rate": 0.0004950495049504951,
      "logits/chosen": -3.5499522275821747,
      "logits/rejected": -3.580069779457358,
      "logps/chosen": -139.49810218811035,
      "logps/rejected": -189.3823013305664,
      "loss": 0.46622467041015625,
      "mean_token_accuracy": 0.4368501454591751,
      "num_tokens": 6422.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06323989480733871,
      "rewards/margins": 0.9142910540103912,
      "rewards/rejected": -0.8510511815547943,
      "step": 2
    },
    {
      "entropy": 1.5055575668811798,
      "epoch": 0.029850746268656716,
      "grad_norm": 3.6899547576904297,
      "learning_rate": 0.0004900990099009901,
      "logits/chosen": -3.5475161144530585,
      "logits/rejected": -3.5951742990321627,
      "logps/chosen": -96.94854736328125,
      "logps/rejected": -122.58767223358154,
      "loss": 0.6513766050338745,
      "mean_token_accuracy": 0.45520826429128647,
      "num_tokens": 8997.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.8459513187408447,
      "rewards/margins": 1.247560515999794,
      "rewards/rejected": -2.0935118198394775,
      "step": 3
    },
    {
      "entropy": 1.557283103466034,
      "epoch": 0.03980099502487562,
      "grad_norm": 0.0978235974907875,
      "learning_rate": 0.00048514851485148515,
      "logits/chosen": -3.4936563969802714,
      "logits/rejected": -3.5099735853856826,
      "logps/chosen": -159.4117660522461,
      "logps/rejected": -184.10985946655273,
      "loss": 0.5663926005363464,
      "mean_token_accuracy": 0.4151092618703842,
      "num_tokens": 12160.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -1.603164166212082,
      "rewards/margins": 0.9312166273593903,
      "rewards/rejected": -2.534380793571472,
      "step": 4
    },
    {
      "entropy": 1.5153527557849884,
      "epoch": 0.04975124378109453,
      "grad_norm": 0.7491629719734192,
      "learning_rate": 0.0004801980198019802,
      "logits/chosen": -3.6313017387350977,
      "logits/rejected": -3.6386909825311573,
      "logps/chosen": -136.62432289123535,
      "logps/rejected": -162.2756748199463,
      "loss": 0.5463283061981201,
      "mean_token_accuracy": 0.4367467537522316,
      "num_tokens": 15284.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5465116687119007,
      "rewards/margins": 1.0278603285551071,
      "rewards/rejected": -1.5743719935417175,
      "step": 5
    },
    {
      "entropy": 1.4582557380199432,
      "epoch": 0.05970149253731343,
      "grad_norm": 0.7366572022438049,
      "learning_rate": 0.00047524752475247524,
      "logits/chosen": -3.5624190347055085,
      "logits/rejected": -3.5716472006137434,
      "logps/chosen": -151.5631103515625,
      "logps/rejected": -242.12555694580078,
      "loss": 0.4732470214366913,
      "mean_token_accuracy": 0.46974872797727585,
      "num_tokens": 18935.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0975639000535011,
      "rewards/margins": 1.8435385078191757,
      "rewards/rejected": -2.9411023259162903,
      "step": 6
    },
    {
      "entropy": 1.382529228925705,
      "epoch": 0.06965174129353234,
      "grad_norm": 2.1164298057556152,
      "learning_rate": 0.0004702970297029703,
      "logits/chosen": -3.5944972735203464,
      "logits/rejected": -3.5765365597770478,
      "logps/chosen": -151.23340702056885,
      "logps/rejected": -164.38666248321533,
      "loss": 0.5946930646896362,
      "mean_token_accuracy": 0.44173406809568405,
      "num_tokens": 21990.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -1.0693217813968658,
      "rewards/margins": 0.6622129492461681,
      "rewards/rejected": -1.7315346747636795,
      "step": 7
    },
    {
      "entropy": 1.5175149738788605,
      "epoch": 0.07960199004975124,
      "grad_norm": 1.9517371654510498,
      "learning_rate": 0.0004653465346534654,
      "logits/chosen": -3.5927788076349674,
      "logits/rejected": -3.584108080993631,
      "logps/chosen": -212.95052337646484,
      "logps/rejected": -229.85908699035645,
      "loss": 0.7136555910110474,
      "mean_token_accuracy": 0.471999429166317,
      "num_tokens": 26289.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -1.3788724839687347,
      "rewards/margins": 0.41184403374791145,
      "rewards/rejected": -1.7907164692878723,
      "step": 8
    },
    {
      "entropy": 1.6294221878051758,
      "epoch": 0.08955223880597014,
      "grad_norm": 5.569645881652832,
      "learning_rate": 0.0004603960396039604,
      "logits/chosen": -3.5307916447552246,
      "logits/rejected": -3.5973216295285475,
      "logps/chosen": -155.5925750732422,
      "logps/rejected": -159.6125602722168,
      "loss": 0.7543402910232544,
      "mean_token_accuracy": 0.4240734949707985,
      "num_tokens": 29297.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -1.32593235373497,
      "rewards/margins": 0.4513932764530182,
      "rewards/rejected": -1.7773255854845047,
      "step": 9
    },
    {
      "entropy": 1.6049813628196716,
      "epoch": 0.09950248756218906,
      "grad_norm": 2.697402000427246,
      "learning_rate": 0.00045544554455445547,
      "logits/chosen": -3.537555654849764,
      "logits/rejected": -3.55802165123894,
      "logps/chosen": -134.05085563659668,
      "logps/rejected": -144.08281135559082,
      "loss": 0.6336124539375305,
      "mean_token_accuracy": 0.41080383211374283,
      "num_tokens": 32149.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.8984666168689728,
      "rewards/margins": 0.2784314379096031,
      "rewards/rejected": -1.1768980920314789,
      "step": 10
    },
    {
      "entropy": 1.6381469070911407,
      "epoch": 0.10945273631840796,
      "grad_norm": Infinity,
      "learning_rate": 0.0004504950495049505,
      "logits/chosen": -3.5665754825972718,
      "logits/rejected": -3.4803866010674835,
      "logps/chosen": -151.97227668762207,
      "logps/rejected": -165.6332607269287,
      "loss": 0.7078717947006226,
      "mean_token_accuracy": 0.4435800090432167,
      "num_tokens": 35261.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.9129189550876617,
      "rewards/margins": 0.44023333117365837,
      "rewards/rejected": -1.3531522750854492,
      "step": 11
    },
    {
      "entropy": 1.5729329586029053,
      "epoch": 0.11940298507462686,
      "grad_norm": 1.798696756362915,
      "learning_rate": 0.00044554455445544556,
      "logits/chosen": -3.5680972100252037,
      "logits/rejected": -3.5636713320968143,
      "logps/chosen": -106.49790382385254,
      "logps/rejected": -130.71097564697266,
      "loss": 0.5336892604827881,
      "mean_token_accuracy": 0.433380089700222,
      "num_tokens": 37813.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.2621241584420204,
      "rewards/margins": 0.8355743233114481,
      "rewards/rejected": -1.0976984649896622,
      "step": 12
    },
    {
      "entropy": 1.5840542912483215,
      "epoch": 0.12935323383084577,
      "grad_norm": 3.1827797889709473,
      "learning_rate": 0.0004405940594059406,
      "logits/chosen": -3.5452509024889998,
      "logits/rejected": -3.530621630940143,
      "logps/chosen": -131.4068202972412,
      "logps/rejected": -145.15791511535645,
      "loss": 0.6936072111129761,
      "mean_token_accuracy": 0.4328671544790268,
      "num_tokens": 40883.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.9353857561945915,
      "rewards/margins": 0.30992990732192993,
      "rewards/rejected": -1.245315708220005,
      "step": 13
    },
    {
      "entropy": 1.6584091484546661,
      "epoch": 0.13930348258706468,
      "grad_norm": 2.5575995445251465,
      "learning_rate": 0.00043564356435643565,
      "logits/chosen": -3.571977708398152,
      "logits/rejected": -3.5559217437615476,
      "logps/chosen": -156.84790420532227,
      "logps/rejected": -214.96884536743164,
      "loss": 0.4404676556587219,
      "mean_token_accuracy": 0.43554289638996124,
      "num_tokens": 44189.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3778061270713806,
      "rewards/margins": 1.154350906610489,
      "rewards/rejected": -2.5321571230888367,
      "step": 14
    },
    {
      "entropy": 1.5668798983097076,
      "epoch": 0.14925373134328357,
      "grad_norm": 1.0555850267410278,
      "learning_rate": 0.0004306930693069307,
      "logits/chosen": -3.5467018552189504,
      "logits/rejected": -3.5639543923338195,
      "logps/chosen": -137.75764083862305,
      "logps/rejected": -212.9980926513672,
      "loss": 0.6105657815933228,
      "mean_token_accuracy": 0.3983445391058922,
      "num_tokens": 47240.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -2.1039093732833862,
      "rewards/margins": 0.9429394379258156,
      "rewards/rejected": -3.046848803758621,
      "step": 15
    },
    {
      "entropy": 1.525937557220459,
      "epoch": 0.15920398009950248,
      "grad_norm": 0.5652996897697449,
      "learning_rate": 0.00042574257425742574,
      "logits/chosen": -3.592827924032985,
      "logits/rejected": -3.56554933953146,
      "logps/chosen": -148.5086841583252,
      "logps/rejected": -158.31574058532715,
      "loss": 0.5453071594238281,
      "mean_token_accuracy": 0.41627876460552216,
      "num_tokens": 50021.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.3691547214984894,
      "rewards/margins": 0.7764396816492081,
      "rewards/rejected": -3.1455944180488586,
      "step": 16
    },
    {
      "entropy": 1.5125811696052551,
      "epoch": 0.1691542288557214,
      "grad_norm": 4.395788669586182,
      "learning_rate": 0.0004207920792079208,
      "logits/chosen": -3.6114905508093926,
      "logits/rejected": -3.6018918848109203,
      "logps/chosen": -211.8555679321289,
      "logps/rejected": -233.0864772796631,
      "loss": 0.6787844896316528,
      "mean_token_accuracy": 0.4354413077235222,
      "num_tokens": 53786.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -3.3658860325813293,
      "rewards/margins": 1.0309943780303001,
      "rewards/rejected": -4.3968804478645325,
      "step": 17
    },
    {
      "entropy": 1.493565171957016,
      "epoch": 0.1791044776119403,
      "grad_norm": Infinity,
      "learning_rate": 0.00041584158415841583,
      "logits/chosen": -3.5052047762225076,
      "logits/rejected": -3.5252457174716327,
      "logps/chosen": -211.47866821289062,
      "logps/rejected": -226.36801528930664,
      "loss": 0.612738847732544,
      "mean_token_accuracy": 0.4072455167770386,
      "num_tokens": 57371.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.439986944198608,
      "rewards/margins": 0.8419481515884399,
      "rewards/rejected": -5.281934976577759,
      "step": 18
    },
    {
      "entropy": 1.4729647934436798,
      "epoch": 0.1890547263681592,
      "grad_norm": 2.456820487976074,
      "learning_rate": 0.0004108910891089109,
      "logits/chosen": -3.5512168301622737,
      "logits/rejected": -3.538663341941865,
      "logps/chosen": -148.29982948303223,
      "logps/rejected": -166.7068576812744,
      "loss": 0.6629408001899719,
      "mean_token_accuracy": 0.35727057605981827,
      "num_tokens": 60022.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -3.792637586593628,
      "rewards/margins": 0.5820458717644215,
      "rewards/rejected": -4.374683439731598,
      "step": 19
    },
    {
      "entropy": 1.5030970871448517,
      "epoch": 0.19900497512437812,
      "grad_norm": 4.276188373565674,
      "learning_rate": 0.000405940594059406,
      "logits/chosen": -3.5231190638874805,
      "logits/rejected": -3.5430015732000153,
      "logps/chosen": -197.88331604003906,
      "logps/rejected": -207.0079689025879,
      "loss": 0.5883949995040894,
      "mean_token_accuracy": 0.4058200493454933,
      "num_tokens": 63234.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -4.501765310764313,
      "rewards/margins": 0.6971243321895599,
      "rewards/rejected": -5.19888961315155,
      "step": 20
    },
    {
      "entropy": 1.485507309436798,
      "epoch": 0.208955223880597,
      "grad_norm": 2.7109107971191406,
      "learning_rate": 0.000400990099009901,
      "logits/chosen": -3.5226335405352645,
      "logits/rejected": -3.531911987571154,
      "logps/chosen": -250.09426879882812,
      "logps/rejected": -251.03947830200195,
      "loss": 0.588646650314331,
      "mean_token_accuracy": 0.38159702718257904,
      "num_tokens": 66663.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -7.025094747543335,
      "rewards/margins": 0.5600872486829758,
      "rewards/rejected": -7.585182189941406,
      "step": 21
    },
    {
      "entropy": 1.4331533908843994,
      "epoch": 0.21890547263681592,
      "grad_norm": 0.2803345322608948,
      "learning_rate": 0.00039603960396039607,
      "logits/chosen": -3.422363599396214,
      "logits/rejected": -3.436769247049262,
      "logps/chosen": -186.57451629638672,
      "logps/rejected": -203.19062423706055,
      "loss": 0.6102136373519897,
      "mean_token_accuracy": 0.41628195345401764,
      "num_tokens": 69753.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -4.635616421699524,
      "rewards/margins": 0.5115245580673218,
      "rewards/rejected": -5.147140979766846,
      "step": 22
    },
    {
      "entropy": 1.7424571514129639,
      "epoch": 0.22885572139303484,
      "grad_norm": 0.896638035774231,
      "learning_rate": 0.0003910891089108911,
      "logits/chosen": -3.5223754972126913,
      "logits/rejected": -3.5675395646362458,
      "logps/chosen": -193.18716049194336,
      "logps/rejected": -212.6921501159668,
      "loss": 0.5874005556106567,
      "mean_token_accuracy": 0.3610313981771469,
      "num_tokens": 72943.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -5.1880112290382385,
      "rewards/margins": 0.5615221261978149,
      "rewards/rejected": -5.749533414840698,
      "step": 23
    },
    {
      "entropy": 1.5620428621768951,
      "epoch": 0.23880597014925373,
      "grad_norm": 0.8171165585517883,
      "learning_rate": 0.00038613861386138616,
      "logits/chosen": -3.444536389293048,
      "logits/rejected": -3.469452442213975,
      "logps/chosen": -227.75585174560547,
      "logps/rejected": -244.11057662963867,
      "loss": 0.5816839933395386,
      "mean_token_accuracy": 0.39418596029281616,
      "num_tokens": 76389.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -6.575283408164978,
      "rewards/margins": 0.8996190428733826,
      "rewards/rejected": -7.474902510643005,
      "step": 24
    },
    {
      "entropy": 1.682222694158554,
      "epoch": 0.24875621890547264,
      "grad_norm": 3.5681254863739014,
      "learning_rate": 0.0003811881188118812,
      "logits/chosen": -3.470989559019067,
      "logits/rejected": -3.4263661433950654,
      "logps/chosen": -215.18242645263672,
      "logps/rejected": -212.655517578125,
      "loss": 0.7612518072128296,
      "mean_token_accuracy": 0.33192718774080276,
      "num_tokens": 79652.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -6.581342101097107,
      "rewards/margins": -0.09777332842350006,
      "rewards/rejected": -6.483568787574768,
      "step": 25
    },
    {
      "entropy": 1.7008473575115204,
      "epoch": 0.25870646766169153,
      "grad_norm": 0.08288310468196869,
      "learning_rate": 0.00037623762376237625,
      "logits/chosen": -3.427816639852287,
      "logits/rejected": -3.4542770548219104,
      "logps/chosen": -218.20597076416016,
      "logps/rejected": -248.2343292236328,
      "loss": 0.6077355742454529,
      "mean_token_accuracy": 0.35978567600250244,
      "num_tokens": 82832.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -7.347757697105408,
      "rewards/margins": 0.7340568602085114,
      "rewards/rejected": -8.081814527511597,
      "step": 26
    },
    {
      "entropy": 1.811392456293106,
      "epoch": 0.26865671641791045,
      "grad_norm": 0.4933086335659027,
      "learning_rate": 0.0003712871287128713,
      "logits/chosen": -3.530475011062849,
      "logits/rejected": -3.513883089770881,
      "logps/chosen": -324.6820068359375,
      "logps/rejected": -361.88365936279297,
      "loss": 0.5700176954269409,
      "mean_token_accuracy": 0.3622868210077286,
      "num_tokens": 87715.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -8.779233574867249,
      "rewards/margins": 0.7017057538032532,
      "rewards/rejected": -9.480939149856567,
      "step": 27
    },
    {
      "entropy": 1.9288294613361359,
      "epoch": 0.27860696517412936,
      "grad_norm": 2.3309881687164307,
      "learning_rate": 0.00036633663366336634,
      "logits/chosen": -3.5844996946988785,
      "logits/rejected": -3.5284301241131,
      "logps/chosen": -184.63220977783203,
      "logps/rejected": -240.8532257080078,
      "loss": 0.47455936670303345,
      "mean_token_accuracy": 0.3226698935031891,
      "num_tokens": 90862.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -5.509602308273315,
      "rewards/margins": 1.977087453007698,
      "rewards/rejected": -7.486689925193787,
      "step": 28
    },
    {
      "entropy": 1.827368676662445,
      "epoch": 0.2885572139303483,
      "grad_norm": 0.7671760320663452,
      "learning_rate": 0.0003613861386138614,
      "logits/chosen": -3.5442204947172145,
      "logits/rejected": -3.5596477134863482,
      "logps/chosen": -158.0522174835205,
      "logps/rejected": -175.73583793640137,
      "loss": 0.583824098110199,
      "mean_token_accuracy": 0.3381008803844452,
      "num_tokens": 93730.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.713171184062958,
      "rewards/margins": 0.6338318586349487,
      "rewards/rejected": -5.347002983093262,
      "step": 29
    },
    {
      "entropy": 2.166943669319153,
      "epoch": 0.29850746268656714,
      "grad_norm": Infinity,
      "learning_rate": 0.0003564356435643564,
      "logits/chosen": -3.5212162470658805,
      "logits/rejected": -3.521852987485434,
      "logps/chosen": -309.9493179321289,
      "logps/rejected": -350.4508514404297,
      "loss": 0.5989641547203064,
      "mean_token_accuracy": 0.30863772332668304,
      "num_tokens": 97866.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -9.270570039749146,
      "rewards/margins": 1.3084884360432625,
      "rewards/rejected": -10.579058527946472,
      "step": 30
    },
    {
      "entropy": 1.8375334739685059,
      "epoch": 0.30845771144278605,
      "grad_norm": 9.901069641113281,
      "learning_rate": 0.00035148514851485147,
      "logits/chosen": -3.456909141037305,
      "logits/rejected": -3.4208388982393494,
      "logps/chosen": -210.7388458251953,
      "logps/rejected": -186.36470794677734,
      "loss": 1.2342932224273682,
      "mean_token_accuracy": 0.31642504036426544,
      "num_tokens": 100664.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -7.150167465209961,
      "rewards/margins": -0.6264755576848984,
      "rewards/rejected": -6.523691773414612,
      "step": 31
    },
    {
      "entropy": 1.99248006939888,
      "epoch": 0.31840796019900497,
      "grad_norm": 4.666665077209473,
      "learning_rate": 0.0003465346534653465,
      "logits/chosen": -3.613257063639754,
      "logits/rejected": -3.629373806723718,
      "logps/chosen": -213.52509307861328,
      "logps/rejected": -228.46065139770508,
      "loss": 0.7087903022766113,
      "mean_token_accuracy": 0.35015513002872467,
      "num_tokens": 103912.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -6.4269896149635315,
      "rewards/margins": 0.1060914397239685,
      "rewards/rejected": -6.533081233501434,
      "step": 32
    },
    {
      "entropy": 2.011792302131653,
      "epoch": 0.3283582089552239,
      "grad_norm": 0.9864645600318909,
      "learning_rate": 0.0003415841584158416,
      "logits/chosen": -3.6115896043063813,
      "logits/rejected": -3.624950827978018,
      "logps/chosen": -237.15367889404297,
      "logps/rejected": -255.65070724487305,
      "loss": 0.574335515499115,
      "mean_token_accuracy": 0.34920939058065414,
      "num_tokens": 107516.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -6.345386028289795,
      "rewards/margins": 0.8882534243166447,
      "rewards/rejected": -7.233639478683472,
      "step": 33
    },
    {
      "entropy": 1.9972886443138123,
      "epoch": 0.3383084577114428,
      "grad_norm": 1.5521798133850098,
      "learning_rate": 0.00033663366336633666,
      "logits/chosen": -3.6060157953194127,
      "logits/rejected": -3.5974054100688786,
      "logps/chosen": -170.52621269226074,
      "logps/rejected": -218.68214797973633,
      "loss": 0.5241377353668213,
      "mean_token_accuracy": 0.36179573833942413,
      "num_tokens": 110501.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -4.737287938594818,
      "rewards/margins": 2.1275550723075867,
      "rewards/rejected": -6.864843249320984,
      "step": 34
    },
    {
      "entropy": 2.147545635700226,
      "epoch": 0.3482587064676617,
      "grad_norm": 3.7898976802825928,
      "learning_rate": 0.0003316831683168317,
      "logits/chosen": -3.662060887563785,
      "logits/rejected": -3.643096818576444,
      "logps/chosen": -220.84019088745117,
      "logps/rejected": -218.89766311645508,
      "loss": 0.6805493831634521,
      "mean_token_accuracy": 0.34033310413360596,
      "num_tokens": 113765.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -6.760529637336731,
      "rewards/margins": 0.028392881155014038,
      "rewards/rejected": -6.788922429084778,
      "step": 35
    },
    {
      "entropy": 2.204443633556366,
      "epoch": 0.3582089552238806,
      "grad_norm": 3.009458541870117,
      "learning_rate": 0.00032673267326732675,
      "logits/chosen": -3.6537463903752347,
      "logits/rejected": -3.639793618003779,
      "logps/chosen": -257.6618881225586,
      "logps/rejected": -279.3141975402832,
      "loss": 0.5332838892936707,
      "mean_token_accuracy": 0.3123948536813259,
      "num_tokens": 117530.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -7.534723997116089,
      "rewards/margins": 0.9232382774353027,
      "rewards/rejected": -8.457962155342102,
      "step": 36
    },
    {
      "entropy": 2.0133582651615143,
      "epoch": 0.3681592039800995,
      "grad_norm": 6.925239086151123,
      "learning_rate": 0.0003217821782178218,
      "logits/chosen": -3.4493966091905524,
      "logits/rejected": -3.4655641391055774,
      "logps/chosen": -189.9989013671875,
      "logps/rejected": -197.52683639526367,
      "loss": 0.6808904409408569,
      "mean_token_accuracy": 0.3597341701388359,
      "num_tokens": 120290.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -6.549296140670776,
      "rewards/margins": 0.3740836828947067,
      "rewards/rejected": -6.9233797788619995,
      "step": 37
    },
    {
      "entropy": 1.799951583147049,
      "epoch": 0.3781094527363184,
      "grad_norm": Infinity,
      "learning_rate": 0.00031683168316831684,
      "logits/chosen": -3.2567474854555534,
      "logits/rejected": -3.2621365465303196,
      "logps/chosen": -305.88188552856445,
      "logps/rejected": -318.30445098876953,
      "loss": 1.0073232650756836,
      "mean_token_accuracy": 0.30564460158348083,
      "num_tokens": 123971.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -13.314107418060303,
      "rewards/margins": 0.5497575104236603,
      "rewards/rejected": -13.863865375518799,
      "step": 38
    },
    {
      "entropy": 1.9222525358200073,
      "epoch": 0.3880597014925373,
      "grad_norm": 8.6959867477417,
      "learning_rate": 0.0003118811881188119,
      "logits/chosen": -3.2874606994512017,
      "logits/rejected": -3.2915129031186225,
      "logps/chosen": -294.8932342529297,
      "logps/rejected": -334.9911651611328,
      "loss": 0.7118847370147705,
      "mean_token_accuracy": 0.31055251508951187,
      "num_tokens": 127590.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -12.726699829101562,
      "rewards/margins": 1.7574191242456436,
      "rewards/rejected": -14.484118700027466,
      "step": 39
    },
    {
      "entropy": 1.763464778661728,
      "epoch": 0.39800995024875624,
      "grad_norm": 5.233345985412598,
      "learning_rate": 0.00030693069306930693,
      "logits/chosen": -3.0751955432610805,
      "logits/rejected": -3.1166185180383543,
      "logps/chosen": -236.8795928955078,
      "logps/rejected": -275.63719177246094,
      "loss": 0.4635728895664215,
      "mean_token_accuracy": 0.2654060050845146,
      "num_tokens": 130444.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -11.302006244659424,
      "rewards/margins": 1.8581123352050781,
      "rewards/rejected": -13.160118579864502,
      "step": 40
    },
    {
      "entropy": 2.066802978515625,
      "epoch": 0.4079601990049751,
      "grad_norm": 17.57449722290039,
      "learning_rate": 0.000301980198019802,
      "logits/chosen": -2.980425116300123,
      "logits/rejected": -2.970705539767077,
      "logps/chosen": -260.0305938720703,
      "logps/rejected": -314.4711227416992,
      "loss": 0.9843884706497192,
      "mean_token_accuracy": 0.2243056520819664,
      "num_tokens": 133312.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -14.370165824890137,
      "rewards/margins": 2.6685269474983215,
      "rewards/rejected": -17.03869318962097,
      "step": 41
    },
    {
      "entropy": 2.221681296825409,
      "epoch": 0.417910447761194,
      "grad_norm": 14.223358154296875,
      "learning_rate": 0.000297029702970297,
      "logits/chosen": -3.159763682014305,
      "logits/rejected": -3.1037028438918695,
      "logps/chosen": -295.029541015625,
      "logps/rejected": -307.0481491088867,
      "loss": 1.0131914615631104,
      "mean_token_accuracy": 0.21281880140304565,
      "num_tokens": 136429.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -13.900337934494019,
      "rewards/margins": 1.0990875959396362,
      "rewards/rejected": -14.999425649642944,
      "step": 42
    },
    {
      "entropy": 1.8978886902332306,
      "epoch": 0.42786069651741293,
      "grad_norm": 40.302528381347656,
      "learning_rate": 0.00029207920792079207,
      "logits/chosen": -3.1809646094391315,
      "logits/rejected": -3.196636022508817,
      "logps/chosen": -303.73348236083984,
      "logps/rejected": -304.0575942993164,
      "loss": 1.2173784971237183,
      "mean_token_accuracy": 0.24012192711234093,
      "num_tokens": 139652.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -14.823058605194092,
      "rewards/margins": -0.07431718707084656,
      "rewards/rejected": -14.748740911483765,
      "step": 43
    },
    {
      "entropy": 2.3116554617881775,
      "epoch": 0.43781094527363185,
      "grad_norm": 72.7015609741211,
      "learning_rate": 0.0002871287128712871,
      "logits/chosen": -3.1047942161569058,
      "logits/rejected": -3.089049966469761,
      "logps/chosen": -260.9189682006836,
      "logps/rejected": -283.27161026000977,
      "loss": 1.208611249923706,
      "mean_token_accuracy": 0.22318781167268753,
      "num_tokens": 142532.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -14.062894105911255,
      "rewards/margins": 1.0884355306625366,
      "rewards/rejected": -15.151329278945923,
      "step": 44
    },
    {
      "entropy": 2.28976833820343,
      "epoch": 0.44776119402985076,
      "grad_norm": Infinity,
      "learning_rate": 0.00028217821782178216,
      "logits/chosen": -3.254444216008772,
      "logits/rejected": -3.2204778741231186,
      "logps/chosen": -429.227294921875,
      "logps/rejected": -428.4982452392578,
      "loss": 1.3327971696853638,
      "mean_token_accuracy": 0.2500062696635723,
      "num_tokens": 146559.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -21.970479130744934,
      "rewards/margins": -0.4409611225128174,
      "rewards/rejected": -21.529518008232117,
      "step": 45
    },
    {
      "entropy": 2.065894603729248,
      "epoch": 0.4577114427860697,
      "grad_norm": 24.154191970825195,
      "learning_rate": 0.00027722772277227726,
      "logits/chosen": -3.1520684797411356,
      "logits/rejected": -3.1568709494368896,
      "logps/chosen": -262.4333801269531,
      "logps/rejected": -261.6428565979004,
      "loss": 1.616957187652588,
      "mean_token_accuracy": 0.2732032462954521,
      "num_tokens": 149674.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -13.10332190990448,
      "rewards/margins": -0.2920948714017868,
      "rewards/rejected": -12.811226963996887,
      "step": 46
    },
    {
      "entropy": 1.9271219968795776,
      "epoch": 0.46766169154228854,
      "grad_norm": 62.61008834838867,
      "learning_rate": 0.0002722772277227723,
      "logits/chosen": -3.222555282865147,
      "logits/rejected": -3.2629351656574213,
      "logps/chosen": -255.8454475402832,
      "logps/rejected": -277.0890884399414,
      "loss": 1.713848352432251,
      "mean_token_accuracy": 0.28745026886463165,
      "num_tokens": 152868.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -11.964468240737915,
      "rewards/margins": -0.021911442279815674,
      "rewards/rejected": -11.942557096481323,
      "step": 47
    },
    {
      "entropy": 1.9222969710826874,
      "epoch": 0.47761194029850745,
      "grad_norm": 24.73862648010254,
      "learning_rate": 0.00026732673267326735,
      "logits/chosen": -3.2014811024082013,
      "logits/rejected": -3.235856109758436,
      "logps/chosen": -253.8110809326172,
      "logps/rejected": -279.10169982910156,
      "loss": 1.1895209550857544,
      "mean_token_accuracy": 0.289677482098341,
      "num_tokens": 156141.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -11.022257328033447,
      "rewards/margins": 0.13694772124290466,
      "rewards/rejected": -11.159205198287964,
      "step": 48
    },
    {
      "entropy": 1.8481970429420471,
      "epoch": 0.48756218905472637,
      "grad_norm": 14.322184562683105,
      "learning_rate": 0.0002623762376237624,
      "logits/chosen": -3.3439408915743605,
      "logits/rejected": -3.303895267821285,
      "logps/chosen": -220.04124069213867,
      "logps/rejected": -226.87024688720703,
      "loss": 0.8866377472877502,
      "mean_token_accuracy": 0.33033835887908936,
      "num_tokens": 159135.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -8.551008820533752,
      "rewards/margins": -0.14264798164367676,
      "rewards/rejected": -8.408360838890076,
      "step": 49
    },
    {
      "entropy": 1.9352594017982483,
      "epoch": 0.4975124378109453,
      "grad_norm": 0.009451066143810749,
      "learning_rate": 0.00025742574257425744,
      "logits/chosen": -3.385222989788937,
      "logits/rejected": -3.3840081123920838,
      "logps/chosen": -216.81492614746094,
      "logps/rejected": -257.5557556152344,
      "loss": 0.6498254537582397,
      "mean_token_accuracy": 0.3011988401412964,
      "num_tokens": 162285.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -7.384112596511841,
      "rewards/margins": 1.437139630317688,
      "rewards/rejected": -8.821252346038818,
      "step": 50
    },
    {
      "entropy": 1.9567635655403137,
      "epoch": 0.5074626865671642,
      "grad_norm": 11.31760025024414,
      "learning_rate": 0.0002524752475247525,
      "logits/chosen": -3.428618842247525,
      "logits/rejected": -3.476428795573361,
      "logps/chosen": -246.91674423217773,
      "logps/rejected": -261.93321990966797,
      "loss": 0.9736576676368713,
      "mean_token_accuracy": 0.29628537595272064,
      "num_tokens": 165657.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -8.399488687515259,
      "rewards/margins": 0.008383013308048248,
      "rewards/rejected": -8.407871842384338,
      "step": 51
    },
    {
      "entropy": 2.0954816043376923,
      "epoch": 0.5174129353233831,
      "grad_norm": 7.176389217376709,
      "learning_rate": 0.00024752475247524753,
      "logits/chosen": -3.571719104940522,
      "logits/rejected": -3.5774720027492,
      "logps/chosen": -164.6639518737793,
      "logps/rejected": -174.26453971862793,
      "loss": 1.0176879167556763,
      "mean_token_accuracy": 0.33037129044532776,
      "num_tokens": 168527.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -4.919166803359985,
      "rewards/margins": -0.36104128509759903,
      "rewards/rejected": -4.558125615119934,
      "step": 52
    },
    {
      "entropy": 2.1325119733810425,
      "epoch": 0.527363184079602,
      "grad_norm": 7.1155500411987305,
      "learning_rate": 0.00024257425742574257,
      "logits/chosen": -3.611224298480606,
      "logits/rejected": -3.606798095712098,
      "logps/chosen": -228.4919891357422,
      "logps/rejected": -211.7003345489502,
      "loss": 1.0112618207931519,
      "mean_token_accuracy": 0.34222544729709625,
      "num_tokens": 171891.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -4.663272202014923,
      "rewards/margins": -0.35346102714538574,
      "rewards/rejected": -4.309811294078827,
      "step": 53
    },
    {
      "entropy": 2.099044680595398,
      "epoch": 0.5373134328358209,
      "grad_norm": 4.623640537261963,
      "learning_rate": 0.00023762376237623762,
      "logits/chosen": -3.6421679058682925,
      "logits/rejected": -3.629708488565886,
      "logps/chosen": -150.8334503173828,
      "logps/rejected": -156.62395095825195,
      "loss": 0.8465480804443359,
      "mean_token_accuracy": 0.3315627798438072,
      "num_tokens": 174498.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.7043431997299194,
      "rewards/margins": 0.09302212297916412,
      "rewards/rejected": -3.7973653078079224,
      "step": 54
    },
    {
      "entropy": 2.0214365124702454,
      "epoch": 0.5472636815920398,
      "grad_norm": 4.609769344329834,
      "learning_rate": 0.0002326732673267327,
      "logits/chosen": -3.6433888397539675,
      "logits/rejected": -3.6304691103005715,
      "logps/chosen": -198.38171768188477,
      "logps/rejected": -235.6202049255371,
      "loss": 0.8365243673324585,
      "mean_token_accuracy": 0.39510079473257065,
      "num_tokens": 178185.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.9414217472076416,
      "rewards/margins": 0.17493990063667297,
      "rewards/rejected": -4.116361618041992,
      "step": 55
    },
    {
      "entropy": 2.2303199768066406,
      "epoch": 0.5572139303482587,
      "grad_norm": 2.9026308059692383,
      "learning_rate": 0.00022772277227722774,
      "logits/chosen": -3.6601714084022925,
      "logits/rejected": -3.6551866016141643,
      "logps/chosen": -138.404541015625,
      "logps/rejected": -181.2952823638916,
      "loss": 0.6974964141845703,
      "mean_token_accuracy": 0.308276005089283,
      "num_tokens": 181000.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -2.957142174243927,
      "rewards/margins": 0.10662916302680969,
      "rewards/rejected": -3.063771367073059,
      "step": 56
    },
    {
      "entropy": 2.0440548956394196,
      "epoch": 0.5671641791044776,
      "grad_norm": 3.176905393600464,
      "learning_rate": 0.00022277227722772278,
      "logits/chosen": -3.5878639508662946,
      "logits/rejected": -3.5972853573972476,
      "logps/chosen": -181.04140853881836,
      "logps/rejected": -196.8503646850586,
      "loss": 0.7555087804794312,
      "mean_token_accuracy": 0.36560703814029694,
      "num_tokens": 184440.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.725131630897522,
      "rewards/margins": 0.2023322656750679,
      "rewards/rejected": -3.927464008331299,
      "step": 57
    },
    {
      "entropy": 2.1503366231918335,
      "epoch": 0.5771144278606966,
      "grad_norm": 2.925879716873169,
      "learning_rate": 0.00021782178217821783,
      "logits/chosen": -3.635705258630424,
      "logits/rejected": -3.626571030941132,
      "logps/chosen": -153.08165740966797,
      "logps/rejected": -151.79789352416992,
      "loss": 0.8062557578086853,
      "mean_token_accuracy": 0.3421657532453537,
      "num_tokens": 187161.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.27903288602829,
      "rewards/margins": -0.01948907971382141,
      "rewards/rejected": -3.259543776512146,
      "step": 58
    },
    {
      "entropy": 2.067517399787903,
      "epoch": 0.5870646766169154,
      "grad_norm": 5.860987186431885,
      "learning_rate": 0.00021287128712871287,
      "logits/chosen": -3.6160732060464547,
      "logits/rejected": -3.603459410032726,
      "logps/chosen": -237.19454193115234,
      "logps/rejected": -241.7457733154297,
      "loss": 0.7830701470375061,
      "mean_token_accuracy": 0.3710586279630661,
      "num_tokens": 190961.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -4.790164351463318,
      "rewards/margins": 0.013833608478307724,
      "rewards/rejected": -4.803997993469238,
      "step": 59
    },
    {
      "entropy": 2.0456930100917816,
      "epoch": 0.5970149253731343,
      "grad_norm": 2.530921220779419,
      "learning_rate": 0.00020792079207920792,
      "logits/chosen": -3.634478617453735,
      "logits/rejected": -3.617604305001727,
      "logps/chosen": -164.7590560913086,
      "logps/rejected": -183.34884643554688,
      "loss": 0.6686393022537231,
      "mean_token_accuracy": 0.3322089686989784,
      "num_tokens": 193853.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.466741681098938,
      "rewards/margins": 0.10381258279085159,
      "rewards/rejected": -3.570554256439209,
      "step": 60
    },
    {
      "entropy": 2.0229939222335815,
      "epoch": 0.6069651741293532,
      "grad_norm": 1.2661631107330322,
      "learning_rate": 0.000202970297029703,
      "logits/chosen": -3.5366310534711625,
      "logits/rejected": -3.5608453098695625,
      "logps/chosen": -173.31506729125977,
      "logps/rejected": -224.96598434448242,
      "loss": 0.6006392240524292,
      "mean_token_accuracy": 0.36497529596090317,
      "num_tokens": 197089.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -3.186018943786621,
      "rewards/margins": 0.6044751890003681,
      "rewards/rejected": -3.790493965148926,
      "step": 61
    },
    {
      "entropy": 1.9178842902183533,
      "epoch": 0.6169154228855721,
      "grad_norm": 1.8906898498535156,
      "learning_rate": 0.00019801980198019803,
      "logits/chosen": -3.608905749310737,
      "logits/rejected": -3.524314705366162,
      "logps/chosen": -144.59884452819824,
      "logps/rejected": -172.51303100585938,
      "loss": 0.5483641028404236,
      "mean_token_accuracy": 0.3645987957715988,
      "num_tokens": 200171.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -2.4504005908966064,
      "rewards/margins": 0.7510848864912987,
      "rewards/rejected": -3.2014856338500977,
      "step": 62
    },
    {
      "entropy": 1.9568755626678467,
      "epoch": 0.6268656716417911,
      "grad_norm": 3.163543224334717,
      "learning_rate": 0.00019306930693069308,
      "logits/chosen": -3.514222002398174,
      "logits/rejected": -3.5172996107979926,
      "logps/chosen": -204.0835952758789,
      "logps/rejected": -196.35980606079102,
      "loss": 1.0338683128356934,
      "mean_token_accuracy": 0.3876264691352844,
      "num_tokens": 203615.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -3.8272334337234497,
      "rewards/margins": -0.41473425552248955,
      "rewards/rejected": -3.4124991297721863,
      "step": 63
    },
    {
      "entropy": 1.7704353034496307,
      "epoch": 0.6368159203980099,
      "grad_norm": 3.762464761734009,
      "learning_rate": 0.00018811881188118812,
      "logits/chosen": -3.58424572632331,
      "logits/rejected": -3.59264863110509,
      "logps/chosen": -140.27431678771973,
      "logps/rejected": -139.8353385925293,
      "loss": 0.8362555503845215,
      "mean_token_accuracy": 0.3757285177707672,
      "num_tokens": 206327.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -2.7881844639778137,
      "rewards/margins": -0.17374923825263977,
      "rewards/rejected": -2.6144352555274963,
      "step": 64
    },
    {
      "entropy": 1.8312126398086548,
      "epoch": 0.6467661691542289,
      "grad_norm": 1.7864776849746704,
      "learning_rate": 0.00018316831683168317,
      "logits/chosen": -3.5429987235441445,
      "logits/rejected": -3.564540953208156,
      "logps/chosen": -172.8157958984375,
      "logps/rejected": -174.86798095703125,
      "loss": 0.6336662769317627,
      "mean_token_accuracy": 0.38997988402843475,
      "num_tokens": 209337.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -3.506571590900421,
      "rewards/margins": 0.20759320259094238,
      "rewards/rejected": -3.7141648530960083,
      "step": 65
    },
    {
      "entropy": 1.7894618809223175,
      "epoch": 0.6567164179104478,
      "grad_norm": 18.43994140625,
      "learning_rate": 0.0001782178217821782,
      "logits/chosen": -3.5626331625353282,
      "logits/rejected": -3.558179466696584,
      "logps/chosen": -216.60082626342773,
      "logps/rejected": -254.75148391723633,
      "loss": 0.7142460942268372,
      "mean_token_accuracy": 0.3956107571721077,
      "num_tokens": 212880.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -4.492651045322418,
      "rewards/margins": 0.7144668847322464,
      "rewards/rejected": -5.207117915153503,
      "step": 66
    },
    {
      "entropy": 1.8538159728050232,
      "epoch": 0.6666666666666666,
      "grad_norm": 14.287700653076172,
      "learning_rate": 0.00017326732673267326,
      "logits/chosen": -3.6201702161488374,
      "logits/rejected": -3.635657981136088,
      "logps/chosen": -208.26702499389648,
      "logps/rejected": -188.26987075805664,
      "loss": 1.3694218397140503,
      "mean_token_accuracy": 0.37898362427949905,
      "num_tokens": 216222.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.368560254573822,
      "rewards/margins": -0.7072555869817734,
      "rewards/rejected": -3.6613046526908875,
      "step": 67
    },
    {
      "entropy": 1.788241446018219,
      "epoch": 0.6766169154228856,
      "grad_norm": 7.528784275054932,
      "learning_rate": 0.00016831683168316833,
      "logits/chosen": -3.5787578586561186,
      "logits/rejected": -3.5687125724199507,
      "logps/chosen": -166.56489372253418,
      "logps/rejected": -154.00261116027832,
      "loss": 1.1346752643585205,
      "mean_token_accuracy": 0.3375133126974106,
      "num_tokens": 219010.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -3.564320147037506,
      "rewards/margins": -0.4889891818165779,
      "rewards/rejected": -3.0753310918807983,
      "step": 68
    },
    {
      "entropy": 1.8404236435890198,
      "epoch": 0.6865671641791045,
      "grad_norm": 4.7393646240234375,
      "learning_rate": 0.00016336633663366338,
      "logits/chosen": -3.5680984551891988,
      "logits/rejected": -3.578736825609027,
      "logps/chosen": -151.18361282348633,
      "logps/rejected": -150.8898162841797,
      "loss": 0.9233277440071106,
      "mean_token_accuracy": 0.40693292766809464,
      "num_tokens": 221766.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.6996166706085205,
      "rewards/margins": -0.023674920201301575,
      "rewards/rejected": -3.67594176530838,
      "step": 69
    },
    {
      "entropy": 1.8105555176734924,
      "epoch": 0.6965174129353234,
      "grad_norm": 2.983847141265869,
      "learning_rate": 0.00015841584158415842,
      "logits/chosen": -3.593700880438685,
      "logits/rejected": -3.608964660947262,
      "logps/chosen": -146.78189277648926,
      "logps/rejected": -164.89960098266602,
      "loss": 0.6955384612083435,
      "mean_token_accuracy": 0.37945864349603653,
      "num_tokens": 224529.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -3.2332761883735657,
      "rewards/margins": 0.1823255941271782,
      "rewards/rejected": -3.415601670742035,
      "step": 70
    },
    {
      "entropy": 1.7670131027698517,
      "epoch": 0.7064676616915423,
      "grad_norm": 2.6227941513061523,
      "learning_rate": 0.00015346534653465347,
      "logits/chosen": -3.6208933497371607,
      "logits/rejected": -3.627287218750385,
      "logps/chosen": -171.93946075439453,
      "logps/rejected": -174.2279167175293,
      "loss": 0.6700834035873413,
      "mean_token_accuracy": 0.388210229575634,
      "num_tokens": 227740.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -3.4097407460212708,
      "rewards/margins": 0.34128284454345703,
      "rewards/rejected": -3.751023590564728,
      "step": 71
    },
    {
      "entropy": 1.8676961362361908,
      "epoch": 0.7164179104477612,
      "grad_norm": 2.375143527984619,
      "learning_rate": 0.0001485148514851485,
      "logits/chosen": -3.6235203552351987,
      "logits/rejected": -3.63041797805766,
      "logps/chosen": -182.70703506469727,
      "logps/rejected": -204.6234588623047,
      "loss": 0.6822044849395752,
      "mean_token_accuracy": 0.37428663671016693,
      "num_tokens": 231122.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.4435550570487976,
      "rewards/margins": 0.35684891045093536,
      "rewards/rejected": -3.8004040122032166,
      "step": 72
    },
    {
      "entropy": 1.827594518661499,
      "epoch": 0.7263681592039801,
      "grad_norm": 0.6919447779655457,
      "learning_rate": 0.00014356435643564356,
      "logits/chosen": -3.6002415933798546,
      "logits/rejected": -3.5960349776366445,
      "logps/chosen": -144.23443603515625,
      "logps/rejected": -201.61253356933594,
      "loss": 0.5005671977996826,
      "mean_token_accuracy": 0.339895598590374,
      "num_tokens": 234134.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -3.2676995396614075,
      "rewards/margins": 1.224392369389534,
      "rewards/rejected": -4.4920918345451355,
      "step": 73
    },
    {
      "entropy": 1.748055875301361,
      "epoch": 0.736318407960199,
      "grad_norm": 2.5934488773345947,
      "learning_rate": 0.00013861386138613863,
      "logits/chosen": -3.6360650808702615,
      "logits/rejected": -3.5972090839376483,
      "logps/chosen": -157.06155395507812,
      "logps/rejected": -154.848783493042,
      "loss": 0.7558881044387817,
      "mean_token_accuracy": 0.38590408116579056,
      "num_tokens": 237300.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.0302443504333496,
      "rewards/margins": 0.035845797508955,
      "rewards/rejected": -3.066090077161789,
      "step": 74
    },
    {
      "entropy": 1.684916764497757,
      "epoch": 0.746268656716418,
      "grad_norm": 2.008575677871704,
      "learning_rate": 0.00013366336633663367,
      "logits/chosen": -3.561989163656944,
      "logits/rejected": -3.56625081249695,
      "logps/chosen": -185.2477912902832,
      "logps/rejected": -220.90989685058594,
      "loss": 0.6018548607826233,
      "mean_token_accuracy": 0.41026031970977783,
      "num_tokens": 240884.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.9729064106941223,
      "rewards/margins": 0.33660563081502914,
      "rewards/rejected": -4.30951201915741,
      "step": 75
    },
    {
      "entropy": 1.702199250459671,
      "epoch": 0.7562189054726368,
      "grad_norm": 4.6139235496521,
      "learning_rate": 0.00012871287128712872,
      "logits/chosen": -3.5974828000994803,
      "logits/rejected": -3.604220855457153,
      "logps/chosen": -207.3554458618164,
      "logps/rejected": -214.70018196105957,
      "loss": 0.9295954704284668,
      "mean_token_accuracy": 0.4100513458251953,
      "num_tokens": 244380.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -4.185361862182617,
      "rewards/margins": -0.16199729591608047,
      "rewards/rejected": -4.023364543914795,
      "step": 76
    },
    {
      "entropy": 1.8483723402023315,
      "epoch": 0.7661691542288557,
      "grad_norm": 4.52067756652832,
      "learning_rate": 0.00012376237623762376,
      "logits/chosen": -3.615742114915774,
      "logits/rejected": -3.630499670149322,
      "logps/chosen": -183.99293518066406,
      "logps/rejected": -172.64349746704102,
      "loss": 0.9373040199279785,
      "mean_token_accuracy": 0.34607332944869995,
      "num_tokens": 247230.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.952219247817993,
      "rewards/margins": -0.03562957048416138,
      "rewards/rejected": -3.9165897369384766,
      "step": 77
    },
    {
      "entropy": 1.6928546726703644,
      "epoch": 0.7761194029850746,
      "grad_norm": 3.354624032974243,
      "learning_rate": 0.00011881188118811881,
      "logits/chosen": -3.6518346370167287,
      "logits/rejected": -3.6789896636578354,
      "logps/chosen": -170.9540023803711,
      "logps/rejected": -189.51920700073242,
      "loss": 0.6489702463150024,
      "mean_token_accuracy": 0.41937771439552307,
      "num_tokens": 250533.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -3.277673453092575,
      "rewards/margins": 0.238661028444767,
      "rewards/rejected": -3.5163345336914062,
      "step": 78
    },
    {
      "entropy": 1.7784149944782257,
      "epoch": 0.7860696517412935,
      "grad_norm": 2.8312323093414307,
      "learning_rate": 0.00011386138613861387,
      "logits/chosen": -3.678140706310251,
      "logits/rejected": -3.6888157082532564,
      "logps/chosen": -171.8286895751953,
      "logps/rejected": -217.10009002685547,
      "loss": 0.6263619065284729,
      "mean_token_accuracy": 0.36294757574796677,
      "num_tokens": 253574.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.541302859783173,
      "rewards/margins": 0.7501096352934837,
      "rewards/rejected": -5.291412353515625,
      "step": 79
    },
    {
      "entropy": 1.6919234693050385,
      "epoch": 0.7960199004975125,
      "grad_norm": 0.7620548605918884,
      "learning_rate": 0.00010891089108910891,
      "logits/chosen": -3.7173505595829215,
      "logits/rejected": -3.7161475484288866,
      "logps/chosen": -222.93682098388672,
      "logps/rejected": -222.69513702392578,
      "loss": 0.6783262491226196,
      "mean_token_accuracy": 0.385500892996788,
      "num_tokens": 257005.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -4.736607253551483,
      "rewards/margins": 0.03428628295660019,
      "rewards/rejected": -4.770893633365631,
      "step": 80
    },
    {
      "entropy": 1.6837327778339386,
      "epoch": 0.8059701492537313,
      "grad_norm": 3.8109116554260254,
      "learning_rate": 0.00010396039603960396,
      "logits/chosen": -3.744877125633101,
      "logits/rejected": -3.7239091993002296,
      "logps/chosen": -233.0377197265625,
      "logps/rejected": -260.4305229187012,
      "loss": 0.6774035692214966,
      "mean_token_accuracy": 0.37072674185037613,
      "num_tokens": 261021.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -4.784064710140228,
      "rewards/margins": 0.45506689697504044,
      "rewards/rejected": -5.239131569862366,
      "step": 81
    },
    {
      "entropy": 1.6132145822048187,
      "epoch": 0.8159203980099502,
      "grad_norm": 4.082960605621338,
      "learning_rate": 9.900990099009902e-05,
      "logits/chosen": -3.6440604838830586,
      "logits/rejected": -3.6654163244894704,
      "logps/chosen": -196.36359786987305,
      "logps/rejected": -189.5399055480957,
      "loss": 0.8034707307815552,
      "mean_token_accuracy": 0.4102801978588104,
      "num_tokens": 264256.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -4.631353974342346,
      "rewards/margins": -0.18269547820091248,
      "rewards/rejected": -4.448658347129822,
      "step": 82
    },
    {
      "entropy": 1.642210841178894,
      "epoch": 0.8258706467661692,
      "grad_norm": 3.894784688949585,
      "learning_rate": 9.405940594059406e-05,
      "logits/chosen": -3.703474447193526,
      "logits/rejected": -3.711402166511322,
      "logps/chosen": -250.06790161132812,
      "logps/rejected": -248.12013244628906,
      "loss": 0.8837608098983765,
      "mean_token_accuracy": 0.38086505979299545,
      "num_tokens": 268126.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -5.739135384559631,
      "rewards/margins": -0.24859696626663208,
      "rewards/rejected": -5.490538477897644,
      "step": 83
    },
    {
      "entropy": 1.6012614071369171,
      "epoch": 0.835820895522388,
      "grad_norm": 3.3155040740966797,
      "learning_rate": 8.91089108910891e-05,
      "logits/chosen": -3.64063474797791,
      "logits/rejected": -3.6373772644695217,
      "logps/chosen": -133.76382446289062,
      "logps/rejected": -143.0687713623047,
      "loss": 0.8610319495201111,
      "mean_token_accuracy": 0.3686061054468155,
      "num_tokens": 270528.0,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -3.8289989233016968,
      "rewards/margins": -0.20909389853477478,
      "rewards/rejected": -3.6199050545692444,
      "step": 84
    },
    {
      "entropy": 1.6356386542320251,
      "epoch": 0.845771144278607,
      "grad_norm": 6.067594051361084,
      "learning_rate": 8.415841584158417e-05,
      "logits/chosen": -3.6797450831804857,
      "logits/rejected": -3.69026861592658,
      "logps/chosen": -190.2202491760254,
      "logps/rejected": -178.37620735168457,
      "loss": 0.8410797715187073,
      "mean_token_accuracy": 0.3780713751912117,
      "num_tokens": 273971.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -4.713983058929443,
      "rewards/margins": -0.10549770295619965,
      "rewards/rejected": -4.608485221862793,
      "step": 85
    },
    {
      "entropy": 1.6179614663124084,
      "epoch": 0.8557213930348259,
      "grad_norm": 2.606879949569702,
      "learning_rate": 7.920792079207921e-05,
      "logits/chosen": -3.714054792664533,
      "logits/rejected": -3.7162360998405983,
      "logps/chosen": -187.36741065979004,
      "logps/rejected": -188.57626342773438,
      "loss": 0.6618068814277649,
      "mean_token_accuracy": 0.40294698625802994,
      "num_tokens": 277206.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -4.31531286239624,
      "rewards/margins": 0.24175719916820526,
      "rewards/rejected": -4.557070016860962,
      "step": 86
    },
    {
      "entropy": 1.6011567115783691,
      "epoch": 0.8656716417910447,
      "grad_norm": 2.2501895427703857,
      "learning_rate": 7.425742574257426e-05,
      "logits/chosen": -3.674183280463941,
      "logits/rejected": -3.6626142261745214,
      "logps/chosen": -153.10731315612793,
      "logps/rejected": -150.5159568786621,
      "loss": 0.6958032250404358,
      "mean_token_accuracy": 0.36470960080623627,
      "num_tokens": 279780.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.18908166885376,
      "rewards/margins": 0.0875798761844635,
      "rewards/rejected": -4.276661396026611,
      "step": 87
    },
    {
      "entropy": 1.6125659942626953,
      "epoch": 0.8756218905472637,
      "grad_norm": 5.258417129516602,
      "learning_rate": 6.930693069306931e-05,
      "logits/chosen": -3.7041569457218415,
      "logits/rejected": -3.7053109725079976,
      "logps/chosen": -231.12665176391602,
      "logps/rejected": -218.75960540771484,
      "loss": 0.7343534827232361,
      "mean_token_accuracy": 0.3844173774123192,
      "num_tokens": 283242.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -5.803511381149292,
      "rewards/margins": -0.03953322768211365,
      "rewards/rejected": -5.7639782428741455,
      "step": 88
    },
    {
      "entropy": 1.655126303434372,
      "epoch": 0.8855721393034826,
      "grad_norm": 3.7742555141448975,
      "learning_rate": 6.435643564356436e-05,
      "logits/chosen": -3.698541244359123,
      "logits/rejected": -3.7294442870484055,
      "logps/chosen": -184.96338653564453,
      "logps/rejected": -195.39525604248047,
      "loss": 0.7234901785850525,
      "mean_token_accuracy": 0.3443598970770836,
      "num_tokens": 286250.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -4.933422684669495,
      "rewards/margins": 0.19522131979465485,
      "rewards/rejected": -5.128643989562988,
      "step": 89
    },
    {
      "entropy": 1.564221054315567,
      "epoch": 0.8955223880597015,
      "grad_norm": 2.220445394515991,
      "learning_rate": 5.9405940594059404e-05,
      "logits/chosen": -3.7234489777398343,
      "logits/rejected": -3.695775704686508,
      "logps/chosen": -200.81872940063477,
      "logps/rejected": -230.38358306884766,
      "loss": 0.6132370233535767,
      "mean_token_accuracy": 0.37841925024986267,
      "num_tokens": 290217.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.426435708999634,
      "rewards/margins": 0.2670505791902542,
      "rewards/rejected": -4.693486332893372,
      "step": 90
    },
    {
      "entropy": 1.563878208398819,
      "epoch": 0.9054726368159204,
      "grad_norm": 1.4845564365386963,
      "learning_rate": 5.4455445544554456e-05,
      "logits/chosen": -3.675358395963568,
      "logits/rejected": -3.666738373474767,
      "logps/chosen": -255.80266189575195,
      "logps/rejected": -266.2362861633301,
      "loss": 0.658227801322937,
      "mean_token_accuracy": 0.3786882683634758,
      "num_tokens": 294045.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -5.8976744413375854,
      "rewards/margins": 0.33874283730983734,
      "rewards/rejected": -6.2364174127578735,
      "step": 91
    },
    {
      "entropy": 1.6665741205215454,
      "epoch": 0.9154228855721394,
      "grad_norm": 6.198132038116455,
      "learning_rate": 4.950495049504951e-05,
      "logits/chosen": -3.6663319924446323,
      "logits/rejected": -3.7296222855776184,
      "logps/chosen": -158.00820922851562,
      "logps/rejected": -178.97606658935547,
      "loss": 1.0925066471099854,
      "mean_token_accuracy": 0.3629995733499527,
      "num_tokens": 296857.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -3.999560624361038,
      "rewards/margins": 0.033911485224962234,
      "rewards/rejected": -4.033472061157227,
      "step": 92
    },
    {
      "entropy": 1.644345223903656,
      "epoch": 0.9253731343283582,
      "grad_norm": 2.0987441539764404,
      "learning_rate": 4.455445544554455e-05,
      "logits/chosen": -3.6583529962844343,
      "logits/rejected": -3.6667724616445185,
      "logps/chosen": -195.10536193847656,
      "logps/rejected": -285.5465202331543,
      "loss": 0.5438169240951538,
      "mean_token_accuracy": 0.42512599378824234,
      "num_tokens": 300856.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -4.106789469718933,
      "rewards/margins": 0.9598179310560226,
      "rewards/rejected": -5.066607356071472,
      "step": 93
    },
    {
      "entropy": 1.5577265322208405,
      "epoch": 0.9353233830845771,
      "grad_norm": 4.220263957977295,
      "learning_rate": 3.9603960396039605e-05,
      "logits/chosen": -3.6748759341594157,
      "logits/rejected": -3.729366666919669,
      "logps/chosen": -210.3987045288086,
      "logps/rejected": -252.40514373779297,
      "loss": 0.839250385761261,
      "mean_token_accuracy": 0.40358515083789825,
      "num_tokens": 304623.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.8432682156562805,
      "rewards/margins": 0.40834885090589523,
      "rewards/rejected": -5.251616835594177,
      "step": 94
    },
    {
      "entropy": 1.729375958442688,
      "epoch": 0.945273631840796,
      "grad_norm": 2.3955490589141846,
      "learning_rate": 3.465346534653466e-05,
      "logits/chosen": -3.658530322956607,
      "logits/rejected": -3.6282934600825714,
      "logps/chosen": -268.2776565551758,
      "logps/rejected": -297.23538970947266,
      "loss": 0.6312628984451294,
      "mean_token_accuracy": 0.36635589599609375,
      "num_tokens": 308746.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -5.5482582449913025,
      "rewards/margins": 0.396299347281456,
      "rewards/rejected": -5.944557547569275,
      "step": 95
    },
    {
      "entropy": 1.6423245668411255,
      "epoch": 0.9552238805970149,
      "grad_norm": 2.077113151550293,
      "learning_rate": 2.9702970297029702e-05,
      "logits/chosen": -3.592531715367838,
      "logits/rejected": -3.6782985785875457,
      "logps/chosen": -203.75262069702148,
      "logps/rejected": -228.7814712524414,
      "loss": 0.5487721562385559,
      "mean_token_accuracy": 0.4399268254637718,
      "num_tokens": 312417.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -3.5327599346637726,
      "rewards/margins": 0.6648375540971756,
      "rewards/rejected": -4.197597444057465,
      "step": 96
    },
    {
      "entropy": 1.763981431722641,
      "epoch": 0.9651741293532339,
      "grad_norm": 2.8148458003997803,
      "learning_rate": 2.4752475247524754e-05,
      "logits/chosen": -3.667448573612207,
      "logits/rejected": -3.6800848210863935,
      "logps/chosen": -160.70482063293457,
      "logps/rejected": -204.31268310546875,
      "loss": 0.6979650259017944,
      "mean_token_accuracy": 0.37325895577669144,
      "num_tokens": 315621.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.5530824065208435,
      "rewards/margins": 0.3659425973892212,
      "rewards/rejected": -3.91902494430542,
      "step": 97
    },
    {
      "entropy": 1.8011619448661804,
      "epoch": 0.9751243781094527,
      "grad_norm": 2.6068601608276367,
      "learning_rate": 1.9801980198019803e-05,
      "logits/chosen": -3.7408815612054003,
      "logits/rejected": -3.7402128550609928,
      "logps/chosen": -191.71960067749023,
      "logps/rejected": -189.78856658935547,
      "loss": 0.6842182278633118,
      "mean_token_accuracy": 0.36129410564899445,
      "num_tokens": 318586.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -4.195027410984039,
      "rewards/margins": 0.11620645970106125,
      "rewards/rejected": -4.311233758926392,
      "step": 98
    },
    {
      "entropy": 1.663473755121231,
      "epoch": 0.9850746268656716,
      "grad_norm": 2.518073797225952,
      "learning_rate": 1.4851485148514851e-05,
      "logits/chosen": -3.670500626902857,
      "logits/rejected": -3.652272407139832,
      "logps/chosen": -211.21680641174316,
      "logps/rejected": -222.02769088745117,
      "loss": 0.7135859727859497,
      "mean_token_accuracy": 0.3450269028544426,
      "num_tokens": 321874.0,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -5.294373691082001,
      "rewards/margins": -0.0056114718317985535,
      "rewards/rejected": -5.288762152194977,
      "step": 99
    },
    {
      "entropy": 1.6547743380069733,
      "epoch": 0.9950248756218906,
      "grad_norm": 1.6882256269454956,
      "learning_rate": 9.900990099009901e-06,
      "logits/chosen": -3.680744065646073,
      "logits/rejected": -3.708333845446694,
      "logps/chosen": -186.25439071655273,
      "logps/rejected": -214.3514518737793,
      "loss": 0.638218879699707,
      "mean_token_accuracy": 0.34647873789072037,
      "num_tokens": 325099.0,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -4.351440787315369,
      "rewards/margins": 0.6065050661563873,
      "rewards/rejected": -4.957945823669434,
      "step": 100
    },
    {
      "entropy": 1.6515916585922241,
      "epoch": 1.0,
      "grad_norm": 0.471053808927536,
      "learning_rate": 4.950495049504951e-06,
      "logits/chosen": -3.6315787960064374,
      "logits/rejected": -3.648785787208058,
      "logps/chosen": -136.69910430908203,
      "logps/rejected": -147.1917724609375,
      "loss": 0.5413510799407959,
      "mean_token_accuracy": 0.3588639050722122,
      "num_tokens": 326333.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.114208221435547,
      "rewards/margins": 0.6123562753200531,
      "rewards/rejected": -3.726564407348633,
      "step": 101
    }
  ],
  "logging_steps": 1,
  "max_steps": 101,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3309706881761280.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}